lsst.fgcmcal gb8350603e9+7916d6664b
Loading...
Searching...
No Matches
utilities.py
Go to the documentation of this file.
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Utility functions for fgcmcal.
22
23This file contains utility functions that are used by more than one task,
24and do not need to be part of a task.
25"""
26
27import numpy as np
28import os
29import re
30
31from lsst.daf.base import PropertyList
32from lsst.daf.butler import Timespan
33import lsst.afw.table as afwTable
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.geom as geom
37from lsst.obs.base import createInitialSkyWcs
38
39import fgcm
40
41
42FGCM_EXP_FIELD = 'VISIT'
43FGCM_CCD_FIELD = 'DETECTOR'
44FGCM_ILLEGAL_VALUE = -9999.0
45
46
47def makeConfigDict(config, log, camera, maxIter,
48 resetFitParameters, outputZeropoints,
49 lutFilterNames, tract=None):
50 """
51 Make the FGCM fit cycle configuration dict
52
53 Parameters
54 ----------
55 config: `lsst.fgcmcal.FgcmFitCycleConfig`
56 Configuration object
57 log: `lsst.log.Log`
58 LSST log object
59 camera: `lsst.afw.cameraGeom.Camera`
60 Camera from the butler
61 maxIter: `int`
62 Maximum number of iterations
63 resetFitParameters: `bool`
64 Reset fit parameters before fitting?
65 outputZeropoints: `bool`
66 Compute zeropoints for output?
67 lutFilterNames : array-like, `str`
68 Array of physical filter names in the LUT.
69 tract: `int`, optional
70 Tract number for extending the output file name for debugging.
71 Default is None.
72
73 Returns
74 -------
75 configDict: `dict`
76 Configuration dictionary for fgcm
77 """
78 # Extract the bands that are _not_ being fit for fgcm configuration
79 notFitBands = [b for b in config.bands if b not in config.fitBands]
80
81 # process the starColorCuts
82 starColorCutList = []
83 for ccut in config.starColorCuts:
84 if ccut == 'NO_DATA':
85 # No color cuts to apply.
86 break
87 parts = ccut.split(',')
88 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
89
90 # process the refStarColorCuts
91 refStarColorCutList = []
92 for ccut in config.refStarColorCuts:
93 if ccut == 'NO_DATA':
94 # No color cuts to apply.
95 break
96 parts = ccut.split(',')
97 refStarColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
98
99 # TODO: Having direct access to the mirror area from the camera would be
100 # useful. See DM-16489.
101 # Mirror area in cm**2
102 if config.mirrorArea is None:
103 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
104 else:
105 # Convert to square cm.
106 mirrorArea = config.mirrorArea * 100.**2.
107
108 # Get approximate average camera gain:
109 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
110 cameraGain = float(np.median(gains))
111
112 # Cut down the filter map to those that are in the LUT
113 filterToBand = {filterName: config.physicalFilterMap[filterName] for
114 filterName in lutFilterNames}
115
116 if tract is None:
117 outfileBase = config.outfileBase
118 else:
119 outfileBase = '%s-%06d' % (config.outfileBase, tract)
120
121 # create a configuration dictionary for fgcmFitCycle
122 configDict = {'outfileBase': outfileBase,
123 'logger': log,
124 'exposureFile': None,
125 'obsFile': None,
126 'indexFile': None,
127 'lutFile': None,
128 'mirrorArea': mirrorArea,
129 'cameraGain': cameraGain,
130 'ccdStartIndex': camera[0].getId(),
131 'expField': FGCM_EXP_FIELD,
132 'ccdField': FGCM_CCD_FIELD,
133 'seeingField': 'DELTA_APER',
134 'fwhmField': 'PSFSIGMA',
135 'skyBrightnessField': 'SKYBACKGROUND',
136 'deepFlag': 'DEEPFLAG', # unused
137 'bands': list(config.bands),
138 'fitBands': list(config.fitBands),
139 'notFitBands': notFitBands,
140 'requiredBands': list(config.requiredBands),
141 'filterToBand': filterToBand,
142 'logLevel': 'INFO',
143 'nCore': config.nCore,
144 'nStarPerRun': config.nStarPerRun,
145 'nExpPerRun': config.nExpPerRun,
146 'reserveFraction': config.reserveFraction,
147 'freezeStdAtmosphere': config.freezeStdAtmosphere,
148 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
149 'superStarSubCCDDict': dict(config.superStarSubCcdDict),
150 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
151 'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
152 'superStarSigmaClip': config.superStarSigmaClip,
153 'superStarPlotCCDResiduals': config.superStarPlotCcdResiduals,
154 'focalPlaneSigmaClip': config.focalPlaneSigmaClip,
155 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
156 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
157 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
158 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict),
159 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder,
160 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd,
161 'cycleNumber': config.cycleNumber,
162 'maxIter': maxIter,
163 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
164 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
165 'UTBoundary': config.utBoundary,
166 'washMJDs': config.washMjds,
167 'epochMJDs': config.epochMjds,
168 'coatingMJDs': config.coatingMjds,
169 'minObsPerBand': config.minObsPerBand,
170 'latitude': config.latitude,
171 'defaultCameraOrientation': config.defaultCameraOrientation,
172 'brightObsGrayMax': config.brightObsGrayMax,
173 'minStarPerCCD': config.minStarPerCcd,
174 'minCCDPerExp': config.minCcdPerExp,
175 'maxCCDGrayErr': config.maxCcdGrayErr,
176 'minStarPerExp': config.minStarPerExp,
177 'minExpPerNight': config.minExpPerNight,
178 'expGrayInitialCut': config.expGrayInitialCut,
179 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
180 'expGrayHighCutDict': dict(config.expGrayHighCutDict),
181 'expGrayRecoverCut': config.expGrayRecoverCut,
182 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
183 'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
184 'refStarSnMin': config.refStarSnMin,
185 'refStarOutlierNSig': config.refStarOutlierNSig,
186 'applyRefStarColorCuts': config.applyRefStarColorCuts,
187 'useExposureReferenceOffset': config.useExposureReferenceOffset,
188 'illegalValue': FGCM_ILLEGAL_VALUE, # internally used by fgcm.
189 'starColorCuts': starColorCutList,
190 'refStarColorCuts': refStarColorCutList,
191 'aperCorrFitNBins': config.aperCorrFitNBins,
192 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
193 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
194 'sedTermDict': config.sedterms.toDict()['data'],
195 'colorSplitBands': list(config.colorSplitBands),
196 'sigFgcmMaxErr': config.sigFgcmMaxErr,
197 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
198 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
199 'approxThroughputDict': dict(config.approxThroughputDict),
200 'sigmaCalRange': list(config.sigmaCalRange),
201 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
202 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
203 'sigma0Phot': config.sigma0Phot,
204 'mapLongitudeRef': config.mapLongitudeRef,
205 'mapNSide': config.mapNSide,
206 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
207 'varMinBand': 2,
208 'useRetrievedPwv': False,
209 'useNightlyRetrievedPwv': False,
210 'pwvRetrievalSmoothBlock': 25,
211 'useQuadraticPwv': config.useQuadraticPwv,
212 'useRetrievedTauInit': False,
213 'tauRetrievalMinCCDPerNight': 500,
214 'modelMagErrors': config.modelMagErrors,
215 'instrumentParsPerBand': config.instrumentParsPerBand,
216 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
217 'fitMirrorChromaticity': config.fitMirrorChromaticity,
218 'fitCCDChromaticityDict': dict(config.fitCcdChromaticityDict),
219 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
220 'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
221 'autoHighCutNSig': config.autoHighCutNSig,
222 'deltaAperInnerRadiusArcsec': config.deltaAperInnerRadiusArcsec,
223 'deltaAperOuterRadiusArcsec': config.deltaAperOuterRadiusArcsec,
224 'deltaAperFitMinNgoodObs': config.deltaAperFitMinNgoodObs,
225 'deltaAperFitPerCcdNx': config.deltaAperFitPerCcdNx,
226 'deltaAperFitPerCcdNy': config.deltaAperFitPerCcdNy,
227 'deltaAperFitSpatialNside': config.deltaAperFitSpatialNside,
228 'doComputeDeltaAperExposures': config.doComputeDeltaAperPerVisit,
229 'doComputeDeltaAperStars': config.doComputeDeltaAperPerStar,
230 'doComputeDeltaAperMap': config.doComputeDeltaAperMap,
231 'doComputeDeltaAperPerCcd': config.doComputeDeltaAperPerCcd,
232 'printOnly': False,
233 'quietMode': config.quietMode,
234 'randomSeed': config.randomSeed,
235 'outputStars': False,
236 'outputPath': os.path.abspath('.'),
237 'clobber': True,
238 'useSedLUT': False,
239 'resetParameters': resetFitParameters,
240 'doPlots': config.doPlots,
241 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
242 'outputZeropoints': outputZeropoints}
243
244 return configDict
245
246
247def translateFgcmLut(lutCat, physicalFilterMap):
248 """
249 Translate the FGCM look-up-table into an fgcm-compatible object
250
251 Parameters
252 ----------
253 lutCat: `lsst.afw.table.BaseCatalog`
254 Catalog describing the FGCM look-up table
255 physicalFilterMap: `dict`
256 Physical filter to band mapping
257
258 Returns
259 -------
260 fgcmLut: `lsst.fgcm.FgcmLut`
261 Lookup table for FGCM
262 lutIndexVals: `numpy.ndarray`
263 Numpy array with LUT index information for FGCM
264 lutStd: `numpy.ndarray`
265 Numpy array with LUT standard throughput values for FGCM
266
267 Notes
268 -----
269 After running this code, it is wise to `del lutCat` to clear the memory.
270 """
271
272 # first we need the lutIndexVals
273 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U')
274 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U')
275
276 # Note that any discrepancies between config values will raise relevant
277 # exceptions in the FGCM code.
278
279 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
280 lutFilterNames.size),
281 ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
282 lutStdFilterNames.size),
283 ('PMB', 'f8', lutCat[0]['pmb'].size),
284 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
285 ('PMBELEVATION', 'f8'),
286 ('LAMBDANORM', 'f8'),
287 ('PWV', 'f8', lutCat[0]['pwv'].size),
288 ('O3', 'f8', lutCat[0]['o3'].size),
289 ('TAU', 'f8', lutCat[0]['tau'].size),
290 ('ALPHA', 'f8', lutCat[0]['alpha'].size),
291 ('ZENITH', 'f8', lutCat[0]['zenith'].size),
292 ('NCCD', 'i4')])
293
294 lutIndexVals['FILTERNAMES'][:] = lutFilterNames
295 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
296 lutIndexVals['PMB'][:] = lutCat[0]['pmb']
297 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
298 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
299 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
300 lutIndexVals['PWV'][:] = lutCat[0]['pwv']
301 lutIndexVals['O3'][:] = lutCat[0]['o3']
302 lutIndexVals['TAU'][:] = lutCat[0]['tau']
303 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
304 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
305 lutIndexVals['NCCD'] = lutCat[0]['nCcd']
306
307 # now we need the Standard Values
308 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
309 ('PWVSTD', 'f8'),
310 ('O3STD', 'f8'),
311 ('TAUSTD', 'f8'),
312 ('ALPHASTD', 'f8'),
313 ('ZENITHSTD', 'f8'),
314 ('LAMBDARANGE', 'f8', 2),
315 ('LAMBDASTEP', 'f8'),
316 ('LAMBDASTD', 'f8', lutFilterNames.size),
317 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
318 ('I0STD', 'f8', lutFilterNames.size),
319 ('I1STD', 'f8', lutFilterNames.size),
320 ('I10STD', 'f8', lutFilterNames.size),
321 ('I2STD', 'f8', lutFilterNames.size),
322 ('LAMBDAB', 'f8', lutFilterNames.size),
323 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
324 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
325 lutStd['PMBSTD'] = lutCat[0]['pmbStd']
326 lutStd['PWVSTD'] = lutCat[0]['pwvStd']
327 lutStd['O3STD'] = lutCat[0]['o3Std']
328 lutStd['TAUSTD'] = lutCat[0]['tauStd']
329 lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
330 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
331 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
332 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
333 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
334 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
335 lutStd['I0STD'][:] = lutCat[0]['i0Std']
336 lutStd['I1STD'][:] = lutCat[0]['i1Std']
337 lutStd['I10STD'][:] = lutCat[0]['i10Std']
338 lutStd['I2STD'][:] = lutCat[0]['i2Std']
339 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
340 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
341 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
342
343 lutTypes = [row['luttype'] for row in lutCat]
344
345 # And the flattened look-up-table
346 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
347 ('I1', 'f4')])
348
349 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
350 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
351
352 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
353 ('D_O3', 'f4'),
354 ('D_LNTAU', 'f4'),
355 ('D_ALPHA', 'f4'),
356 ('D_SECZENITH', 'f4'),
357 ('D_LNPWV_I1', 'f4'),
358 ('D_O3_I1', 'f4'),
359 ('D_LNTAU_I1', 'f4'),
360 ('D_ALPHA_I1', 'f4'),
361 ('D_SECZENITH_I1', 'f4')])
362
363 for name in lutDerivFlat.dtype.names:
364 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
365
366 # The fgcm.FgcmLUT() class copies all the LUT information into special
367 # shared memory objects that will not blow up the memory usage when used
368 # with python multiprocessing. Once all the numbers are copied, the
369 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
370 # will fall out of scope and can be cleaned up by the garbage collector.
371 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
372 filterToBand=physicalFilterMap)
373
374 return fgcmLut, lutIndexVals, lutStd
375
376
378 """
379 Translate the FGCM visit catalog to an fgcm-compatible object
380
381 Parameters
382 ----------
383 visitCat: `lsst.afw.table.BaseCatalog`
384 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
385
386 Returns
387 -------
388 fgcmExpInfo: `numpy.ndarray`
389 Numpy array for visit information for FGCM
390
391 Notes
392 -----
393 After running this code, it is wise to `del visitCat` to clear the memory.
394 """
395
396 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
397 ('MJD', 'f8'),
398 ('EXPTIME', 'f8'),
399 ('PSFSIGMA', 'f8'),
400 ('DELTA_APER', 'f8'),
401 ('SKYBACKGROUND', 'f8'),
402 ('DEEPFLAG', 'i2'),
403 ('TELHA', 'f8'),
404 ('TELRA', 'f8'),
405 ('TELDEC', 'f8'),
406 ('TELROT', 'f8'),
407 ('PMB', 'f8'),
408 ('FILTERNAME', 'a50')])
409 fgcmExpInfo['VISIT'][:] = visitCat['visit']
410 fgcmExpInfo['MJD'][:] = visitCat['mjd']
411 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
412 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
413 fgcmExpInfo['TELHA'][:] = visitCat['telha']
414 fgcmExpInfo['TELRA'][:] = visitCat['telra']
415 fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
416 fgcmExpInfo['TELROT'][:] = visitCat['telrot']
417 fgcmExpInfo['PMB'][:] = visitCat['pmb']
418 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
419 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
420 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
421 # Note that we have to go through asAstropy() to get a string
422 # array out of an afwTable. This is faster than a row-by-row loop.
423 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter']
424
425 return fgcmExpInfo
426
427
429 """
430 Compute the median pixel scale in the camera
431
432 Returns
433 -------
434 pixelScale: `float`
435 Average pixel scale (arcsecond) over the camera
436 """
437
438 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
439 orientation = 0.0*geom.degrees
440 flipX = False
441
442 # Create a temporary visitInfo for input to createInitialSkyWcs
443 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
444 boresightRotAngle=orientation,
445 rotType=afwImage.RotType.SKY)
446
447 pixelScales = np.zeros(len(camera))
448 for i, detector in enumerate(camera):
449 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
450 pixelScales[i] = wcs.getPixelScale().asArcseconds()
451
452 ok, = np.where(pixelScales > 0.0)
453 return np.median(pixelScales[ok])
454
455
457 """
458 Compute the approximate pixel area bounded fields from the camera
459 geometry.
460
461 Parameters
462 ----------
463 camera: `lsst.afw.cameraGeom.Camera`
464
465 Returns
466 -------
467 approxPixelAreaFields: `dict`
468 Dictionary of approximate area fields, keyed with detector ID
469 """
470
471 areaScaling = 1. / computeReferencePixelScale(camera)**2.
472
473 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
474 # since we are looking for relative scales
475 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
476
477 flipX = False
478 # Create a temporary visitInfo for input to createInitialSkyWcs
479 # The orientation does not matter for the area computation
480 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
481 boresightRotAngle=0.0*geom.degrees,
482 rotType=afwImage.RotType.SKY)
483
484 approxPixelAreaFields = {}
485
486 for i, detector in enumerate(camera):
487 key = detector.getId()
488
489 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
490 bbox = detector.getBBox()
491
492 areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
493 unit=geom.arcseconds, scaling=areaScaling)
494 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
495
496 approxPixelAreaFields[key] = approxAreaField
497
498 return approxPixelAreaFields
499
500
501def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
502 """
503 Make the zeropoint schema
504
505 Parameters
506 ----------
507 superStarChebyshevSize: `int`
508 Length of the superstar chebyshev array
509 zptChebyshevSize: `int`
510 Length of the zeropoint chebyshev array
511
512 Returns
513 -------
514 zptSchema: `lsst.afw.table.schema`
515 """
516
517 zptSchema = afwTable.Schema()
518
519 zptSchema.addField('visit', type=np.int64, doc='Visit number')
520 zptSchema.addField('detector', type=np.int32, doc='Detector ID number')
521 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
522 '1: Photometric, used in fit; '
523 '2: Photometric, not used in fit; '
524 '4: Non-photometric, on partly photometric night; '
525 '8: Non-photometric, on non-photometric night; '
526 '16: No zeropoint could be determined; '
527 '32: Too few stars for reliable gray computation'))
528 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
529 zptSchema.addField('fgcmZptErr', type=np.float64,
530 doc='Error on zeropoint, estimated from repeatability + number of obs')
531 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
532 doc='maximum x/maximum y to scale to apply chebyshev parameters')
533 zptSchema.addField('fgcmfZptCheb', type='ArrayD',
534 size=zptChebyshevSize,
535 doc='Chebyshev parameters (flattened) for zeropoint')
536 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
537 size=superStarChebyshevSize,
538 doc='Chebyshev parameters (flattened) for superStarFlat')
539 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
540 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
541 zptSchema.addField('fgcmR0', type=np.float64,
542 doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
543 zptSchema.addField('fgcmR10', type=np.float64,
544 doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
545 zptSchema.addField('fgcmGry', type=np.float64,
546 doc='Estimated gray extinction relative to atmospheric solution; '
547 'only for fgcmFlag <= 4 (see fgcmFlag) ')
548 zptSchema.addField('fgcmDeltaChrom', type=np.float64,
549 doc='Mean chromatic correction for stars in this ccd; '
550 'only for fgcmFlag <= 4 (see fgcmFlag)')
551 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
552 zptSchema.addField('fgcmTilings', type=np.float64,
553 doc='Number of photometric tilings used for solution for ccd')
554 zptSchema.addField('fgcmFpGry', type=np.float64,
555 doc='Average gray extinction over the full focal plane '
556 '(same for all ccds in a visit)')
557 zptSchema.addField('fgcmFpGryBlue', type=np.float64,
558 doc='Average gray extinction over the full focal plane '
559 'for 25% bluest stars')
560 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
561 doc='Error on Average gray extinction over the full focal plane '
562 'for 25% bluest stars')
563 zptSchema.addField('fgcmFpGryRed', type=np.float64,
564 doc='Average gray extinction over the full focal plane '
565 'for 25% reddest stars')
566 zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
567 doc='Error on Average gray extinction over the full focal plane '
568 'for 25% reddest stars')
569 zptSchema.addField('fgcmFpVar', type=np.float64,
570 doc='Variance of gray extinction over the full focal plane '
571 '(same for all ccds in a visit)')
572 zptSchema.addField('fgcmDust', type=np.float64,
573 doc='Gray dust extinction from the primary/corrector'
574 'at the time of the exposure')
575 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
576 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
577 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
578 doc=('Local background correction from brightest percentile '
579 '(value set by deltaMagBkgOffsetPercentile) calibration '
580 'stars.'))
581 zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
582 zptSchema.addField('filtername', type=str, size=30, doc='Filter name')
583
584 return zptSchema
585
586
587def makeZptCat(zptSchema, zpStruct):
588 """
589 Make the zeropoint catalog for persistence
590
591 Parameters
592 ----------
593 zptSchema: `lsst.afw.table.Schema`
594 Zeropoint catalog schema
595 zpStruct: `numpy.ndarray`
596 Zeropoint structure from fgcm
597
598 Returns
599 -------
600 zptCat: `afwTable.BaseCatalog`
601 Zeropoint catalog for persistence
602 """
603
604 zptCat = afwTable.BaseCatalog(zptSchema)
605 zptCat.reserve(zpStruct.size)
606
607 for filterName in zpStruct['FILTERNAME']:
608 rec = zptCat.addNew()
609 rec['filtername'] = filterName.decode('utf-8')
610
611 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD]
612 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD]
613 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
614 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
615 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
616 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
617 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
618 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
619 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
620 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
621 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
622 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
623 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
624 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
625 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
626 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
627 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
628 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
629 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
630 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
631 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
632 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
633 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
634 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
635 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
636 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
637 zptCat['exptime'][:] = zpStruct['EXPTIME']
638
639 return zptCat
640
641
643 """
644 Make the atmosphere schema
645
646 Returns
647 -------
648 atmSchema: `lsst.afw.table.Schema`
649 """
650
651 atmSchema = afwTable.Schema()
652
653 atmSchema.addField('visit', type=np.int64, doc='Visit number')
654 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
655 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
656 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
657 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
658 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
659 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
660 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
661 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
662
663 return atmSchema
664
665
666def makeAtmCat(atmSchema, atmStruct):
667 """
668 Make the atmosphere catalog for persistence
669
670 Parameters
671 ----------
672 atmSchema: `lsst.afw.table.Schema`
673 Atmosphere catalog schema
674 atmStruct: `numpy.ndarray`
675 Atmosphere structure from fgcm
676
677 Returns
678 -------
679 atmCat: `lsst.afw.table.BaseCatalog`
680 Atmosphere catalog for persistence
681 """
682
683 atmCat = afwTable.BaseCatalog(atmSchema)
684 atmCat.resize(atmStruct.size)
685
686 atmCat['visit'][:] = atmStruct['VISIT']
687 atmCat['pmb'][:] = atmStruct['PMB']
688 atmCat['pwv'][:] = atmStruct['PWV']
689 atmCat['tau'][:] = atmStruct['TAU']
690 atmCat['alpha'][:] = atmStruct['ALPHA']
691 atmCat['o3'][:] = atmStruct['O3']
692 atmCat['secZenith'][:] = atmStruct['SECZENITH']
693 atmCat['cTrans'][:] = atmStruct['CTRANS']
694 atmCat['lamStd'][:] = atmStruct['LAMSTD']
695
696 return atmCat
697
698
699def makeStdSchema(nBands):
700 """
701 Make the standard star schema
702
703 Parameters
704 ----------
705 nBands: `int`
706 Number of bands in standard star catalog
707
708 Returns
709 -------
710 stdSchema: `lsst.afw.table.Schema`
711 """
712
713 stdSchema = afwTable.SimpleTable.makeMinimalSchema()
714 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
715 size=nBands)
716 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
717 size=nBands)
718 stdSchema.addField('mag_std_noabs', type='ArrayF',
719 doc='Standard magnitude (no absolute calibration)',
720 size=nBands)
721 stdSchema.addField('magErr_std', type='ArrayF',
722 doc='Standard magnitude error',
723 size=nBands)
724 stdSchema.addField('npsfcand', type='ArrayI',
725 doc='Number of observations flagged as psf candidates',
726 size=nBands)
727 stdSchema.addField('delta_aper', type='ArrayF',
728 doc='Delta mag (small - large aperture)',
729 size=nBands)
730
731 return stdSchema
732
733
734def makeStdCat(stdSchema, stdStruct, goodBands):
735 """
736 Make the standard star catalog for persistence
737
738 Parameters
739 ----------
740 stdSchema: `lsst.afw.table.Schema`
741 Standard star catalog schema
742 stdStruct: `numpy.ndarray`
743 Standard star structure in FGCM format
744 goodBands: `list`
745 List of good band names used in stdStruct
746
747 Returns
748 -------
749 stdCat: `lsst.afw.table.BaseCatalog`
750 Standard star catalog for persistence
751 """
752
753 stdCat = afwTable.SimpleCatalog(stdSchema)
754 stdCat.resize(stdStruct.size)
755
756 stdCat['id'][:] = stdStruct['FGCM_ID']
757 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
758 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
759 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
760 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
761 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
762 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
763 if 'NPSFCAND' in stdStruct.dtype.names:
764 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
765 stdCat['delta_aper'][:, :] = stdStruct['DELTA_APER'][:, :]
766
767 md = PropertyList()
768 md.set("BANDS", list(goodBands))
769 stdCat.setMetadata(md)
770
771 return stdCat
772
773
775 """
776 Compute the radius associated with a CircularApertureFlux or ApFlux field.
777
778 Parameters
779 ----------
780 fluxField : `str`
781 CircularApertureFlux or ApFlux
782
783 Returns
784 -------
785 apertureRadius : `float`
786 Radius of the aperture field, in pixels.
787
788 Raises
789 ------
790 RuntimeError: Raised if flux field is not a CircularApertureFlux,
791 ApFlux, or apFlux.
792 """
793 # TODO: Move this method to more general stack method in DM-25775
794 m = re.search(r'(CircularApertureFlux|ApFlux|apFlux)_(\d+)_(\d+)_', fluxField)
795
796 if m is None:
797 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
798
799 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
800
801 return apertureRadius
802
803
804def extractReferenceMags(refStars, bands, filterMap):
805 """
806 Extract reference magnitudes from refStars for given bands and
807 associated filterMap.
808
809 Parameters
810 ----------
811 refStars : `astropy.table.Table` or `lsst.afw.table.BaseCatalog`
812 FGCM reference star catalog.
813 bands : `list`
814 List of bands for calibration.
815 filterMap: `dict`
816 FGCM mapping of filter to band.
817
818 Returns
819 -------
820 refMag : `np.ndarray`
821 nstar x nband array of reference magnitudes.
822 refMagErr : `np.ndarray`
823 nstar x nband array of reference magnitude errors.
824 """
825 hasAstropyMeta = False
826 try:
827 meta = refStars.meta
828 hasAstropyMeta = True
829 except AttributeError:
830 meta = refStars.getMetadata()
831
832 if 'FILTERNAMES' in meta:
833 if hasAstropyMeta:
834 filternames = meta['FILTERNAMES']
835 else:
836 filternames = meta.getArray('FILTERNAMES')
837
838 # The reference catalog that fgcm wants has one entry per band
839 # in the config file
840 refMag = np.zeros((len(refStars), len(bands)),
841 dtype=refStars['refMag'].dtype) + 99.0
842 refMagErr = np.zeros_like(refMag) + 99.0
843 for i, filtername in enumerate(filternames):
844 # We are allowed to run the fit configured so that we do not
845 # use every column in the reference catalog.
846 try:
847 band = filterMap[filtername]
848 except KeyError:
849 continue
850 try:
851 ind = bands.index(band)
852 except ValueError:
853 continue
854
855 refMag[:, ind] = refStars['refMag'][:, i]
856 refMagErr[:, ind] = refStars['refMagErr'][:, i]
857 else:
858 raise RuntimeError("FGCM reference stars missing FILTERNAMES metadata.")
859
860 return refMag, refMagErr
861
862
863def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
864 # For static calibrations, we search with a timespan that has unbounded
865 # begin and end; we'll get an error if there's more than one match (because
866 # then it's not static).
867 timespan = Timespan(begin=None, end=None)
868 result = []
869 # First iterate over all of the data IDs for this dataset type that are
870 # consistent with the quantum data ID.
871 for dataId in registry.queryDataIds(datasetType.dimensions, dataId=quantumDataId):
872 # Find the dataset with this data ID using the unbounded timespan.
873 if ref := registry.findDataset(datasetType, dataId, collections=collections, timespan=timespan):
874 result.append(ref)
875 return result
translateVisitCatalog(visitCat)
Definition utilities.py:377
computeApertureRadiusFromName(fluxField)
Definition utilities.py:774
makeStdCat(stdSchema, stdStruct, goodBands)
Definition utilities.py:734
makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
Definition utilities.py:49
makeZptCat(zptSchema, zpStruct)
Definition utilities.py:587
lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition utilities.py:863
computeReferencePixelScale(camera)
Definition utilities.py:428
makeAtmCat(atmSchema, atmStruct)
Definition utilities.py:666
translateFgcmLut(lutCat, physicalFilterMap)
Definition utilities.py:247
computeApproxPixelAreaFields(camera)
Definition utilities.py:456
makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition utilities.py:501
extractReferenceMags(refStars, bands, filterMap)
Definition utilities.py:804