Coverage for python/lsst/fgcmcal/utilities.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Utility functions for fgcmcal.
23This file contains utility functions that are used by more than one task,
24and do not need to be part of a task.
25"""
27import numpy as np
28import re
30from lsst.daf.base import PropertyList
31import lsst.daf.persistence as dafPersist
32import lsst.afw.cameraGeom as afwCameraGeom
33import lsst.afw.table as afwTable
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.geom as geom
37from lsst.obs.base import createInitialSkyWcs
38from lsst.obs.base import Instrument
41import fgcm
44FGCM_EXP_FIELD = 'VISIT'
45FGCM_CCD_FIELD = 'DETECTOR'
48def makeConfigDict(config, log, camera, maxIter,
49 resetFitParameters, outputZeropoints, tract=None):
50 """
51 Make the FGCM fit cycle configuration dict
53 Parameters
54 ----------
55 config: `lsst.fgcmcal.FgcmFitCycleConfig`
56 Configuration object
57 log: `lsst.log.Log`
58 LSST log object
59 camera: `lsst.afw.cameraGeom.Camera`
60 Camera from the butler
61 maxIter: `int`
62 Maximum number of iterations
63 resetFitParameters: `bool`
64 Reset fit parameters before fitting?
65 outputZeropoints: `bool`
66 Compute zeropoints for output?
67 tract: `int`, optional
68 Tract number for extending the output file name for debugging.
69 Default is None.
71 Returns
72 -------
73 configDict: `dict`
74 Configuration dictionary for fgcm
75 """
76 # Extract the bands that are _not_ being fit for fgcm configuration
77 notFitBands = [b for b in config.bands if b not in config.fitBands]
79 # process the starColorCuts
80 starColorCutList = []
81 for ccut in config.starColorCuts:
82 parts = ccut.split(',')
83 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
85 # TODO: Having direct access to the mirror area from the camera would be
86 # useful. See DM-16489.
87 # Mirror area in cm**2
88 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
90 # Get approximate average camera gain:
91 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
92 cameraGain = float(np.median(gains))
94 if tract is None:
95 outfileBase = config.outfileBase
96 else:
97 outfileBase = '%s-%06d' % (config.outfileBase, tract)
99 # create a configuration dictionary for fgcmFitCycle
100 configDict = {'outfileBase': outfileBase,
101 'logger': log,
102 'exposureFile': None,
103 'obsFile': None,
104 'indexFile': None,
105 'lutFile': None,
106 'mirrorArea': mirrorArea,
107 'cameraGain': cameraGain,
108 'ccdStartIndex': camera[0].getId(),
109 'expField': FGCM_EXP_FIELD,
110 'ccdField': FGCM_CCD_FIELD,
111 'seeingField': 'DELTA_APER',
112 'fwhmField': 'PSFSIGMA',
113 'skyBrightnessField': 'SKYBACKGROUND',
114 'deepFlag': 'DEEPFLAG', # unused
115 'bands': list(config.bands),
116 'fitBands': list(config.fitBands),
117 'notFitBands': notFitBands,
118 'requiredBands': list(config.requiredBands),
119 'filterToBand': dict(config.physicalFilterMap),
120 'logLevel': 'INFO',
121 'nCore': config.nCore,
122 'nStarPerRun': config.nStarPerRun,
123 'nExpPerRun': config.nExpPerRun,
124 'reserveFraction': config.reserveFraction,
125 'freezeStdAtmosphere': config.freezeStdAtmosphere,
126 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
127 'superStarSubCCDDict': dict(config.superStarSubCcdDict),
128 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
129 'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
130 'superStarSigmaClip': config.superStarSigmaClip,
131 'focalPlaneSigmaClip': config.focalPlaneSigmaClip,
132 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
133 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
134 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
135 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict),
136 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder,
137 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd,
138 'cycleNumber': config.cycleNumber,
139 'maxIter': maxIter,
140 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
141 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
142 'UTBoundary': config.utBoundary,
143 'washMJDs': config.washMjds,
144 'epochMJDs': config.epochMjds,
145 'coatingMJDs': config.coatingMjds,
146 'minObsPerBand': config.minObsPerBand,
147 'latitude': config.latitude,
148 'brightObsGrayMax': config.brightObsGrayMax,
149 'minStarPerCCD': config.minStarPerCcd,
150 'minCCDPerExp': config.minCcdPerExp,
151 'maxCCDGrayErr': config.maxCcdGrayErr,
152 'minStarPerExp': config.minStarPerExp,
153 'minExpPerNight': config.minExpPerNight,
154 'expGrayInitialCut': config.expGrayInitialCut,
155 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
156 'expGrayHighCutDict': dict(config.expGrayHighCutDict),
157 'expGrayRecoverCut': config.expGrayRecoverCut,
158 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
159 'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
160 'refStarSnMin': config.refStarSnMin,
161 'refStarOutlierNSig': config.refStarOutlierNSig,
162 'applyRefStarColorCuts': config.applyRefStarColorCuts,
163 'illegalValue': -9999.0, # internally used by fgcm.
164 'starColorCuts': starColorCutList,
165 'aperCorrFitNBins': config.aperCorrFitNBins,
166 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
167 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
168 'sedTermDict': config.sedterms.toDict()['data'],
169 'colorSplitBands': list(config.colorSplitBands),
170 'sigFgcmMaxErr': config.sigFgcmMaxErr,
171 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
172 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
173 'approxThroughputDict': dict(config.approxThroughputDict),
174 'sigmaCalRange': list(config.sigmaCalRange),
175 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
176 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
177 'sigma0Phot': config.sigma0Phot,
178 'mapLongitudeRef': config.mapLongitudeRef,
179 'mapNSide': config.mapNSide,
180 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
181 'varMinBand': 2,
182 'useRetrievedPwv': False,
183 'useNightlyRetrievedPwv': False,
184 'pwvRetrievalSmoothBlock': 25,
185 'useQuadraticPwv': config.useQuadraticPwv,
186 'useRetrievedTauInit': False,
187 'tauRetrievalMinCCDPerNight': 500,
188 'modelMagErrors': config.modelMagErrors,
189 'instrumentParsPerBand': config.instrumentParsPerBand,
190 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
191 'fitMirrorChromaticity': config.fitMirrorChromaticity,
192 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
193 'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
194 'autoHighCutNSig': config.autoHighCutNSig,
195 'printOnly': False,
196 'quietMode': config.quietMode,
197 'randomSeed': config.randomSeed,
198 'outputStars': False,
199 'clobber': True,
200 'useSedLUT': False,
201 'resetParameters': resetFitParameters,
202 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
203 'outputZeropoints': outputZeropoints}
205 return configDict
208def translateFgcmLut(lutCat, physicalFilterMap):
209 """
210 Translate the FGCM look-up-table into an fgcm-compatible object
212 Parameters
213 ----------
214 lutCat: `lsst.afw.table.BaseCatalog`
215 Catalog describing the FGCM look-up table
216 physicalFilterMap: `dict`
217 Physical filter to band mapping
219 Returns
220 -------
221 fgcmLut: `lsst.fgcm.FgcmLut`
222 Lookup table for FGCM
223 lutIndexVals: `numpy.ndarray`
224 Numpy array with LUT index information for FGCM
225 lutStd: `numpy.ndarray`
226 Numpy array with LUT standard throughput values for FGCM
228 Notes
229 -----
230 After running this code, it is wise to `del lutCat` to clear the memory.
231 """
233 # first we need the lutIndexVals
234 # dtype is set for py2/py3/fits/fgcm compatibility
235 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='a')
236 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='a')
238 # Note that any discrepancies between config values will raise relevant
239 # exceptions in the FGCM code.
241 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
242 lutFilterNames.size),
243 ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
244 lutStdFilterNames.size),
245 ('PMB', 'f8', lutCat[0]['pmb'].size),
246 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
247 ('PMBELEVATION', 'f8'),
248 ('LAMBDANORM', 'f8'),
249 ('PWV', 'f8', lutCat[0]['pwv'].size),
250 ('O3', 'f8', lutCat[0]['o3'].size),
251 ('TAU', 'f8', lutCat[0]['tau'].size),
252 ('ALPHA', 'f8', lutCat[0]['alpha'].size),
253 ('ZENITH', 'f8', lutCat[0]['zenith'].size),
254 ('NCCD', 'i4')])
256 lutIndexVals['FILTERNAMES'][:] = lutFilterNames
257 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
258 lutIndexVals['PMB'][:] = lutCat[0]['pmb']
259 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
260 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
261 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
262 lutIndexVals['PWV'][:] = lutCat[0]['pwv']
263 lutIndexVals['O3'][:] = lutCat[0]['o3']
264 lutIndexVals['TAU'][:] = lutCat[0]['tau']
265 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
266 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
267 lutIndexVals['NCCD'] = lutCat[0]['nCcd']
269 # now we need the Standard Values
270 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
271 ('PWVSTD', 'f8'),
272 ('O3STD', 'f8'),
273 ('TAUSTD', 'f8'),
274 ('ALPHASTD', 'f8'),
275 ('ZENITHSTD', 'f8'),
276 ('LAMBDARANGE', 'f8', 2),
277 ('LAMBDASTEP', 'f8'),
278 ('LAMBDASTD', 'f8', lutFilterNames.size),
279 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
280 ('I0STD', 'f8', lutFilterNames.size),
281 ('I1STD', 'f8', lutFilterNames.size),
282 ('I10STD', 'f8', lutFilterNames.size),
283 ('I2STD', 'f8', lutFilterNames.size),
284 ('LAMBDAB', 'f8', lutFilterNames.size),
285 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
286 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
287 lutStd['PMBSTD'] = lutCat[0]['pmbStd']
288 lutStd['PWVSTD'] = lutCat[0]['pwvStd']
289 lutStd['O3STD'] = lutCat[0]['o3Std']
290 lutStd['TAUSTD'] = lutCat[0]['tauStd']
291 lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
292 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
293 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
294 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
295 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
296 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
297 lutStd['I0STD'][:] = lutCat[0]['i0Std']
298 lutStd['I1STD'][:] = lutCat[0]['i1Std']
299 lutStd['I10STD'][:] = lutCat[0]['i10Std']
300 lutStd['I2STD'][:] = lutCat[0]['i2Std']
301 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
302 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
303 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
305 lutTypes = [row['luttype'] for row in lutCat]
307 # And the flattened look-up-table
308 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
309 ('I1', 'f4')])
311 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
312 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
314 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
315 ('D_O3', 'f4'),
316 ('D_LNTAU', 'f4'),
317 ('D_ALPHA', 'f4'),
318 ('D_SECZENITH', 'f4'),
319 ('D_LNPWV_I1', 'f4'),
320 ('D_O3_I1', 'f4'),
321 ('D_LNTAU_I1', 'f4'),
322 ('D_ALPHA_I1', 'f4'),
323 ('D_SECZENITH_I1', 'f4')])
325 for name in lutDerivFlat.dtype.names:
326 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
328 # The fgcm.FgcmLUT() class copies all the LUT information into special
329 # shared memory objects that will not blow up the memory usage when used
330 # with python multiprocessing. Once all the numbers are copied, the
331 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
332 # will fall out of scope and can be cleaned up by the garbage collector.
333 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
334 filterToBand=physicalFilterMap)
336 return fgcmLut, lutIndexVals, lutStd
339def translateVisitCatalog(visitCat):
340 """
341 Translate the FGCM visit catalog to an fgcm-compatible object
343 Parameters
344 ----------
345 visitCat: `lsst.afw.table.BaseCatalog`
346 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
348 Returns
349 -------
350 fgcmExpInfo: `numpy.ndarray`
351 Numpy array for visit information for FGCM
353 Notes
354 -----
355 After running this code, it is wise to `del visitCat` to clear the memory.
356 """
358 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
359 ('MJD', 'f8'),
360 ('EXPTIME', 'f8'),
361 ('PSFSIGMA', 'f8'),
362 ('DELTA_APER', 'f8'),
363 ('SKYBACKGROUND', 'f8'),
364 ('DEEPFLAG', 'i2'),
365 ('TELHA', 'f8'),
366 ('TELRA', 'f8'),
367 ('TELDEC', 'f8'),
368 ('TELROT', 'f8'),
369 ('PMB', 'f8'),
370 ('FILTERNAME', 'a50')])
371 fgcmExpInfo['VISIT'][:] = visitCat['visit']
372 fgcmExpInfo['MJD'][:] = visitCat['mjd']
373 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
374 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
375 fgcmExpInfo['TELHA'][:] = visitCat['telha']
376 fgcmExpInfo['TELRA'][:] = visitCat['telra']
377 fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
378 fgcmExpInfo['TELROT'][:] = visitCat['telrot']
379 fgcmExpInfo['PMB'][:] = visitCat['pmb']
380 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
381 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
382 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
383 # Note that we have to go through asAstropy() to get a string
384 # array out of an afwTable. This is faster than a row-by-row loop.
385 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter']
387 return fgcmExpInfo
390def computeCcdOffsets(camera, defaultOrientation):
391 """
392 Compute the CCD offsets in ra/dec and x/y space
394 Parameters
395 ----------
396 camera: `lsst.afw.cameraGeom.Camera`
397 defaultOrientation: `float`
398 Default camera orientation (degrees)
400 Returns
401 -------
402 ccdOffsets: `numpy.ndarray`
403 Numpy array with ccd offset information for input to FGCM.
404 Angular units are degrees, and x/y units are pixels.
405 """
406 # TODO: DM-21215 will fully generalize to arbitrary camera orientations
408 # and we need to know the ccd offsets from the camera geometry
409 ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'),
410 ('DELTA_RA', 'f8'),
411 ('DELTA_DEC', 'f8'),
412 ('RA_SIZE', 'f8'),
413 ('DEC_SIZE', 'f8'),
414 ('X_SIZE', 'i4'),
415 ('Y_SIZE', 'i4')])
417 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
418 # since we are looking for relative positions
419 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
421 # TODO: DM-17597 will update testdata_jointcal so that the test data
422 # does not have nan as the boresight angle for HSC data. For the
423 # time being, there is this ungainly hack.
424 if camera.getName() == 'HSC' and np.isnan(defaultOrientation):
425 orientation = 270*geom.degrees
426 else:
427 orientation = defaultOrientation*geom.degrees
428 flipX = False
430 # Create a temporary visitInfo for input to createInitialSkyWcs
431 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
432 boresightRotAngle=orientation,
433 rotType=afwImage.RotType.SKY)
435 for i, detector in enumerate(camera):
436 ccdOffsets['CCDNUM'][i] = detector.getId()
438 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
440 detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS))
441 ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees()
442 ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees()
444 bbox = detector.getBBox()
446 detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin()))
447 detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax()))
449 ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees())
450 ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees())
452 ccdOffsets['X_SIZE'][i] = bbox.getMaxX()
453 ccdOffsets['Y_SIZE'][i] = bbox.getMaxY()
455 return ccdOffsets
458def computeReferencePixelScale(camera):
459 """
460 Compute the median pixel scale in the camera
462 Returns
463 -------
464 pixelScale: `float`
465 Average pixel scale (arcsecond) over the camera
466 """
468 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
469 orientation = 0.0*geom.degrees
470 flipX = False
472 # Create a temporary visitInfo for input to createInitialSkyWcs
473 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
474 boresightRotAngle=orientation,
475 rotType=afwImage.RotType.SKY)
477 pixelScales = np.zeros(len(camera))
478 for i, detector in enumerate(camera):
479 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
480 pixelScales[i] = wcs.getPixelScale().asArcseconds()
482 ok, = np.where(pixelScales > 0.0)
483 return np.median(pixelScales[ok])
486def computeApproxPixelAreaFields(camera):
487 """
488 Compute the approximate pixel area bounded fields from the camera
489 geometry.
491 Parameters
492 ----------
493 camera: `lsst.afw.cameraGeom.Camera`
495 Returns
496 -------
497 approxPixelAreaFields: `dict`
498 Dictionary of approximate area fields, keyed with detector ID
499 """
501 areaScaling = 1. / computeReferencePixelScale(camera)**2.
503 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
504 # since we are looking for relative scales
505 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
507 flipX = False
508 # Create a temporary visitInfo for input to createInitialSkyWcs
509 # The orientation does not matter for the area computation
510 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
511 boresightRotAngle=0.0*geom.degrees,
512 rotType=afwImage.RotType.SKY)
514 approxPixelAreaFields = {}
516 for i, detector in enumerate(camera):
517 key = detector.getId()
519 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
520 bbox = detector.getBBox()
522 areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
523 unit=geom.arcseconds, scaling=areaScaling)
524 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
526 approxPixelAreaFields[key] = approxAreaField
528 return approxPixelAreaFields
531def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
532 """
533 Make the zeropoint schema
535 Parameters
536 ----------
537 superStarChebyshevSize: `int`
538 Length of the superstar chebyshev array
539 zptChebyshevSize: `int`
540 Length of the zeropoint chebyshev array
542 Returns
543 -------
544 zptSchema: `lsst.afw.table.schema`
545 """
547 zptSchema = afwTable.Schema()
549 zptSchema.addField('visit', type=np.int32, doc='Visit number')
550 zptSchema.addField('detector', type=np.int32, doc='Detector ID number')
551 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
552 '1: Photometric, used in fit; '
553 '2: Photometric, not used in fit; '
554 '4: Non-photometric, on partly photometric night; '
555 '8: Non-photometric, on non-photometric night; '
556 '16: No zeropoint could be determined; '
557 '32: Too few stars for reliable gray computation'))
558 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
559 zptSchema.addField('fgcmZptErr', type=np.float64,
560 doc='Error on zeropoint, estimated from repeatability + number of obs')
561 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
562 doc='maximum x/maximum y to scale to apply chebyshev parameters')
563 zptSchema.addField('fgcmfZptCheb', type='ArrayD',
564 size=zptChebyshevSize,
565 doc='Chebyshev parameters (flattened) for zeropoint')
566 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
567 size=superStarChebyshevSize,
568 doc='Chebyshev parameters (flattened) for superStarFlat')
569 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
570 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
571 zptSchema.addField('fgcmR0', type=np.float64,
572 doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
573 zptSchema.addField('fgcmR10', type=np.float64,
574 doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
575 zptSchema.addField('fgcmGry', type=np.float64,
576 doc='Estimated gray extinction relative to atmospheric solution; '
577 'only for fgcmFlag <= 4 (see fgcmFlag) ')
578 zptSchema.addField('fgcmDeltaChrom', type=np.float64,
579 doc='Mean chromatic correction for stars in this ccd; '
580 'only for fgcmFlag <= 4 (see fgcmFlag)')
581 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
582 zptSchema.addField('fgcmTilings', type=np.float64,
583 doc='Number of photometric tilings used for solution for ccd')
584 zptSchema.addField('fgcmFpGry', type=np.float64,
585 doc='Average gray extinction over the full focal plane '
586 '(same for all ccds in a visit)')
587 zptSchema.addField('fgcmFpGryBlue', type=np.float64,
588 doc='Average gray extinction over the full focal plane '
589 'for 25% bluest stars')
590 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
591 doc='Error on Average gray extinction over the full focal plane '
592 'for 25% bluest stars')
593 zptSchema.addField('fgcmFpGryRed', type=np.float64,
594 doc='Average gray extinction over the full focal plane '
595 'for 25% reddest stars')
596 zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
597 doc='Error on Average gray extinction over the full focal plane '
598 'for 25% reddest stars')
599 zptSchema.addField('fgcmFpVar', type=np.float64,
600 doc='Variance of gray extinction over the full focal plane '
601 '(same for all ccds in a visit)')
602 zptSchema.addField('fgcmDust', type=np.float64,
603 doc='Gray dust extinction from the primary/corrector'
604 'at the time of the exposure')
605 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
606 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
607 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
608 doc=('Local background correction from brightest percentile '
609 '(value set by deltaMagBkgOffsetPercentile) calibration '
610 'stars.'))
611 zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
612 zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
614 return zptSchema
617def makeZptCat(zptSchema, zpStruct):
618 """
619 Make the zeropoint catalog for persistence
621 Parameters
622 ----------
623 zptSchema: `lsst.afw.table.Schema`
624 Zeropoint catalog schema
625 zpStruct: `numpy.ndarray`
626 Zeropoint structure from fgcm
628 Returns
629 -------
630 zptCat: `afwTable.BaseCatalog`
631 Zeropoint catalog for persistence
632 """
634 zptCat = afwTable.BaseCatalog(zptSchema)
635 zptCat.reserve(zpStruct.size)
637 for filterName in zpStruct['FILTERNAME']:
638 rec = zptCat.addNew()
639 rec['filtername'] = filterName.decode('utf-8')
641 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD]
642 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD]
643 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
644 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
645 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
646 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
647 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
648 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
649 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
650 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
651 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
652 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
653 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
654 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
655 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
656 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
657 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
658 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
659 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
660 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
661 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
662 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
663 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
664 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
665 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
666 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
667 zptCat['exptime'][:] = zpStruct['EXPTIME']
669 return zptCat
672def makeAtmSchema():
673 """
674 Make the atmosphere schema
676 Returns
677 -------
678 atmSchema: `lsst.afw.table.Schema`
679 """
681 atmSchema = afwTable.Schema()
683 atmSchema.addField('visit', type=np.int32, doc='Visit number')
684 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
685 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
686 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
687 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
688 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
689 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
690 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
691 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
693 return atmSchema
696def makeAtmCat(atmSchema, atmStruct):
697 """
698 Make the atmosphere catalog for persistence
700 Parameters
701 ----------
702 atmSchema: `lsst.afw.table.Schema`
703 Atmosphere catalog schema
704 atmStruct: `numpy.ndarray`
705 Atmosphere structure from fgcm
707 Returns
708 -------
709 atmCat: `lsst.afw.table.BaseCatalog`
710 Atmosphere catalog for persistence
711 """
713 atmCat = afwTable.BaseCatalog(atmSchema)
714 atmCat.resize(atmStruct.size)
716 atmCat['visit'][:] = atmStruct['VISIT']
717 atmCat['pmb'][:] = atmStruct['PMB']
718 atmCat['pwv'][:] = atmStruct['PWV']
719 atmCat['tau'][:] = atmStruct['TAU']
720 atmCat['alpha'][:] = atmStruct['ALPHA']
721 atmCat['o3'][:] = atmStruct['O3']
722 atmCat['secZenith'][:] = atmStruct['SECZENITH']
723 atmCat['cTrans'][:] = atmStruct['CTRANS']
724 atmCat['lamStd'][:] = atmStruct['LAMSTD']
726 return atmCat
729def makeStdSchema(nBands):
730 """
731 Make the standard star schema
733 Parameters
734 ----------
735 nBands: `int`
736 Number of bands in standard star catalog
738 Returns
739 -------
740 stdSchema: `lsst.afw.table.Schema`
741 """
743 stdSchema = afwTable.SimpleTable.makeMinimalSchema()
744 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
745 size=nBands)
746 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
747 size=nBands)
748 stdSchema.addField('mag_std_noabs', type='ArrayF',
749 doc='Standard magnitude (no absolute calibration)',
750 size=nBands)
751 stdSchema.addField('magErr_std', type='ArrayF',
752 doc='Standard magnitude error',
753 size=nBands)
754 stdSchema.addField('npsfcand', type='ArrayI',
755 doc='Number of observations flagged as psf candidates',
756 size=nBands)
758 return stdSchema
761def makeStdCat(stdSchema, stdStruct, goodBands):
762 """
763 Make the standard star catalog for persistence
765 Parameters
766 ----------
767 stdSchema: `lsst.afw.table.Schema`
768 Standard star catalog schema
769 stdStruct: `numpy.ndarray`
770 Standard star structure in FGCM format
771 goodBands: `list`
772 List of good band names used in stdStruct
774 Returns
775 -------
776 stdCat: `lsst.afw.table.BaseCatalog`
777 Standard star catalog for persistence
778 """
780 stdCat = afwTable.SimpleCatalog(stdSchema)
781 stdCat.resize(stdStruct.size)
783 stdCat['id'][:] = stdStruct['FGCM_ID']
784 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
785 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
786 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
787 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
788 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
789 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
790 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
792 md = PropertyList()
793 md.set("BANDS", list(goodBands))
794 stdCat.setMetadata(md)
796 return stdCat
799def computeApertureRadiusFromDataRef(dataRef, fluxField):
800 """
801 Compute the radius associated with a CircularApertureFlux field or
802 associated slot.
804 Parameters
805 ----------
806 dataRef : `lsst.daf.persistence.ButlerDataRef` or
807 `lsst.daf.butler.DeferredDatasetHandle`
808 fluxField : `str`
809 CircularApertureFlux or associated slot.
811 Returns
812 -------
813 apertureRadius : `float`
814 Radius of the aperture field, in pixels.
816 Raises
817 ------
818 RuntimeError: Raised if flux field is not a CircularApertureFlux, ApFlux,
819 or associated slot.
820 """
821 # TODO: Move this method to more general stack method in DM-25775
822 if isinstance(dataRef, dafPersist.ButlerDataRef):
823 # Gen2 dataRef
824 datasetType = dataRef.butlerSubset.datasetType
825 else:
826 # Gen3 dataRef
827 datasetType = dataRef.ref.datasetType.name
829 if datasetType == 'src':
830 schema = dataRef.get(datasetType='src_schema').schema
831 try:
832 fluxFieldName = schema[fluxField].asField().getName()
833 except LookupError:
834 raise RuntimeError("Could not find %s or associated slot in schema." % (fluxField))
835 # This may also raise a RuntimeError
836 apertureRadius = computeApertureRadiusFromName(fluxFieldName)
837 else:
838 # This is a sourceTable_visit
839 apertureRadius = computeApertureRadiusFromName(fluxField)
841 return apertureRadius
844def computeApertureRadiusFromName(fluxField):
845 """
846 Compute the radius associated with a CircularApertureFlux or ApFlux field.
848 Parameters
849 ----------
850 fluxField : `str`
851 CircularApertureFlux or ApFlux
853 Returns
854 -------
855 apertureRadius : `float`
856 Radius of the aperture field, in pixels.
858 Raises
859 ------
860 RuntimeError: Raised if flux field is not a CircularApertureFlux
861 or ApFlux.
862 """
863 # TODO: Move this method to more general stack method in DM-25775
864 m = re.search(r'(CircularApertureFlux|ApFlux)_(\d+)_(\d+)_', fluxField)
866 if m is None:
867 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
869 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
871 return apertureRadius
874def extractReferenceMags(refStars, bands, filterMap):
875 """
876 Extract reference magnitudes from refStars for given bands and
877 associated filterMap.
879 Parameters
880 ----------
881 refStars : `lsst.afw.table.BaseCatalog`
882 FGCM reference star catalog
883 bands : `list`
884 List of bands for calibration
885 filterMap: `dict`
886 FGCM mapping of filter to band
888 Returns
889 -------
890 refMag : `np.ndarray`
891 nstar x nband array of reference magnitudes
892 refMagErr : `np.ndarray`
893 nstar x nband array of reference magnitude errors
894 """
895 # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent
896 # against index errors and allow more flexibility in fitting after
897 # the build stars step.
899 md = refStars.getMetadata()
900 if 'FILTERNAMES' in md:
901 filternames = md.getArray('FILTERNAMES')
903 # The reference catalog that fgcm wants has one entry per band
904 # in the config file
905 refMag = np.zeros((len(refStars), len(bands)),
906 dtype=refStars['refMag'].dtype) + 99.0
907 refMagErr = np.zeros_like(refMag) + 99.0
908 for i, filtername in enumerate(filternames):
909 # We are allowed to run the fit configured so that we do not
910 # use every column in the reference catalog.
911 try:
912 band = filterMap[filtername]
913 except KeyError:
914 continue
915 try:
916 ind = bands.index(band)
917 except ValueError:
918 continue
920 refMag[:, ind] = refStars['refMag'][:, i]
921 refMagErr[:, ind] = refStars['refMagErr'][:, i]
923 else:
924 # Continue to use old catalogs as before.
925 refMag = refStars['refMag'][:, :]
926 refMagErr = refStars['refMagErr'][:, :]
928 return refMag, refMagErr
931def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
932 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
933 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
935 return registry.queryDatasets(datasetType,
936 dataId=quantumDataId,
937 collections=[unboundedCollection])