Coverage for python/lsst/fgcmcal/utilities.py: 9%
274 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-13 09:25 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-13 09:25 +0000
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Utility functions for fgcmcal.
23This file contains utility functions that are used by more than one task,
24and do not need to be part of a task.
25"""
27import numpy as np
28import os
29import re
31from lsst.daf.base import PropertyList
32import lsst.afw.table as afwTable
33import lsst.afw.image as afwImage
34import lsst.afw.math as afwMath
35import lsst.geom as geom
36from lsst.obs.base import createInitialSkyWcs
37from lsst.pipe.base import Instrument
39import fgcm
42FGCM_EXP_FIELD = 'VISIT'
43FGCM_CCD_FIELD = 'DETECTOR'
44FGCM_ILLEGAL_VALUE = -9999.0
47def makeConfigDict(config, log, camera, maxIter,
48 resetFitParameters, outputZeropoints,
49 lutFilterNames, tract=None):
50 """
51 Make the FGCM fit cycle configuration dict
53 Parameters
54 ----------
55 config: `lsst.fgcmcal.FgcmFitCycleConfig`
56 Configuration object
57 log: `lsst.log.Log`
58 LSST log object
59 camera: `lsst.afw.cameraGeom.Camera`
60 Camera from the butler
61 maxIter: `int`
62 Maximum number of iterations
63 resetFitParameters: `bool`
64 Reset fit parameters before fitting?
65 outputZeropoints: `bool`
66 Compute zeropoints for output?
67 lutFilterNames : array-like, `str`
68 Array of physical filter names in the LUT.
69 tract: `int`, optional
70 Tract number for extending the output file name for debugging.
71 Default is None.
73 Returns
74 -------
75 configDict: `dict`
76 Configuration dictionary for fgcm
77 """
78 # Extract the bands that are _not_ being fit for fgcm configuration
79 notFitBands = [b for b in config.bands if b not in config.fitBands]
81 # process the starColorCuts
82 starColorCutList = []
83 for ccut in config.starColorCuts:
84 if ccut == 'NO_DATA':
85 # No color cuts to apply.
86 break
87 parts = ccut.split(',')
88 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
90 # process the refStarColorCuts
91 refStarColorCutList = []
92 for ccut in config.refStarColorCuts:
93 if ccut == 'NO_DATA':
94 # No color cuts to apply.
95 break
96 parts = ccut.split(',')
97 refStarColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
99 # TODO: Having direct access to the mirror area from the camera would be
100 # useful. See DM-16489.
101 # Mirror area in cm**2
102 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
104 # Get approximate average camera gain:
105 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
106 cameraGain = float(np.median(gains))
108 # Cut down the filter map to those that are in the LUT
109 filterToBand = {filterName: config.physicalFilterMap[filterName] for
110 filterName in lutFilterNames}
112 if tract is None:
113 outfileBase = config.outfileBase
114 else:
115 outfileBase = '%s-%06d' % (config.outfileBase, tract)
117 # create a configuration dictionary for fgcmFitCycle
118 configDict = {'outfileBase': outfileBase,
119 'logger': log,
120 'exposureFile': None,
121 'obsFile': None,
122 'indexFile': None,
123 'lutFile': None,
124 'mirrorArea': mirrorArea,
125 'cameraGain': cameraGain,
126 'ccdStartIndex': camera[0].getId(),
127 'expField': FGCM_EXP_FIELD,
128 'ccdField': FGCM_CCD_FIELD,
129 'seeingField': 'DELTA_APER',
130 'fwhmField': 'PSFSIGMA',
131 'skyBrightnessField': 'SKYBACKGROUND',
132 'deepFlag': 'DEEPFLAG', # unused
133 'bands': list(config.bands),
134 'fitBands': list(config.fitBands),
135 'notFitBands': notFitBands,
136 'requiredBands': list(config.requiredBands),
137 'filterToBand': filterToBand,
138 'logLevel': 'INFO',
139 'nCore': config.nCore,
140 'nStarPerRun': config.nStarPerRun,
141 'nExpPerRun': config.nExpPerRun,
142 'reserveFraction': config.reserveFraction,
143 'freezeStdAtmosphere': config.freezeStdAtmosphere,
144 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
145 'superStarSubCCDDict': dict(config.superStarSubCcdDict),
146 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
147 'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
148 'superStarSigmaClip': config.superStarSigmaClip,
149 'focalPlaneSigmaClip': config.focalPlaneSigmaClip,
150 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
151 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
152 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
153 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict),
154 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder,
155 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd,
156 'cycleNumber': config.cycleNumber,
157 'maxIter': maxIter,
158 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
159 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
160 'UTBoundary': config.utBoundary,
161 'washMJDs': config.washMjds,
162 'epochMJDs': config.epochMjds,
163 'coatingMJDs': config.coatingMjds,
164 'minObsPerBand': config.minObsPerBand,
165 'latitude': config.latitude,
166 'defaultCameraOrientation': config.defaultCameraOrientation,
167 'brightObsGrayMax': config.brightObsGrayMax,
168 'minStarPerCCD': config.minStarPerCcd,
169 'minCCDPerExp': config.minCcdPerExp,
170 'maxCCDGrayErr': config.maxCcdGrayErr,
171 'minStarPerExp': config.minStarPerExp,
172 'minExpPerNight': config.minExpPerNight,
173 'expGrayInitialCut': config.expGrayInitialCut,
174 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
175 'expGrayHighCutDict': dict(config.expGrayHighCutDict),
176 'expGrayRecoverCut': config.expGrayRecoverCut,
177 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
178 'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
179 'refStarSnMin': config.refStarSnMin,
180 'refStarOutlierNSig': config.refStarOutlierNSig,
181 'applyRefStarColorCuts': config.applyRefStarColorCuts,
182 'useExposureReferenceOffset': config.useExposureReferenceOffset,
183 'illegalValue': FGCM_ILLEGAL_VALUE, # internally used by fgcm.
184 'starColorCuts': starColorCutList,
185 'refStarColorCuts': refStarColorCutList,
186 'aperCorrFitNBins': config.aperCorrFitNBins,
187 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
188 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
189 'sedTermDict': config.sedterms.toDict()['data'],
190 'colorSplitBands': list(config.colorSplitBands),
191 'sigFgcmMaxErr': config.sigFgcmMaxErr,
192 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
193 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
194 'approxThroughputDict': dict(config.approxThroughputDict),
195 'sigmaCalRange': list(config.sigmaCalRange),
196 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
197 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
198 'sigma0Phot': config.sigma0Phot,
199 'mapLongitudeRef': config.mapLongitudeRef,
200 'mapNSide': config.mapNSide,
201 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
202 'varMinBand': 2,
203 'useRetrievedPwv': False,
204 'useNightlyRetrievedPwv': False,
205 'pwvRetrievalSmoothBlock': 25,
206 'useQuadraticPwv': config.useQuadraticPwv,
207 'useRetrievedTauInit': False,
208 'tauRetrievalMinCCDPerNight': 500,
209 'modelMagErrors': config.modelMagErrors,
210 'instrumentParsPerBand': config.instrumentParsPerBand,
211 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
212 'fitMirrorChromaticity': config.fitMirrorChromaticity,
213 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
214 'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
215 'autoHighCutNSig': config.autoHighCutNSig,
216 'deltaAperInnerRadiusArcsec': config.deltaAperInnerRadiusArcsec,
217 'deltaAperOuterRadiusArcsec': config.deltaAperOuterRadiusArcsec,
218 'deltaAperFitMinNgoodObs': config.deltaAperFitMinNgoodObs,
219 'deltaAperFitPerCcdNx': config.deltaAperFitPerCcdNx,
220 'deltaAperFitPerCcdNy': config.deltaAperFitPerCcdNy,
221 'deltaAperFitSpatialNside': config.deltaAperFitSpatialNside,
222 'doComputeDeltaAperExposures': config.doComputeDeltaAperPerVisit,
223 'doComputeDeltaAperStars': config.doComputeDeltaAperPerStar,
224 'doComputeDeltaAperMap': config.doComputeDeltaAperMap,
225 'doComputeDeltaAperPerCcd': config.doComputeDeltaAperPerCcd,
226 'printOnly': False,
227 'quietMode': config.quietMode,
228 'randomSeed': config.randomSeed,
229 'outputStars': False,
230 'outputPath': os.path.abspath('.'),
231 'clobber': True,
232 'useSedLUT': False,
233 'resetParameters': resetFitParameters,
234 'doPlots': config.doPlots,
235 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
236 'outputZeropoints': outputZeropoints}
238 return configDict
241def translateFgcmLut(lutCat, physicalFilterMap):
242 """
243 Translate the FGCM look-up-table into an fgcm-compatible object
245 Parameters
246 ----------
247 lutCat: `lsst.afw.table.BaseCatalog`
248 Catalog describing the FGCM look-up table
249 physicalFilterMap: `dict`
250 Physical filter to band mapping
252 Returns
253 -------
254 fgcmLut: `lsst.fgcm.FgcmLut`
255 Lookup table for FGCM
256 lutIndexVals: `numpy.ndarray`
257 Numpy array with LUT index information for FGCM
258 lutStd: `numpy.ndarray`
259 Numpy array with LUT standard throughput values for FGCM
261 Notes
262 -----
263 After running this code, it is wise to `del lutCat` to clear the memory.
264 """
266 # first we need the lutIndexVals
267 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U')
268 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U')
270 # Note that any discrepancies between config values will raise relevant
271 # exceptions in the FGCM code.
273 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
274 lutFilterNames.size),
275 ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
276 lutStdFilterNames.size),
277 ('PMB', 'f8', lutCat[0]['pmb'].size),
278 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
279 ('PMBELEVATION', 'f8'),
280 ('LAMBDANORM', 'f8'),
281 ('PWV', 'f8', lutCat[0]['pwv'].size),
282 ('O3', 'f8', lutCat[0]['o3'].size),
283 ('TAU', 'f8', lutCat[0]['tau'].size),
284 ('ALPHA', 'f8', lutCat[0]['alpha'].size),
285 ('ZENITH', 'f8', lutCat[0]['zenith'].size),
286 ('NCCD', 'i4')])
288 lutIndexVals['FILTERNAMES'][:] = lutFilterNames
289 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
290 lutIndexVals['PMB'][:] = lutCat[0]['pmb']
291 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
292 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
293 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
294 lutIndexVals['PWV'][:] = lutCat[0]['pwv']
295 lutIndexVals['O3'][:] = lutCat[0]['o3']
296 lutIndexVals['TAU'][:] = lutCat[0]['tau']
297 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
298 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
299 lutIndexVals['NCCD'] = lutCat[0]['nCcd']
301 # now we need the Standard Values
302 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
303 ('PWVSTD', 'f8'),
304 ('O3STD', 'f8'),
305 ('TAUSTD', 'f8'),
306 ('ALPHASTD', 'f8'),
307 ('ZENITHSTD', 'f8'),
308 ('LAMBDARANGE', 'f8', 2),
309 ('LAMBDASTEP', 'f8'),
310 ('LAMBDASTD', 'f8', lutFilterNames.size),
311 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
312 ('I0STD', 'f8', lutFilterNames.size),
313 ('I1STD', 'f8', lutFilterNames.size),
314 ('I10STD', 'f8', lutFilterNames.size),
315 ('I2STD', 'f8', lutFilterNames.size),
316 ('LAMBDAB', 'f8', lutFilterNames.size),
317 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
318 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
319 lutStd['PMBSTD'] = lutCat[0]['pmbStd']
320 lutStd['PWVSTD'] = lutCat[0]['pwvStd']
321 lutStd['O3STD'] = lutCat[0]['o3Std']
322 lutStd['TAUSTD'] = lutCat[0]['tauStd']
323 lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
324 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
325 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
326 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
327 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
328 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
329 lutStd['I0STD'][:] = lutCat[0]['i0Std']
330 lutStd['I1STD'][:] = lutCat[0]['i1Std']
331 lutStd['I10STD'][:] = lutCat[0]['i10Std']
332 lutStd['I2STD'][:] = lutCat[0]['i2Std']
333 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
334 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
335 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
337 lutTypes = [row['luttype'] for row in lutCat]
339 # And the flattened look-up-table
340 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
341 ('I1', 'f4')])
343 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
344 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
346 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
347 ('D_O3', 'f4'),
348 ('D_LNTAU', 'f4'),
349 ('D_ALPHA', 'f4'),
350 ('D_SECZENITH', 'f4'),
351 ('D_LNPWV_I1', 'f4'),
352 ('D_O3_I1', 'f4'),
353 ('D_LNTAU_I1', 'f4'),
354 ('D_ALPHA_I1', 'f4'),
355 ('D_SECZENITH_I1', 'f4')])
357 for name in lutDerivFlat.dtype.names:
358 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
360 # The fgcm.FgcmLUT() class copies all the LUT information into special
361 # shared memory objects that will not blow up the memory usage when used
362 # with python multiprocessing. Once all the numbers are copied, the
363 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
364 # will fall out of scope and can be cleaned up by the garbage collector.
365 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
366 filterToBand=physicalFilterMap)
368 return fgcmLut, lutIndexVals, lutStd
371def translateVisitCatalog(visitCat):
372 """
373 Translate the FGCM visit catalog to an fgcm-compatible object
375 Parameters
376 ----------
377 visitCat: `lsst.afw.table.BaseCatalog`
378 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
380 Returns
381 -------
382 fgcmExpInfo: `numpy.ndarray`
383 Numpy array for visit information for FGCM
385 Notes
386 -----
387 After running this code, it is wise to `del visitCat` to clear the memory.
388 """
390 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
391 ('MJD', 'f8'),
392 ('EXPTIME', 'f8'),
393 ('PSFSIGMA', 'f8'),
394 ('DELTA_APER', 'f8'),
395 ('SKYBACKGROUND', 'f8'),
396 ('DEEPFLAG', 'i2'),
397 ('TELHA', 'f8'),
398 ('TELRA', 'f8'),
399 ('TELDEC', 'f8'),
400 ('TELROT', 'f8'),
401 ('PMB', 'f8'),
402 ('FILTERNAME', 'a50')])
403 fgcmExpInfo['VISIT'][:] = visitCat['visit']
404 fgcmExpInfo['MJD'][:] = visitCat['mjd']
405 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
406 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
407 fgcmExpInfo['TELHA'][:] = visitCat['telha']
408 fgcmExpInfo['TELRA'][:] = visitCat['telra']
409 fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
410 fgcmExpInfo['TELROT'][:] = visitCat['telrot']
411 fgcmExpInfo['PMB'][:] = visitCat['pmb']
412 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
413 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
414 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
415 # Note that we have to go through asAstropy() to get a string
416 # array out of an afwTable. This is faster than a row-by-row loop.
417 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter']
419 return fgcmExpInfo
422def computeReferencePixelScale(camera):
423 """
424 Compute the median pixel scale in the camera
426 Returns
427 -------
428 pixelScale: `float`
429 Average pixel scale (arcsecond) over the camera
430 """
432 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
433 orientation = 0.0*geom.degrees
434 flipX = False
436 # Create a temporary visitInfo for input to createInitialSkyWcs
437 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
438 boresightRotAngle=orientation,
439 rotType=afwImage.RotType.SKY)
441 pixelScales = np.zeros(len(camera))
442 for i, detector in enumerate(camera):
443 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
444 pixelScales[i] = wcs.getPixelScale().asArcseconds()
446 ok, = np.where(pixelScales > 0.0)
447 return np.median(pixelScales[ok])
450def computeApproxPixelAreaFields(camera):
451 """
452 Compute the approximate pixel area bounded fields from the camera
453 geometry.
455 Parameters
456 ----------
457 camera: `lsst.afw.cameraGeom.Camera`
459 Returns
460 -------
461 approxPixelAreaFields: `dict`
462 Dictionary of approximate area fields, keyed with detector ID
463 """
465 areaScaling = 1. / computeReferencePixelScale(camera)**2.
467 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
468 # since we are looking for relative scales
469 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
471 flipX = False
472 # Create a temporary visitInfo for input to createInitialSkyWcs
473 # The orientation does not matter for the area computation
474 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
475 boresightRotAngle=0.0*geom.degrees,
476 rotType=afwImage.RotType.SKY)
478 approxPixelAreaFields = {}
480 for i, detector in enumerate(camera):
481 key = detector.getId()
483 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
484 bbox = detector.getBBox()
486 areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
487 unit=geom.arcseconds, scaling=areaScaling)
488 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
490 approxPixelAreaFields[key] = approxAreaField
492 return approxPixelAreaFields
495def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
496 """
497 Make the zeropoint schema
499 Parameters
500 ----------
501 superStarChebyshevSize: `int`
502 Length of the superstar chebyshev array
503 zptChebyshevSize: `int`
504 Length of the zeropoint chebyshev array
506 Returns
507 -------
508 zptSchema: `lsst.afw.table.schema`
509 """
511 zptSchema = afwTable.Schema()
513 zptSchema.addField('visit', type=np.int64, doc='Visit number')
514 zptSchema.addField('detector', type=np.int32, doc='Detector ID number')
515 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
516 '1: Photometric, used in fit; '
517 '2: Photometric, not used in fit; '
518 '4: Non-photometric, on partly photometric night; '
519 '8: Non-photometric, on non-photometric night; '
520 '16: No zeropoint could be determined; '
521 '32: Too few stars for reliable gray computation'))
522 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
523 zptSchema.addField('fgcmZptErr', type=np.float64,
524 doc='Error on zeropoint, estimated from repeatability + number of obs')
525 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
526 doc='maximum x/maximum y to scale to apply chebyshev parameters')
527 zptSchema.addField('fgcmfZptCheb', type='ArrayD',
528 size=zptChebyshevSize,
529 doc='Chebyshev parameters (flattened) for zeropoint')
530 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
531 size=superStarChebyshevSize,
532 doc='Chebyshev parameters (flattened) for superStarFlat')
533 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
534 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
535 zptSchema.addField('fgcmR0', type=np.float64,
536 doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
537 zptSchema.addField('fgcmR10', type=np.float64,
538 doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
539 zptSchema.addField('fgcmGry', type=np.float64,
540 doc='Estimated gray extinction relative to atmospheric solution; '
541 'only for fgcmFlag <= 4 (see fgcmFlag) ')
542 zptSchema.addField('fgcmDeltaChrom', type=np.float64,
543 doc='Mean chromatic correction for stars in this ccd; '
544 'only for fgcmFlag <= 4 (see fgcmFlag)')
545 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
546 zptSchema.addField('fgcmTilings', type=np.float64,
547 doc='Number of photometric tilings used for solution for ccd')
548 zptSchema.addField('fgcmFpGry', type=np.float64,
549 doc='Average gray extinction over the full focal plane '
550 '(same for all ccds in a visit)')
551 zptSchema.addField('fgcmFpGryBlue', type=np.float64,
552 doc='Average gray extinction over the full focal plane '
553 'for 25% bluest stars')
554 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
555 doc='Error on Average gray extinction over the full focal plane '
556 'for 25% bluest stars')
557 zptSchema.addField('fgcmFpGryRed', type=np.float64,
558 doc='Average gray extinction over the full focal plane '
559 'for 25% reddest stars')
560 zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
561 doc='Error on Average gray extinction over the full focal plane '
562 'for 25% reddest stars')
563 zptSchema.addField('fgcmFpVar', type=np.float64,
564 doc='Variance of gray extinction over the full focal plane '
565 '(same for all ccds in a visit)')
566 zptSchema.addField('fgcmDust', type=np.float64,
567 doc='Gray dust extinction from the primary/corrector'
568 'at the time of the exposure')
569 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
570 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
571 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
572 doc=('Local background correction from brightest percentile '
573 '(value set by deltaMagBkgOffsetPercentile) calibration '
574 'stars.'))
575 zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
576 zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
578 return zptSchema
581def makeZptCat(zptSchema, zpStruct):
582 """
583 Make the zeropoint catalog for persistence
585 Parameters
586 ----------
587 zptSchema: `lsst.afw.table.Schema`
588 Zeropoint catalog schema
589 zpStruct: `numpy.ndarray`
590 Zeropoint structure from fgcm
592 Returns
593 -------
594 zptCat: `afwTable.BaseCatalog`
595 Zeropoint catalog for persistence
596 """
598 zptCat = afwTable.BaseCatalog(zptSchema)
599 zptCat.reserve(zpStruct.size)
601 for filterName in zpStruct['FILTERNAME']:
602 rec = zptCat.addNew()
603 rec['filtername'] = filterName.decode('utf-8')
605 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD]
606 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD]
607 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
608 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
609 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
610 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
611 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
612 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
613 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
614 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
615 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
616 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
617 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
618 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
619 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
620 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
621 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
622 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
623 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
624 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
625 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
626 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
627 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
628 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
629 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
630 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
631 zptCat['exptime'][:] = zpStruct['EXPTIME']
633 return zptCat
636def makeAtmSchema():
637 """
638 Make the atmosphere schema
640 Returns
641 -------
642 atmSchema: `lsst.afw.table.Schema`
643 """
645 atmSchema = afwTable.Schema()
647 atmSchema.addField('visit', type=np.int32, doc='Visit number')
648 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
649 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
650 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
651 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
652 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
653 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
654 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
655 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
657 return atmSchema
660def makeAtmCat(atmSchema, atmStruct):
661 """
662 Make the atmosphere catalog for persistence
664 Parameters
665 ----------
666 atmSchema: `lsst.afw.table.Schema`
667 Atmosphere catalog schema
668 atmStruct: `numpy.ndarray`
669 Atmosphere structure from fgcm
671 Returns
672 -------
673 atmCat: `lsst.afw.table.BaseCatalog`
674 Atmosphere catalog for persistence
675 """
677 atmCat = afwTable.BaseCatalog(atmSchema)
678 atmCat.resize(atmStruct.size)
680 atmCat['visit'][:] = atmStruct['VISIT']
681 atmCat['pmb'][:] = atmStruct['PMB']
682 atmCat['pwv'][:] = atmStruct['PWV']
683 atmCat['tau'][:] = atmStruct['TAU']
684 atmCat['alpha'][:] = atmStruct['ALPHA']
685 atmCat['o3'][:] = atmStruct['O3']
686 atmCat['secZenith'][:] = atmStruct['SECZENITH']
687 atmCat['cTrans'][:] = atmStruct['CTRANS']
688 atmCat['lamStd'][:] = atmStruct['LAMSTD']
690 return atmCat
693def makeStdSchema(nBands):
694 """
695 Make the standard star schema
697 Parameters
698 ----------
699 nBands: `int`
700 Number of bands in standard star catalog
702 Returns
703 -------
704 stdSchema: `lsst.afw.table.Schema`
705 """
707 stdSchema = afwTable.SimpleTable.makeMinimalSchema()
708 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
709 size=nBands)
710 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
711 size=nBands)
712 stdSchema.addField('mag_std_noabs', type='ArrayF',
713 doc='Standard magnitude (no absolute calibration)',
714 size=nBands)
715 stdSchema.addField('magErr_std', type='ArrayF',
716 doc='Standard magnitude error',
717 size=nBands)
718 stdSchema.addField('npsfcand', type='ArrayI',
719 doc='Number of observations flagged as psf candidates',
720 size=nBands)
721 stdSchema.addField('delta_aper', type='ArrayF',
722 doc='Delta mag (small - large aperture)',
723 size=nBands)
725 return stdSchema
728def makeStdCat(stdSchema, stdStruct, goodBands):
729 """
730 Make the standard star catalog for persistence
732 Parameters
733 ----------
734 stdSchema: `lsst.afw.table.Schema`
735 Standard star catalog schema
736 stdStruct: `numpy.ndarray`
737 Standard star structure in FGCM format
738 goodBands: `list`
739 List of good band names used in stdStruct
741 Returns
742 -------
743 stdCat: `lsst.afw.table.BaseCatalog`
744 Standard star catalog for persistence
745 """
747 stdCat = afwTable.SimpleCatalog(stdSchema)
748 stdCat.resize(stdStruct.size)
750 stdCat['id'][:] = stdStruct['FGCM_ID']
751 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
752 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
753 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
754 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
755 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
756 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
757 if 'NPSFCAND' in stdStruct.dtype.names:
758 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
759 stdCat['delta_aper'][:, :] = stdStruct['DELTA_APER'][:, :]
761 md = PropertyList()
762 md.set("BANDS", list(goodBands))
763 stdCat.setMetadata(md)
765 return stdCat
768def computeApertureRadiusFromName(fluxField):
769 """
770 Compute the radius associated with a CircularApertureFlux or ApFlux field.
772 Parameters
773 ----------
774 fluxField : `str`
775 CircularApertureFlux or ApFlux
777 Returns
778 -------
779 apertureRadius : `float`
780 Radius of the aperture field, in pixels.
782 Raises
783 ------
784 RuntimeError: Raised if flux field is not a CircularApertureFlux,
785 ApFlux, or apFlux.
786 """
787 # TODO: Move this method to more general stack method in DM-25775
788 m = re.search(r'(CircularApertureFlux|ApFlux|apFlux)_(\d+)_(\d+)_', fluxField)
790 if m is None:
791 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
793 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
795 return apertureRadius
798def extractReferenceMags(refStars, bands, filterMap):
799 """
800 Extract reference magnitudes from refStars for given bands and
801 associated filterMap.
803 Parameters
804 ----------
805 refStars : `astropy.table.Table` or `lsst.afw.table.BaseCatalog`
806 FGCM reference star catalog.
807 bands : `list`
808 List of bands for calibration.
809 filterMap: `dict`
810 FGCM mapping of filter to band.
812 Returns
813 -------
814 refMag : `np.ndarray`
815 nstar x nband array of reference magnitudes.
816 refMagErr : `np.ndarray`
817 nstar x nband array of reference magnitude errors.
818 """
819 hasAstropyMeta = False
820 try:
821 meta = refStars.meta
822 hasAstropyMeta = True
823 except AttributeError:
824 meta = refStars.getMetadata()
826 if 'FILTERNAMES' in meta:
827 if hasAstropyMeta:
828 filternames = meta['FILTERNAMES']
829 else:
830 filternames = meta.getArray('FILTERNAMES')
832 # The reference catalog that fgcm wants has one entry per band
833 # in the config file
834 refMag = np.zeros((len(refStars), len(bands)),
835 dtype=refStars['refMag'].dtype) + 99.0
836 refMagErr = np.zeros_like(refMag) + 99.0
837 for i, filtername in enumerate(filternames):
838 # We are allowed to run the fit configured so that we do not
839 # use every column in the reference catalog.
840 try:
841 band = filterMap[filtername]
842 except KeyError:
843 continue
844 try:
845 ind = bands.index(band)
846 except ValueError:
847 continue
849 refMag[:, ind] = refStars['refMag'][:, i]
850 refMagErr[:, ind] = refStars['refMagErr'][:, i]
851 else:
852 raise RuntimeError("FGCM reference stars missing FILTERNAMES metadata.")
854 return refMag, refMagErr
857def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
858 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
859 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
861 return registry.queryDatasets(datasetType,
862 dataId=quantumDataId,
863 collections=[unboundedCollection])