Coverage for python/lsst/fgcmcal/utilities.py : 8%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Utility functions for fgcmcal.
23This file contains utility functions that are used by more than one task,
24and do not need to be part of a task.
25"""
27import numpy as np
28import re
30from lsst.daf.base import PropertyList
31import lsst.afw.cameraGeom as afwCameraGeom
32import lsst.afw.table as afwTable
33import lsst.afw.image as afwImage
34import lsst.afw.math as afwMath
35import lsst.geom as geom
36from lsst.obs.base import createInitialSkyWcs
38import fgcm
41def makeConfigDict(config, log, camera, maxIter,
42 resetFitParameters, outputZeropoints, tract=None):
43 """
44 Make the FGCM fit cycle configuration dict
46 Parameters
47 ----------
48 config: `lsst.fgcmcal.FgcmFitCycleConfig`
49 Configuration object
50 log: `lsst.log.Log`
51 LSST log object
52 camera: `lsst.afw.cameraGeom.Camera`
53 Camera from the butler
54 maxIter: `int`
55 Maximum number of iterations
56 resetFitParameters: `bool`
57 Reset fit parameters before fitting?
58 outputZeropoints: `bool`
59 Compute zeropoints for output?
60 tract: `int`, optional
61 Tract number for extending the output file name for debugging.
62 Default is None.
64 Returns
65 -------
66 configDict: `dict`
67 Configuration dictionary for fgcm
68 """
69 # Extract the bands that are _not_ being fit for fgcm configuration
70 notFitBands = [b for b in config.bands if b not in config.fitBands]
72 # process the starColorCuts
73 starColorCutList = []
74 for ccut in config.starColorCuts:
75 parts = ccut.split(',')
76 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
78 # TODO: Having direct access to the mirror area from the camera would be
79 # useful. See DM-16489.
80 # Mirror area in cm**2
81 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
83 # Get approximate average camera gain:
84 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
85 cameraGain = float(np.median(gains))
87 if tract is None:
88 outfileBase = config.outfileBase
89 else:
90 outfileBase = '%s-%06d' % (config.outfileBase, tract)
92 # create a configuration dictionary for fgcmFitCycle
93 configDict = {'outfileBase': outfileBase,
94 'logger': log,
95 'exposureFile': None,
96 'obsFile': None,
97 'indexFile': None,
98 'lutFile': None,
99 'mirrorArea': mirrorArea,
100 'cameraGain': cameraGain,
101 'ccdStartIndex': camera[0].getId(),
102 'expField': 'VISIT',
103 'ccdField': 'CCD',
104 'seeingField': 'DELTA_APER',
105 'fwhmField': 'PSFSIGMA',
106 'skyBrightnessField': 'SKYBACKGROUND',
107 'deepFlag': 'DEEPFLAG', # unused
108 'bands': list(config.bands),
109 'fitBands': list(config.fitBands),
110 'notFitBands': notFitBands,
111 'requiredBands': list(config.requiredBands),
112 'filterToBand': dict(config.filterMap),
113 'logLevel': 'INFO', # FIXME
114 'nCore': config.nCore,
115 'nStarPerRun': config.nStarPerRun,
116 'nExpPerRun': config.nExpPerRun,
117 'reserveFraction': config.reserveFraction,
118 'freezeStdAtmosphere': config.freezeStdAtmosphere,
119 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
120 'superStarSubCCDDict': dict(config.superStarSubCcdDict),
121 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
122 'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
123 'superStarSigmaClip': config.superStarSigmaClip,
124 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
125 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
126 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
127 'cycleNumber': config.cycleNumber,
128 'maxIter': maxIter,
129 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
130 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
131 'UTBoundary': config.utBoundary,
132 'washMJDs': config.washMjds,
133 'epochMJDs': config.epochMjds,
134 'coatingMJDs': config.coatingMjds,
135 'minObsPerBand': config.minObsPerBand,
136 'latitude': config.latitude,
137 'brightObsGrayMax': config.brightObsGrayMax,
138 'minStarPerCCD': config.minStarPerCcd,
139 'minCCDPerExp': config.minCcdPerExp,
140 'maxCCDGrayErr': config.maxCcdGrayErr,
141 'minStarPerExp': config.minStarPerExp,
142 'minExpPerNight': config.minExpPerNight,
143 'expGrayInitialCut': config.expGrayInitialCut,
144 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
145 'expGrayHighCutDict': dict(config.expGrayHighCutDict),
146 'expGrayRecoverCut': config.expGrayRecoverCut,
147 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
148 'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
149 'refStarSnMin': config.refStarSnMin,
150 'refStarOutlierNSig': config.refStarOutlierNSig,
151 'applyRefStarColorCuts': config.applyRefStarColorCuts,
152 'illegalValue': -9999.0, # internally used by fgcm.
153 'starColorCuts': starColorCutList,
154 'aperCorrFitNBins': config.aperCorrFitNBins,
155 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
156 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
157 'sedTermDict': config.sedterms.toDict()['data'],
158 'colorSplitBands': list(config.colorSplitBands),
159 'sigFgcmMaxErr': config.sigFgcmMaxErr,
160 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
161 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
162 'approxThroughputDict': dict(config.approxThroughputDict),
163 'sigmaCalRange': list(config.sigmaCalRange),
164 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
165 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
166 'sigma0Phot': config.sigma0Phot,
167 'mapLongitudeRef': config.mapLongitudeRef,
168 'mapNSide': config.mapNSide,
169 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
170 'varMinBand': 2,
171 'useRetrievedPwv': False,
172 'useNightlyRetrievedPwv': False,
173 'pwvRetrievalSmoothBlock': 25,
174 'useQuadraticPwv': config.useQuadraticPwv,
175 'useRetrievedTauInit': False,
176 'tauRetrievalMinCCDPerNight': 500,
177 'modelMagErrors': config.modelMagErrors,
178 'instrumentParsPerBand': config.instrumentParsPerBand,
179 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
180 'fitMirrorChromaticity': config.fitMirrorChromaticity,
181 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
182 'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
183 'autoHighCutNSig': config.autoHighCutNSig,
184 'printOnly': False,
185 'quietMode': config.quietMode,
186 'outputStars': False,
187 'clobber': True,
188 'useSedLUT': False,
189 'resetParameters': resetFitParameters,
190 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
191 'outputZeropoints': outputZeropoints}
193 return configDict
196def translateFgcmLut(lutCat, filterMap):
197 """
198 Translate the FGCM look-up-table into an fgcm-compatible object
200 Parameters
201 ----------
202 lutCat: `lsst.afw.table.BaseCatalog`
203 Catalog describing the FGCM look-up table
204 filterMap: `dict`
205 Filter to band mapping
207 Returns
208 -------
209 fgcmLut: `lsst.fgcm.FgcmLut`
210 Lookup table for FGCM
211 lutIndexVals: `numpy.ndarray`
212 Numpy array with LUT index information for FGCM
213 lutStd: `numpy.ndarray`
214 Numpy array with LUT standard throughput values for FGCM
216 Notes
217 -----
218 After running this code, it is wise to `del lutCat` to clear the memory.
219 """
221 # first we need the lutIndexVals
222 # dtype is set for py2/py3/fits/fgcm compatibility
223 lutFilterNames = np.array(lutCat[0]['filterNames'].split(','), dtype='a')
224 lutStdFilterNames = np.array(lutCat[0]['stdFilterNames'].split(','), dtype='a')
226 # Note that any discrepancies between config values will raise relevant
227 # exceptions in the FGCM code.
229 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
230 lutFilterNames.size),
231 ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
232 lutStdFilterNames.size),
233 ('PMB', 'f8', lutCat[0]['pmb'].size),
234 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
235 ('PMBELEVATION', 'f8'),
236 ('LAMBDANORM', 'f8'),
237 ('PWV', 'f8', lutCat[0]['pwv'].size),
238 ('O3', 'f8', lutCat[0]['o3'].size),
239 ('TAU', 'f8', lutCat[0]['tau'].size),
240 ('ALPHA', 'f8', lutCat[0]['alpha'].size),
241 ('ZENITH', 'f8', lutCat[0]['zenith'].size),
242 ('NCCD', 'i4')])
244 lutIndexVals['FILTERNAMES'][:] = lutFilterNames
245 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
246 lutIndexVals['PMB'][:] = lutCat[0]['pmb']
247 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
248 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
249 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
250 lutIndexVals['PWV'][:] = lutCat[0]['pwv']
251 lutIndexVals['O3'][:] = lutCat[0]['o3']
252 lutIndexVals['TAU'][:] = lutCat[0]['tau']
253 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
254 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
255 lutIndexVals['NCCD'] = lutCat[0]['nCcd']
257 # now we need the Standard Values
258 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
259 ('PWVSTD', 'f8'),
260 ('O3STD', 'f8'),
261 ('TAUSTD', 'f8'),
262 ('ALPHASTD', 'f8'),
263 ('ZENITHSTD', 'f8'),
264 ('LAMBDARANGE', 'f8', 2),
265 ('LAMBDASTEP', 'f8'),
266 ('LAMBDASTD', 'f8', lutFilterNames.size),
267 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
268 ('I0STD', 'f8', lutFilterNames.size),
269 ('I1STD', 'f8', lutFilterNames.size),
270 ('I10STD', 'f8', lutFilterNames.size),
271 ('I2STD', 'f8', lutFilterNames.size),
272 ('LAMBDAB', 'f8', lutFilterNames.size),
273 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
274 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
275 lutStd['PMBSTD'] = lutCat[0]['pmbStd']
276 lutStd['PWVSTD'] = lutCat[0]['pwvStd']
277 lutStd['O3STD'] = lutCat[0]['o3Std']
278 lutStd['TAUSTD'] = lutCat[0]['tauStd']
279 lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
280 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
281 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
282 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
283 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
284 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
285 lutStd['I0STD'][:] = lutCat[0]['i0Std']
286 lutStd['I1STD'][:] = lutCat[0]['i1Std']
287 lutStd['I10STD'][:] = lutCat[0]['i10Std']
288 lutStd['I2STD'][:] = lutCat[0]['i2Std']
289 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
290 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
291 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
293 lutTypes = [row['luttype'] for row in lutCat]
295 # And the flattened look-up-table
296 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
297 ('I1', 'f4')])
299 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
300 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
302 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
303 ('D_O3', 'f4'),
304 ('D_LNTAU', 'f4'),
305 ('D_ALPHA', 'f4'),
306 ('D_SECZENITH', 'f4'),
307 ('D_LNPWV_I1', 'f4'),
308 ('D_O3_I1', 'f4'),
309 ('D_LNTAU_I1', 'f4'),
310 ('D_ALPHA_I1', 'f4'),
311 ('D_SECZENITH_I1', 'f4')])
313 for name in lutDerivFlat.dtype.names:
314 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
316 # The fgcm.FgcmLUT() class copies all the LUT information into special
317 # shared memory objects that will not blow up the memory usage when used
318 # with python multiprocessing. Once all the numbers are copied, the
319 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
320 # will fall out of scope and can be cleaned up by the garbage collector.
321 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
322 filterToBand=filterMap)
324 return fgcmLut, lutIndexVals, lutStd
327def translateVisitCatalog(visitCat):
328 """
329 Translate the FGCM visit catalog to an fgcm-compatible object
331 Parameters
332 ----------
333 visitCat: `lsst.afw.table.BaseCatalog`
334 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
336 Returns
337 -------
338 fgcmExpInfo: `numpy.ndarray`
339 Numpy array for visit information for FGCM
341 Notes
342 -----
343 After running this code, it is wise to `del visitCat` to clear the memory.
344 """
346 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
347 ('MJD', 'f8'),
348 ('EXPTIME', 'f8'),
349 ('PSFSIGMA', 'f8'),
350 ('DELTA_APER', 'f8'),
351 ('SKYBACKGROUND', 'f8'),
352 ('DEEPFLAG', 'i2'),
353 ('TELHA', 'f8'),
354 ('TELRA', 'f8'),
355 ('TELDEC', 'f8'),
356 ('TELROT', 'f8'),
357 ('PMB', 'f8'),
358 ('FILTERNAME', 'a10')])
359 fgcmExpInfo['VISIT'][:] = visitCat['visit']
360 fgcmExpInfo['MJD'][:] = visitCat['mjd']
361 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
362 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
363 fgcmExpInfo['TELHA'][:] = visitCat['telha']
364 fgcmExpInfo['TELRA'][:] = visitCat['telra']
365 fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
366 fgcmExpInfo['TELROT'][:] = visitCat['telrot']
367 fgcmExpInfo['PMB'][:] = visitCat['pmb']
368 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
369 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
370 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
371 # Note that we have to go through asAstropy() to get a string
372 # array out of an afwTable. This is faster than a row-by-row loop.
373 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['filtername']
375 return fgcmExpInfo
378def computeCcdOffsets(camera, defaultOrientation):
379 """
380 Compute the CCD offsets in ra/dec and x/y space
382 Parameters
383 ----------
384 camera: `lsst.afw.cameraGeom.Camera`
385 defaultOrientation: `float`
386 Default camera orientation (degrees)
388 Returns
389 -------
390 ccdOffsets: `numpy.ndarray`
391 Numpy array with ccd offset information for input to FGCM.
392 Angular units are degrees, and x/y units are pixels.
393 """
394 # TODO: DM-21215 will fully generalize to arbitrary camera orientations
396 # and we need to know the ccd offsets from the camera geometry
397 ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'),
398 ('DELTA_RA', 'f8'),
399 ('DELTA_DEC', 'f8'),
400 ('RA_SIZE', 'f8'),
401 ('DEC_SIZE', 'f8'),
402 ('X_SIZE', 'i4'),
403 ('Y_SIZE', 'i4')])
405 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
406 # since we are looking for relative positions
407 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
409 # TODO: DM-17597 will update testdata_jointcal so that the test data
410 # does not have nan as the boresight angle for HSC data. For the
411 # time being, there is this ungainly hack.
412 if camera.getName() == 'HSC' and np.isnan(defaultOrientation):
413 orientation = 270*geom.degrees
414 else:
415 orientation = defaultOrientation*geom.degrees
416 flipX = False
418 # Create a temporary visitInfo for input to createInitialSkyWcs
419 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
420 boresightRotAngle=orientation,
421 rotType=afwImage.visitInfo.RotType.SKY)
423 for i, detector in enumerate(camera):
424 ccdOffsets['CCDNUM'][i] = detector.getId()
426 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
428 detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS))
429 ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees()
430 ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees()
432 bbox = detector.getBBox()
434 detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin()))
435 detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax()))
437 ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees())
438 ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees())
440 ccdOffsets['X_SIZE'][i] = bbox.getMaxX()
441 ccdOffsets['Y_SIZE'][i] = bbox.getMaxY()
443 return ccdOffsets
446def computeReferencePixelScale(camera):
447 """
448 Compute the median pixel scale in the camera
450 Returns
451 -------
452 pixelScale: `float`
453 Average pixel scale (arcsecond) over the camera
454 """
456 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
457 orientation = 0.0*geom.degrees
458 flipX = False
460 # Create a temporary visitInfo for input to createInitialSkyWcs
461 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
462 boresightRotAngle=orientation,
463 rotType=afwImage.visitInfo.RotType.SKY)
465 pixelScales = np.zeros(len(camera))
466 for i, detector in enumerate(camera):
467 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
468 pixelScales[i] = wcs.getPixelScale().asArcseconds()
470 ok, = np.where(pixelScales > 0.0)
471 return np.median(pixelScales[ok])
474def computeApproxPixelAreaFields(camera):
475 """
476 Compute the approximate pixel area bounded fields from the camera
477 geometry.
479 Parameters
480 ----------
481 camera: `lsst.afw.cameraGeom.Camera`
483 Returns
484 -------
485 approxPixelAreaFields: `dict`
486 Dictionary of approximate area fields, keyed with detector ID
487 """
489 areaScaling = 1. / computeReferencePixelScale(camera)**2.
491 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
492 # since we are looking for relative scales
493 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
495 flipX = False
496 # Create a temporary visitInfo for input to createInitialSkyWcs
497 # The orientation does not matter for the area computation
498 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
499 boresightRotAngle=0.0*geom.degrees,
500 rotType=afwImage.visitInfo.RotType.SKY)
502 approxPixelAreaFields = {}
504 for i, detector in enumerate(camera):
505 key = detector.getId()
507 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
508 bbox = detector.getBBox()
510 areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
511 unit=geom.arcseconds, scaling=areaScaling)
512 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
514 approxPixelAreaFields[key] = approxAreaField
516 return approxPixelAreaFields
519def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
520 """
521 Make the zeropoint schema
523 Parameters
524 ----------
525 superStarChebyshevSize: `int`
526 Length of the superstar chebyshev array
527 zptChebyshevSize: `int`
528 Length of the zeropoint chebyshev array
530 Returns
531 -------
532 zptSchema: `lsst.afw.table.schema`
533 """
535 zptSchema = afwTable.Schema()
537 zptSchema.addField('visit', type=np.int32, doc='Visit number')
538 zptSchema.addField('ccd', type=np.int32, doc='CCD number')
539 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
540 '1: Photometric, used in fit; '
541 '2: Photometric, not used in fit; '
542 '4: Non-photometric, on partly photometric night; '
543 '8: Non-photometric, on non-photometric night; '
544 '16: No zeropoint could be determined; '
545 '32: Too few stars for reliable gray computation'))
546 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
547 zptSchema.addField('fgcmZptErr', type=np.float64,
548 doc='Error on zeropoint, estimated from repeatability + number of obs')
549 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
550 doc='maximum x/maximum y to scale to apply chebyshev parameters')
551 zptSchema.addField('fgcmfZptCheb', type='ArrayD',
552 size=zptChebyshevSize,
553 doc='Chebyshev parameters (flattened) for zeropoint')
554 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
555 size=superStarChebyshevSize,
556 doc='Chebyshev parameters (flattened) for superStarFlat')
557 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
558 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
559 zptSchema.addField('fgcmR0', type=np.float64,
560 doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
561 zptSchema.addField('fgcmR10', type=np.float64,
562 doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
563 zptSchema.addField('fgcmGry', type=np.float64,
564 doc='Estimated gray extinction relative to atmospheric solution; '
565 'only for fgcmFlag <= 4 (see fgcmFlag) ')
566 zptSchema.addField('fgcmDeltaChrom', type=np.float64,
567 doc='Mean chromatic correction for stars in this ccd; '
568 'only for fgcmFlag <= 4 (see fgcmFlag)')
569 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
570 zptSchema.addField('fgcmTilings', type=np.float64,
571 doc='Number of photometric tilings used for solution for ccd')
572 zptSchema.addField('fgcmFpGry', type=np.float64,
573 doc='Average gray extinction over the full focal plane '
574 '(same for all ccds in a visit)')
575 zptSchema.addField('fgcmFpGryBlue', type=np.float64,
576 doc='Average gray extinction over the full focal plane '
577 'for 25% bluest stars')
578 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
579 doc='Error on Average gray extinction over the full focal plane '
580 'for 25% bluest stars')
581 zptSchema.addField('fgcmFpGryRed', type=np.float64,
582 doc='Average gray extinction over the full focal plane '
583 'for 25% reddest stars')
584 zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
585 doc='Error on Average gray extinction over the full focal plane '
586 'for 25% reddest stars')
587 zptSchema.addField('fgcmFpVar', type=np.float64,
588 doc='Variance of gray extinction over the full focal plane '
589 '(same for all ccds in a visit)')
590 zptSchema.addField('fgcmDust', type=np.float64,
591 doc='Gray dust extinction from the primary/corrector'
592 'at the time of the exposure')
593 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
594 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
595 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
596 doc=('Local background correction from brightest percentile '
597 '(value set by deltaMagBkgOffsetPercentile) calibration '
598 'stars.'))
599 zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
600 zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
602 return zptSchema
605def makeZptCat(zptSchema, zpStruct):
606 """
607 Make the zeropoint catalog for persistence
609 Parameters
610 ----------
611 zptSchema: `lsst.afw.table.Schema`
612 Zeropoint catalog schema
613 zpStruct: `numpy.ndarray`
614 Zeropoint structure from fgcm
616 Returns
617 -------
618 zptCat: `afwTable.BaseCatalog`
619 Zeropoint catalog for persistence
620 """
622 zptCat = afwTable.BaseCatalog(zptSchema)
623 zptCat.reserve(zpStruct.size)
625 for filterName in zpStruct['FILTERNAME']:
626 rec = zptCat.addNew()
627 rec['filtername'] = filterName.decode('utf-8')
629 zptCat['visit'][:] = zpStruct['VISIT']
630 zptCat['ccd'][:] = zpStruct['CCD']
631 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
632 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
633 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
634 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
635 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
636 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
637 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
638 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
639 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
640 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
641 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
642 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
643 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
644 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
645 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
646 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
647 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
648 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
649 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
650 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
651 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
652 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
653 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
654 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
655 zptCat['exptime'][:] = zpStruct['EXPTIME']
657 return zptCat
660def makeAtmSchema():
661 """
662 Make the atmosphere schema
664 Returns
665 -------
666 atmSchema: `lsst.afw.table.Schema`
667 """
669 atmSchema = afwTable.Schema()
671 atmSchema.addField('visit', type=np.int32, doc='Visit number')
672 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
673 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
674 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
675 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
676 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
677 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
678 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
679 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
681 return atmSchema
684def makeAtmCat(atmSchema, atmStruct):
685 """
686 Make the atmosphere catalog for persistence
688 Parameters
689 ----------
690 atmSchema: `lsst.afw.table.Schema`
691 Atmosphere catalog schema
692 atmStruct: `numpy.ndarray`
693 Atmosphere structure from fgcm
695 Returns
696 -------
697 atmCat: `lsst.afw.table.BaseCatalog`
698 Atmosphere catalog for persistence
699 """
701 atmCat = afwTable.BaseCatalog(atmSchema)
702 atmCat.resize(atmStruct.size)
704 atmCat['visit'][:] = atmStruct['VISIT']
705 atmCat['pmb'][:] = atmStruct['PMB']
706 atmCat['pwv'][:] = atmStruct['PWV']
707 atmCat['tau'][:] = atmStruct['TAU']
708 atmCat['alpha'][:] = atmStruct['ALPHA']
709 atmCat['o3'][:] = atmStruct['O3']
710 atmCat['secZenith'][:] = atmStruct['SECZENITH']
711 atmCat['cTrans'][:] = atmStruct['CTRANS']
712 atmCat['lamStd'][:] = atmStruct['LAMSTD']
714 return atmCat
717def makeStdSchema(nBands):
718 """
719 Make the standard star schema
721 Parameters
722 ----------
723 nBands: `int`
724 Number of bands in standard star catalog
726 Returns
727 -------
728 stdSchema: `lsst.afw.table.Schema`
729 """
731 stdSchema = afwTable.SimpleTable.makeMinimalSchema()
732 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
733 size=nBands)
734 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
735 size=nBands)
736 stdSchema.addField('mag_std_noabs', type='ArrayF',
737 doc='Standard magnitude (no absolute calibration)',
738 size=nBands)
739 stdSchema.addField('magErr_std', type='ArrayF',
740 doc='Standard magnitude error',
741 size=nBands)
742 stdSchema.addField('npsfcand', type='ArrayI',
743 doc='Number of observations flagged as psf candidates',
744 size=nBands)
746 return stdSchema
749def makeStdCat(stdSchema, stdStruct, goodBands):
750 """
751 Make the standard star catalog for persistence
753 Parameters
754 ----------
755 stdSchema: `lsst.afw.table.Schema`
756 Standard star catalog schema
757 stdStruct: `numpy.ndarray`
758 Standard star structure in FGCM format
759 goodBands: `list`
760 List of good band names used in stdStruct
762 Returns
763 -------
764 stdCat: `lsst.afw.table.BaseCatalog`
765 Standard star catalog for persistence
766 """
768 stdCat = afwTable.SimpleCatalog(stdSchema)
769 stdCat.resize(stdStruct.size)
771 stdCat['id'][:] = stdStruct['FGCM_ID']
772 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
773 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
774 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
775 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
776 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
777 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
778 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
780 md = PropertyList()
781 md.set("BANDS", list(goodBands))
782 stdCat.setMetadata(md)
784 return stdCat
787def computeApertureRadiusFromDataRef(dataRef, fluxField):
788 """
789 Compute the radius associated with a CircularApertureFlux field or
790 associated slot.
792 Parameters
793 ----------
794 dataRef : `lsst.daf.persistence.ButlerDataRef`
795 fluxField : `str`
796 CircularApertureFlux or associated slot.
798 Returns
799 -------
800 apertureRadius : `float`
801 Radius of the aperture field, in pixels.
803 Raises
804 ------
805 RuntimeError: Raised if flux field is not a CircularApertureFlux, ApFlux,
806 or associated slot.
807 """
808 # TODO: Move this method to more general stack method in DM-25775
809 datasetType = dataRef.butlerSubset.datasetType
811 if datasetType == 'src':
812 schema = dataRef.get(datasetType='src_schema').schema
813 try:
814 fluxFieldName = schema[fluxField].asField().getName()
815 except LookupError:
816 raise RuntimeError("Could not find %s or associated slot in schema." % (fluxField))
817 # This may also raise a RuntimeError
818 apertureRadius = computeApertureRadiusFromName(fluxFieldName)
819 else:
820 # This is a sourceTable_visit
821 apertureRadius = computeApertureRadiusFromName(fluxField)
823 return apertureRadius
826def computeApertureRadiusFromName(fluxField):
827 """
828 Compute the radius associated with a CircularApertureFlux or ApFlux field.
830 Parameters
831 ----------
832 fluxField : `str`
833 CircularApertureFlux or ApFlux
835 Returns
836 -------
837 apertureRadius : `float`
838 Radius of the aperture field, in pixels.
840 Raises
841 ------
842 RuntimeError: Raised if flux field is not a CircularApertureFlux
843 or ApFlux.
844 """
845 # TODO: Move this method to more general stack method in DM-25775
846 m = re.search(r'(CircularApertureFlux|ApFlux)_(\d+)_(\d+)_', fluxField)
848 if m is None:
849 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
851 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
853 return apertureRadius
856def extractReferenceMags(refStars, bands, filterMap):
857 """
858 Extract reference magnitudes from refStars for given bands and
859 associated filterMap.
861 Parameters
862 ----------
863 refStars : `lsst.afw.table.BaseCatalog`
864 FGCM reference star catalog
865 bands : `list`
866 List of bands for calibration
867 filterMap: `dict`
868 FGCM mapping of filter to band
870 Returns
871 -------
872 refMag : `np.ndarray`
873 nstar x nband array of reference magnitudes
874 refMagErr : `np.ndarray`
875 nstar x nband array of reference magnitude errors
876 """
877 # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent
878 # against index errors and allow more flexibility in fitting after
879 # the build stars step.
881 md = refStars.getMetadata()
882 if 'FILTERNAMES' in md:
883 filternames = md.getArray('FILTERNAMES')
885 # The reference catalog that fgcm wants has one entry per band
886 # in the config file
887 refMag = np.zeros((len(refStars), len(bands)),
888 dtype=refStars['refMag'].dtype) + 99.0
889 refMagErr = np.zeros_like(refMag) + 99.0
890 for i, filtername in enumerate(filternames):
891 # We are allowed to run the fit configured so that we do not
892 # use every column in the reference catalog.
893 try:
894 band = filterMap[filtername]
895 except KeyError:
896 continue
897 try:
898 ind = bands.index(band)
899 except ValueError:
900 continue
902 refMag[:, ind] = refStars['refMag'][:, i]
903 refMagErr[:, ind] = refStars['refMagErr'][:, i]
905 else:
906 # Continue to use old catalogs as before.
907 refMag = refStars['refMag'][:, :]
908 refMagErr = refStars['refMagErr'][:, :]
910 return refMag, refMagErr