Coverage for python/lsst/fgcmcal/utilities.py : 8%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Utility functions for fgcmcal.
23This file contains utility functions that are used by more than one task,
24and do not need to be part of a task.
25"""
27import numpy as np
28import re
30from lsst.daf.base import PropertyList
31import lsst.afw.cameraGeom as afwCameraGeom
32import lsst.afw.table as afwTable
33import lsst.afw.image as afwImage
34import lsst.afw.math as afwMath
35import lsst.geom as geom
36from lsst.obs.base import createInitialSkyWcs
38import fgcm
41def makeConfigDict(config, log, camera, maxIter,
42 resetFitParameters, outputZeropoints, tract=None):
43 """
44 Make the FGCM fit cycle configuration dict
46 Parameters
47 ----------
48 config: `lsst.fgcmcal.FgcmFitCycleConfig`
49 Configuration object
50 log: `lsst.log.Log`
51 LSST log object
52 camera: `lsst.afw.cameraGeom.Camera`
53 Camera from the butler
54 maxIter: `int`
55 Maximum number of iterations
56 resetFitParameters: `bool`
57 Reset fit parameters before fitting?
58 outputZeropoints: `bool`
59 Compute zeropoints for output?
60 tract: `int`, optional
61 Tract number for extending the output file name for debugging.
62 Default is None.
64 Returns
65 -------
66 configDict: `dict`
67 Configuration dictionary for fgcm
68 """
69 # Extract the bands that are _not_ being fit for fgcm configuration
70 notFitBands = [b for b in config.bands if b not in config.fitBands]
72 # process the starColorCuts
73 starColorCutList = []
74 for ccut in config.starColorCuts:
75 parts = ccut.split(',')
76 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
78 # TODO: Having direct access to the mirror area from the camera would be
79 # useful. See DM-16489.
80 # Mirror area in cm**2
81 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
83 # Get approximate average camera gain:
84 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
85 cameraGain = float(np.median(gains))
87 if tract is None:
88 outfileBase = config.outfileBase
89 else:
90 outfileBase = '%s-%06d' % (config.outfileBase, tract)
92 # create a configuration dictionary for fgcmFitCycle
93 configDict = {'outfileBase': outfileBase,
94 'logger': log,
95 'exposureFile': None,
96 'obsFile': None,
97 'indexFile': None,
98 'lutFile': None,
99 'mirrorArea': mirrorArea,
100 'cameraGain': cameraGain,
101 'ccdStartIndex': camera[0].getId(),
102 'expField': 'VISIT',
103 'ccdField': 'CCD',
104 'seeingField': 'DELTA_APER',
105 'fwhmField': 'PSFSIGMA',
106 'skyBrightnessField': 'SKYBACKGROUND',
107 'deepFlag': 'DEEPFLAG', # unused
108 'bands': list(config.bands),
109 'fitBands': list(config.fitBands),
110 'notFitBands': notFitBands,
111 'requiredBands': list(config.requiredBands),
112 'filterToBand': dict(config.filterMap),
113 'logLevel': 'INFO', # FIXME
114 'nCore': config.nCore,
115 'nStarPerRun': config.nStarPerRun,
116 'nExpPerRun': config.nExpPerRun,
117 'reserveFraction': config.reserveFraction,
118 'freezeStdAtmosphere': config.freezeStdAtmosphere,
119 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
120 'superStarSubCCDDict': dict(config.superStarSubCcdDict),
121 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
122 'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
123 'superStarSigmaClip': config.superStarSigmaClip,
124 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
125 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
126 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
127 'cycleNumber': config.cycleNumber,
128 'maxIter': maxIter,
129 'UTBoundary': config.utBoundary,
130 'washMJDs': config.washMjds,
131 'epochMJDs': config.epochMjds,
132 'coatingMJDs': config.coatingMjds,
133 'minObsPerBand': config.minObsPerBand,
134 'latitude': config.latitude,
135 'brightObsGrayMax': config.brightObsGrayMax,
136 'minStarPerCCD': config.minStarPerCcd,
137 'minCCDPerExp': config.minCcdPerExp,
138 'maxCCDGrayErr': config.maxCcdGrayErr,
139 'minStarPerExp': config.minStarPerExp,
140 'minExpPerNight': config.minExpPerNight,
141 'expGrayInitialCut': config.expGrayInitialCut,
142 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
143 'expGrayHighCutDict': dict(config.expGrayHighCutDict),
144 'expGrayRecoverCut': config.expGrayRecoverCut,
145 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
146 'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
147 'refStarSnMin': config.refStarSnMin,
148 'refStarOutlierNSig': config.refStarOutlierNSig,
149 'applyRefStarColorCuts': config.applyRefStarColorCuts,
150 'illegalValue': -9999.0, # internally used by fgcm.
151 'starColorCuts': starColorCutList,
152 'aperCorrFitNBins': config.aperCorrFitNBins,
153 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
154 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
155 'sedTermDict': config.sedterms.toDict()['data'],
156 'colorSplitBands': list(config.colorSplitBands),
157 'sigFgcmMaxErr': config.sigFgcmMaxErr,
158 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
159 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
160 'approxThroughputDict': dict(config.approxThroughputDict),
161 'sigmaCalRange': list(config.sigmaCalRange),
162 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
163 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
164 'sigma0Phot': config.sigma0Phot,
165 'mapLongitudeRef': config.mapLongitudeRef,
166 'mapNSide': config.mapNSide,
167 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
168 'varMinBand': 2,
169 'useRetrievedPwv': False,
170 'useNightlyRetrievedPwv': False,
171 'pwvRetrievalSmoothBlock': 25,
172 'useQuadraticPwv': config.useQuadraticPwv,
173 'useRetrievedTauInit': False,
174 'tauRetrievalMinCCDPerNight': 500,
175 'modelMagErrors': config.modelMagErrors,
176 'instrumentParsPerBand': config.instrumentParsPerBand,
177 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
178 'fitMirrorChromaticity': config.fitMirrorChromaticity,
179 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
180 'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
181 'autoHighCutNSig': config.autoHighCutNSig,
182 'printOnly': False,
183 'quietMode': config.quietMode,
184 'outputStars': False,
185 'clobber': True,
186 'useSedLUT': False,
187 'resetParameters': resetFitParameters,
188 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
189 'outputZeropoints': outputZeropoints}
191 return configDict
194def translateFgcmLut(lutCat, filterMap):
195 """
196 Translate the FGCM look-up-table into an fgcm-compatible object
198 Parameters
199 ----------
200 lutCat: `lsst.afw.table.BaseCatalog`
201 Catalog describing the FGCM look-up table
202 filterMap: `dict`
203 Filter to band mapping
205 Returns
206 -------
207 fgcmLut: `lsst.fgcm.FgcmLut`
208 Lookup table for FGCM
209 lutIndexVals: `numpy.ndarray`
210 Numpy array with LUT index information for FGCM
211 lutStd: `numpy.ndarray`
212 Numpy array with LUT standard throughput values for FGCM
214 Notes
215 -----
216 After running this code, it is wise to `del lutCat` to clear the memory.
217 """
219 # first we need the lutIndexVals
220 # dtype is set for py2/py3/fits/fgcm compatibility
221 lutFilterNames = np.array(lutCat[0]['filterNames'].split(','), dtype='a')
222 lutStdFilterNames = np.array(lutCat[0]['stdFilterNames'].split(','), dtype='a')
224 # Note that any discrepancies between config values will raise relevant
225 # exceptions in the FGCM code.
227 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
228 lutFilterNames.size),
229 ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
230 lutStdFilterNames.size),
231 ('PMB', 'f8', lutCat[0]['pmb'].size),
232 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
233 ('PMBELEVATION', 'f8'),
234 ('LAMBDANORM', 'f8'),
235 ('PWV', 'f8', lutCat[0]['pwv'].size),
236 ('O3', 'f8', lutCat[0]['o3'].size),
237 ('TAU', 'f8', lutCat[0]['tau'].size),
238 ('ALPHA', 'f8', lutCat[0]['alpha'].size),
239 ('ZENITH', 'f8', lutCat[0]['zenith'].size),
240 ('NCCD', 'i4')])
242 lutIndexVals['FILTERNAMES'][:] = lutFilterNames
243 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
244 lutIndexVals['PMB'][:] = lutCat[0]['pmb']
245 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
246 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
247 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
248 lutIndexVals['PWV'][:] = lutCat[0]['pwv']
249 lutIndexVals['O3'][:] = lutCat[0]['o3']
250 lutIndexVals['TAU'][:] = lutCat[0]['tau']
251 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
252 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
253 lutIndexVals['NCCD'] = lutCat[0]['nCcd']
255 # now we need the Standard Values
256 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
257 ('PWVSTD', 'f8'),
258 ('O3STD', 'f8'),
259 ('TAUSTD', 'f8'),
260 ('ALPHASTD', 'f8'),
261 ('ZENITHSTD', 'f8'),
262 ('LAMBDARANGE', 'f8', 2),
263 ('LAMBDASTEP', 'f8'),
264 ('LAMBDASTD', 'f8', lutFilterNames.size),
265 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
266 ('I0STD', 'f8', lutFilterNames.size),
267 ('I1STD', 'f8', lutFilterNames.size),
268 ('I10STD', 'f8', lutFilterNames.size),
269 ('I2STD', 'f8', lutFilterNames.size),
270 ('LAMBDAB', 'f8', lutFilterNames.size),
271 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
272 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
273 lutStd['PMBSTD'] = lutCat[0]['pmbStd']
274 lutStd['PWVSTD'] = lutCat[0]['pwvStd']
275 lutStd['O3STD'] = lutCat[0]['o3Std']
276 lutStd['TAUSTD'] = lutCat[0]['tauStd']
277 lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
278 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
279 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
280 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
281 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
282 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
283 lutStd['I0STD'][:] = lutCat[0]['i0Std']
284 lutStd['I1STD'][:] = lutCat[0]['i1Std']
285 lutStd['I10STD'][:] = lutCat[0]['i10Std']
286 lutStd['I2STD'][:] = lutCat[0]['i2Std']
287 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
288 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
289 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
291 lutTypes = [row['luttype'] for row in lutCat]
293 # And the flattened look-up-table
294 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
295 ('I1', 'f4')])
297 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
298 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
300 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
301 ('D_O3', 'f4'),
302 ('D_LNTAU', 'f4'),
303 ('D_ALPHA', 'f4'),
304 ('D_SECZENITH', 'f4'),
305 ('D_LNPWV_I1', 'f4'),
306 ('D_O3_I1', 'f4'),
307 ('D_LNTAU_I1', 'f4'),
308 ('D_ALPHA_I1', 'f4'),
309 ('D_SECZENITH_I1', 'f4')])
311 for name in lutDerivFlat.dtype.names:
312 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
314 # The fgcm.FgcmLUT() class copies all the LUT information into special
315 # shared memory objects that will not blow up the memory usage when used
316 # with python multiprocessing. Once all the numbers are copied, the
317 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
318 # will fall out of scope and can be cleaned up by the garbage collector.
319 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
320 filterToBand=filterMap)
322 return fgcmLut, lutIndexVals, lutStd
325def translateVisitCatalog(visitCat):
326 """
327 Translate the FGCM visit catalog to an fgcm-compatible object
329 Parameters
330 ----------
331 visitCat: `lsst.afw.table.BaseCatalog`
332 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
334 Returns
335 -------
336 fgcmExpInfo: `numpy.ndarray`
337 Numpy array for visit information for FGCM
339 Notes
340 -----
341 After running this code, it is wise to `del visitCat` to clear the memory.
342 """
344 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
345 ('MJD', 'f8'),
346 ('EXPTIME', 'f8'),
347 ('PSFSIGMA', 'f8'),
348 ('DELTA_APER', 'f8'),
349 ('SKYBACKGROUND', 'f8'),
350 ('DEEPFLAG', 'i2'),
351 ('TELHA', 'f8'),
352 ('TELRA', 'f8'),
353 ('TELDEC', 'f8'),
354 ('TELROT', 'f8'),
355 ('PMB', 'f8'),
356 ('FILTERNAME', 'a10')])
357 fgcmExpInfo['VISIT'][:] = visitCat['visit']
358 fgcmExpInfo['MJD'][:] = visitCat['mjd']
359 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
360 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
361 fgcmExpInfo['TELHA'][:] = visitCat['telha']
362 fgcmExpInfo['TELRA'][:] = visitCat['telra']
363 fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
364 fgcmExpInfo['TELROT'][:] = visitCat['telrot']
365 fgcmExpInfo['PMB'][:] = visitCat['pmb']
366 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
367 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
368 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
369 # Note that we have to go through asAstropy() to get a string
370 # array out of an afwTable. This is faster than a row-by-row loop.
371 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['filtername']
373 return fgcmExpInfo
376def computeCcdOffsets(camera, defaultOrientation):
377 """
378 Compute the CCD offsets in ra/dec and x/y space
380 Parameters
381 ----------
382 camera: `lsst.afw.cameraGeom.Camera`
383 defaultOrientation: `float`
384 Default camera orientation (degrees)
386 Returns
387 -------
388 ccdOffsets: `numpy.ndarray`
389 Numpy array with ccd offset information for input to FGCM.
390 Angular units are degrees, and x/y units are pixels.
391 """
392 # TODO: DM-21215 will fully generalize to arbitrary camera orientations
394 # and we need to know the ccd offsets from the camera geometry
395 ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'),
396 ('DELTA_RA', 'f8'),
397 ('DELTA_DEC', 'f8'),
398 ('RA_SIZE', 'f8'),
399 ('DEC_SIZE', 'f8'),
400 ('X_SIZE', 'i4'),
401 ('Y_SIZE', 'i4')])
403 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
404 # since we are looking for relative positions
405 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
407 # TODO: DM-17597 will update testdata_jointcal so that the test data
408 # does not have nan as the boresight angle for HSC data. For the
409 # time being, there is this ungainly hack.
410 if camera.getName() == 'HSC' and np.isnan(defaultOrientation):
411 orientation = 270*geom.degrees
412 else:
413 orientation = defaultOrientation*geom.degrees
414 flipX = False
416 # Create a temporary visitInfo for input to createInitialSkyWcs
417 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
418 boresightRotAngle=orientation,
419 rotType=afwImage.visitInfo.RotType.SKY)
421 for i, detector in enumerate(camera):
422 ccdOffsets['CCDNUM'][i] = detector.getId()
424 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
426 detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS))
427 ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees()
428 ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees()
430 bbox = detector.getBBox()
432 detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin()))
433 detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax()))
435 ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees())
436 ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees())
438 ccdOffsets['X_SIZE'][i] = bbox.getMaxX()
439 ccdOffsets['Y_SIZE'][i] = bbox.getMaxY()
441 return ccdOffsets
444def computeReferencePixelScale(camera):
445 """
446 Compute the median pixel scale in the camera
448 Returns
449 -------
450 pixelScale: `float`
451 Average pixel scale (arcsecond) over the camera
452 """
454 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
455 orientation = 0.0*geom.degrees
456 flipX = False
458 # Create a temporary visitInfo for input to createInitialSkyWcs
459 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
460 boresightRotAngle=orientation,
461 rotType=afwImage.visitInfo.RotType.SKY)
463 pixelScales = np.zeros(len(camera))
464 for i, detector in enumerate(camera):
465 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
466 pixelScales[i] = wcs.getPixelScale().asArcseconds()
468 ok, = np.where(pixelScales > 0.0)
469 return np.median(pixelScales[ok])
472def computeApproxPixelAreaFields(camera):
473 """
474 Compute the approximate pixel area bounded fields from the camera
475 geometry.
477 Parameters
478 ----------
479 camera: `lsst.afw.cameraGeom.Camera`
481 Returns
482 -------
483 approxPixelAreaFields: `dict`
484 Dictionary of approximate area fields, keyed with detector ID
485 """
487 areaScaling = 1. / computeReferencePixelScale(camera)**2.
489 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
490 # since we are looking for relative scales
491 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
493 flipX = False
494 # Create a temporary visitInfo for input to createInitialSkyWcs
495 # The orientation does not matter for the area computation
496 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
497 boresightRotAngle=0.0*geom.degrees,
498 rotType=afwImage.visitInfo.RotType.SKY)
500 approxPixelAreaFields = {}
502 for i, detector in enumerate(camera):
503 key = detector.getId()
505 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
506 bbox = detector.getBBox()
508 areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
509 unit=geom.arcseconds, scaling=areaScaling)
510 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
512 approxPixelAreaFields[key] = approxAreaField
514 return approxPixelAreaFields
517def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
518 """
519 Make the zeropoint schema
521 Parameters
522 ----------
523 superStarChebyshevSize: `int`
524 Length of the superstar chebyshev array
525 zptChebyshevSize: `int`
526 Length of the zeropoint chebyshev array
528 Returns
529 -------
530 zptSchema: `lsst.afw.table.schema`
531 """
533 zptSchema = afwTable.Schema()
535 zptSchema.addField('visit', type=np.int32, doc='Visit number')
536 zptSchema.addField('ccd', type=np.int32, doc='CCD number')
537 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
538 '1: Photometric, used in fit; '
539 '2: Photometric, not used in fit; '
540 '4: Non-photometric, on partly photometric night; '
541 '8: Non-photometric, on non-photometric night; '
542 '16: No zeropoint could be determined; '
543 '32: Too few stars for reliable gray computation'))
544 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
545 zptSchema.addField('fgcmZptErr', type=np.float64,
546 doc='Error on zeropoint, estimated from repeatability + number of obs')
547 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
548 doc='maximum x/maximum y to scale to apply chebyshev parameters')
549 zptSchema.addField('fgcmfZptCheb', type='ArrayD',
550 size=zptChebyshevSize,
551 doc='Chebyshev parameters (flattened) for zeropoint')
552 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
553 size=superStarChebyshevSize,
554 doc='Chebyshev parameters (flattened) for superStarFlat')
555 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
556 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
557 zptSchema.addField('fgcmR0', type=np.float64,
558 doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
559 zptSchema.addField('fgcmR10', type=np.float64,
560 doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
561 zptSchema.addField('fgcmGry', type=np.float64,
562 doc='Estimated gray extinction relative to atmospheric solution; '
563 'only for flag <= 4')
564 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
565 zptSchema.addField('fgcmTilings', type=np.float64,
566 doc='Number of photometric tilings used for solution for ccd')
567 zptSchema.addField('fgcmFpGry', type=np.float64,
568 doc='Average gray extinction over the full focal plane '
569 '(same for all ccds in a visit)')
570 zptSchema.addField('fgcmFpGryBlue', type=np.float64,
571 doc='Average gray extinction over the full focal plane '
572 'for 25% bluest stars')
573 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
574 doc='Error on Average gray extinction over the full focal plane '
575 'for 25% bluest stars')
576 zptSchema.addField('fgcmFpGryRed', type=np.float64,
577 doc='Average gray extinction over the full focal plane '
578 'for 25% reddest stars')
579 zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
580 doc='Error on Average gray extinction over the full focal plane '
581 'for 25% reddest stars')
582 zptSchema.addField('fgcmFpVar', type=np.float64,
583 doc='Variance of gray extinction over the full focal plane '
584 '(same for all ccds in a visit)')
585 zptSchema.addField('fgcmDust', type=np.float64,
586 doc='Gray dust extinction from the primary/corrector'
587 'at the time of the exposure')
588 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
589 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
590 zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
591 zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
593 return zptSchema
596def makeZptCat(zptSchema, zpStruct):
597 """
598 Make the zeropoint catalog for persistence
600 Parameters
601 ----------
602 zptSchema: `lsst.afw.table.Schema`
603 Zeropoint catalog schema
604 zpStruct: `numpy.ndarray`
605 Zeropoint structure from fgcm
607 Returns
608 -------
609 zptCat: `afwTable.BaseCatalog`
610 Zeropoint catalog for persistence
611 """
613 zptCat = afwTable.BaseCatalog(zptSchema)
614 zptCat.reserve(zpStruct.size)
616 for filterName in zpStruct['FILTERNAME']:
617 rec = zptCat.addNew()
618 rec['filtername'] = filterName.decode('utf-8')
620 zptCat['visit'][:] = zpStruct['VISIT']
621 zptCat['ccd'][:] = zpStruct['CCD']
622 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
623 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
624 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
625 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
626 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
627 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
628 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
629 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
630 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
631 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
632 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
633 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
634 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
635 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
636 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
637 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
638 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
639 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
640 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
641 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
642 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
643 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
644 zptCat['exptime'][:] = zpStruct['EXPTIME']
646 return zptCat
649def makeAtmSchema():
650 """
651 Make the atmosphere schema
653 Returns
654 -------
655 atmSchema: `lsst.afw.table.Schema`
656 """
658 atmSchema = afwTable.Schema()
660 atmSchema.addField('visit', type=np.int32, doc='Visit number')
661 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
662 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
663 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
664 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
665 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
666 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
667 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
668 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
670 return atmSchema
673def makeAtmCat(atmSchema, atmStruct):
674 """
675 Make the atmosphere catalog for persistence
677 Parameters
678 ----------
679 atmSchema: `lsst.afw.table.Schema`
680 Atmosphere catalog schema
681 atmStruct: `numpy.ndarray`
682 Atmosphere structure from fgcm
684 Returns
685 -------
686 atmCat: `lsst.afw.table.BaseCatalog`
687 Atmosphere catalog for persistence
688 """
690 atmCat = afwTable.BaseCatalog(atmSchema)
691 atmCat.resize(atmStruct.size)
693 atmCat['visit'][:] = atmStruct['VISIT']
694 atmCat['pmb'][:] = atmStruct['PMB']
695 atmCat['pwv'][:] = atmStruct['PWV']
696 atmCat['tau'][:] = atmStruct['TAU']
697 atmCat['alpha'][:] = atmStruct['ALPHA']
698 atmCat['o3'][:] = atmStruct['O3']
699 atmCat['secZenith'][:] = atmStruct['SECZENITH']
700 atmCat['cTrans'][:] = atmStruct['CTRANS']
701 atmCat['lamStd'][:] = atmStruct['LAMSTD']
703 return atmCat
706def makeStdSchema(nBands):
707 """
708 Make the standard star schema
710 Parameters
711 ----------
712 nBands: `int`
713 Number of bands in standard star catalog
715 Returns
716 -------
717 stdSchema: `lsst.afw.table.Schema`
718 """
720 stdSchema = afwTable.SimpleTable.makeMinimalSchema()
721 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
722 size=nBands)
723 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
724 size=nBands)
725 stdSchema.addField('mag_std_noabs', type='ArrayF',
726 doc='Standard magnitude (no absolute calibration)',
727 size=nBands)
728 stdSchema.addField('magErr_std', type='ArrayF',
729 doc='Standard magnitude error',
730 size=nBands)
731 stdSchema.addField('npsfcand', type='ArrayI',
732 doc='Number of observations flagged as psf candidates',
733 size=nBands)
735 return stdSchema
738def makeStdCat(stdSchema, stdStruct, goodBands):
739 """
740 Make the standard star catalog for persistence
742 Parameters
743 ----------
744 stdSchema: `lsst.afw.table.Schema`
745 Standard star catalog schema
746 stdStruct: `numpy.ndarray`
747 Standard star structure in FGCM format
748 goodBands: `list`
749 List of good band names used in stdStruct
751 Returns
752 -------
753 stdCat: `lsst.afw.table.BaseCatalog`
754 Standard star catalog for persistence
755 """
757 stdCat = afwTable.SimpleCatalog(stdSchema)
758 stdCat.resize(stdStruct.size)
760 stdCat['id'][:] = stdStruct['FGCM_ID']
761 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
762 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
763 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
764 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
765 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
766 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
767 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
769 md = PropertyList()
770 md.set("BANDS", list(goodBands))
771 stdCat.setMetadata(md)
773 return stdCat
776def computeApertureRadiusFromDataRef(dataRef, fluxField):
777 """
778 Compute the radius associated with a CircularApertureFlux field or
779 associated slot.
781 Parameters
782 ----------
783 dataRef : `lsst.daf.persistence.ButlerDataRef`
784 fluxField : `str`
785 CircularApertureFlux or associated slot.
787 Returns
788 -------
789 apertureRadius : `float`
790 Radius of the aperture field, in pixels.
792 Raises
793 ------
794 RuntimeError: Raised if flux field is not a CircularApertureFlux, ApFlux,
795 or associated slot.
796 """
797 # TODO: Move this method to more general stack method in DM-25775
798 datasetType = dataRef.butlerSubset.datasetType
800 if datasetType == 'src':
801 schema = dataRef.get(datasetType='src_schema').schema
802 try:
803 fluxFieldName = schema[fluxField].asField().getName()
804 except LookupError:
805 raise RuntimeError("Could not find %s or associated slot in schema." % (fluxField))
806 # This may also raise a RuntimeError
807 apertureRadius = computeApertureRadiusFromName(fluxFieldName)
808 else:
809 # This is a sourceTable_visit
810 apertureRadius = computeApertureRadiusFromName(fluxField)
812 return apertureRadius
815def computeApertureRadiusFromName(fluxField):
816 """
817 Compute the radius associated with a CircularApertureFlux or ApFlux field.
819 Parameters
820 ----------
821 fluxField : `str`
822 CircularApertureFlux or ApFlux
824 Returns
825 -------
826 apertureRadius : `float`
827 Radius of the aperture field, in pixels.
829 Raises
830 ------
831 RuntimeError: Raised if flux field is not a CircularApertureFlux
832 or ApFlux.
833 """
834 # TODO: Move this method to more general stack method in DM-25775
835 m = re.search(r'(CircularApertureFlux|ApFlux)_(\d+)_(\d+)_', fluxField)
837 if m is None:
838 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
840 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
842 return apertureRadius
845def extractReferenceMags(refStars, bands, filterMap):
846 """
847 Extract reference magnitudes from refStars for given bands and
848 associated filterMap.
850 Parameters
851 ----------
852 refStars : `lsst.afw.table.BaseCatalog`
853 FGCM reference star catalog
854 bands : `list`
855 List of bands for calibration
856 filterMap: `dict`
857 FGCM mapping of filter to band
859 Returns
860 -------
861 refMag : `np.ndarray`
862 nstar x nband array of reference magnitudes
863 refMagErr : `np.ndarray`
864 nstar x nband array of reference magnitude errors
865 """
866 # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent
867 # against index errors and allow more flexibility in fitting after
868 # the build stars step.
870 md = refStars.getMetadata()
871 if 'FILTERNAMES' in md:
872 filternames = md.getArray('FILTERNAMES')
874 # The reference catalog that fgcm wants has one entry per band
875 # in the config file
876 refMag = np.zeros((len(refStars), len(bands)),
877 dtype=refStars['refMag'].dtype) + 99.0
878 refMagErr = np.zeros_like(refMag) + 99.0
879 for i, filtername in enumerate(filternames):
880 # We are allowed to run the fit configured so that we do not
881 # use every column in the reference catalog.
882 try:
883 band = filterMap[filtername]
884 except KeyError:
885 continue
886 try:
887 ind = bands.index(band)
888 except ValueError:
889 continue
891 refMag[:, ind] = refStars['refMag'][:, i]
892 refMagErr[:, ind] = refStars['refMagErr'][:, i]
894 else:
895 # Continue to use old catalogs as before.
896 refMag = refStars['refMag'][:, :]
897 refMagErr = refStars['refMagErr'][:, :]
899 return refMag, refMagErr