lsst.fgcmcal g33a6e8c39e+92dd209c29
Loading...
Searching...
No Matches
utilities.py
Go to the documentation of this file.
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Utility functions for fgcmcal.
22
23This file contains utility functions that are used by more than one task,
24and do not need to be part of a task.
25"""
26
27import numpy as np
28import os
29import re
30
31from lsst.daf.base import PropertyList
32from lsst.daf.butler import Timespan
33import lsst.afw.table as afwTable
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.geom as geom
37from lsst.obs.base import createInitialSkyWcs
38
39import fgcm
40
41
42FGCM_EXP_FIELD = 'VISIT'
43FGCM_CCD_FIELD = 'DETECTOR'
44FGCM_ILLEGAL_VALUE = -9999.0
45
46
47def makeConfigDict(config, log, camera, maxIter,
48 resetFitParameters, outputZeropoints,
49 lutFilterNames, tract=None, nCore=1):
50 """
51 Make the FGCM fit cycle configuration dict
52
53 Parameters
54 ----------
55 config : `lsst.fgcmcal.FgcmFitCycleConfig`
56 Configuration object
57 log : `lsst.log.Log`
58 LSST log object
59 camera : `lsst.afw.cameraGeom.Camera`
60 Camera from the butler
61 maxIter : `int`
62 Maximum number of iterations
63 resetFitParameters: `bool`
64 Reset fit parameters before fitting?
65 outputZeropoints : `bool`
66 Compute zeropoints for output?
67 lutFilterNames : array-like, `str`
68 Array of physical filter names in the LUT.
69 tract : `int`, optional
70 Tract number for extending the output file name for debugging.
71 Default is None.
72 nCore : `int`, optional
73 Number of cores to use.
74
75 Returns
76 -------
77 configDict : `dict`
78 Configuration dictionary for fgcm
79 """
80 # Extract the bands that are _not_ being fit for fgcm configuration
81 notFitBands = [b for b in config.bands if b not in config.fitBands]
82
83 # process the starColorCuts
84 starColorCutList = []
85 for ccut in config.starColorCuts:
86 if ccut == 'NO_DATA':
87 # No color cuts to apply.
88 break
89 parts = ccut.split(',')
90 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
91
92 # process the refStarColorCuts
93 refStarColorCutList = []
94 for ccut in config.refStarColorCuts:
95 if ccut == 'NO_DATA':
96 # No color cuts to apply.
97 break
98 parts = ccut.split(',')
99 refStarColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
100
101 # TODO: Having direct access to the mirror area from the camera would be
102 # useful. See DM-16489.
103 # Mirror area in cm**2
104 if config.mirrorArea is None:
105 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
106 else:
107 # Convert to square cm.
108 mirrorArea = config.mirrorArea * 100.**2.
109
110 # Get approximate average camera gain:
111 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
112 cameraGain = float(np.median(gains))
113
114 # Cut down the filter map to those that are in the LUT
115 filterToBand = {filterName: config.physicalFilterMap[filterName] for
116 filterName in lutFilterNames}
117
118 if tract is None:
119 outfileBase = config.outfileBase
120 else:
121 outfileBase = '%s-%06d' % (config.outfileBase, tract)
122
123 # create a configuration dictionary for fgcmFitCycle
124 configDict = {'outfileBase': outfileBase,
125 'logger': log,
126 'exposureFile': None,
127 'obsFile': None,
128 'indexFile': None,
129 'lutFile': None,
130 'mirrorArea': mirrorArea,
131 'cameraGain': cameraGain,
132 'ccdStartIndex': camera[0].getId(),
133 'expField': FGCM_EXP_FIELD,
134 'ccdField': FGCM_CCD_FIELD,
135 'seeingField': 'DELTA_APER',
136 'fwhmField': 'PSFSIGMA',
137 'skyBrightnessField': 'SKYBACKGROUND',
138 'deepFlag': 'DEEPFLAG', # unused
139 'bands': list(config.bands),
140 'fitBands': list(config.fitBands),
141 'notFitBands': notFitBands,
142 'requiredBands': list(config.requiredBands),
143 'filterToBand': filterToBand,
144 'logLevel': 'INFO',
145 'nCore': nCore,
146 'nStarPerRun': config.nStarPerRun,
147 'nExpPerRun': config.nExpPerRun,
148 'reserveFraction': config.reserveFraction,
149 'freezeStdAtmosphere': config.freezeStdAtmosphere,
150 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
151 'superStarSubCCDDict': dict(config.superStarSubCcdDict),
152 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
153 'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
154 'superStarSigmaClip': config.superStarSigmaClip,
155 'superStarPlotCCDResiduals': config.superStarPlotCcdResiduals,
156 'focalPlaneSigmaClip': config.focalPlaneSigmaClip,
157 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
158 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
159 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
160 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict),
161 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder,
162 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd,
163 'cycleNumber': config.cycleNumber,
164 'maxIter': maxIter,
165 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
166 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
167 'UTBoundary': config.utBoundary,
168 'washMJDs': config.washMjds,
169 'epochMJDs': config.epochMjds,
170 'coatingMJDs': config.coatingMjds,
171 'minObsPerBand': config.minObsPerBand,
172 'latitude': config.latitude,
173 'defaultCameraOrientation': config.defaultCameraOrientation,
174 'brightObsGrayMax': config.brightObsGrayMax,
175 'minStarPerCCD': config.minStarPerCcd,
176 'minCCDPerExp': config.minCcdPerExp,
177 'maxCCDGrayErr': config.maxCcdGrayErr,
178 'minStarPerExp': config.minStarPerExp,
179 'minExpPerNight': config.minExpPerNight,
180 'expGrayInitialCut': config.expGrayInitialCut,
181 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
182 'expGrayHighCutDict': dict(config.expGrayHighCutDict),
183 'expGrayRecoverCut': config.expGrayRecoverCut,
184 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
185 'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
186 'refStarSnMin': config.refStarSnMin,
187 'refStarOutlierNSig': config.refStarOutlierNSig,
188 'applyRefStarColorCuts': config.applyRefStarColorCuts,
189 'useExposureReferenceOffset': config.useExposureReferenceOffset,
190 'illegalValue': FGCM_ILLEGAL_VALUE, # internally used by fgcm.
191 'starColorCuts': starColorCutList,
192 'refStarColorCuts': refStarColorCutList,
193 'aperCorrFitNBins': config.aperCorrFitNBins,
194 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
195 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
196 'sedTermDict': config.sedterms.toDict()['data'],
197 'colorSplitBands': list(config.colorSplitBands),
198 'sigFgcmMaxErr': config.sigFgcmMaxErr,
199 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
200 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
201 'approxThroughputDict': dict(config.approxThroughputDict),
202 'sigmaCalRange': list(config.sigmaCalRange),
203 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
204 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
205 'sigma0Phot': config.sigma0Phot,
206 'mapLongitudeRef': config.mapLongitudeRef,
207 'mapNSide': config.mapNSide,
208 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
209 'varMinBand': 2,
210 'useRetrievedPwv': False,
211 'useNightlyRetrievedPwv': False,
212 'pwvRetrievalSmoothBlock': 25,
213 'useQuadraticPwv': config.useQuadraticPwv,
214 'useRetrievedTauInit': False,
215 'tauRetrievalMinCCDPerNight': 500,
216 'modelMagErrors': config.modelMagErrors,
217 'instrumentParsPerBand': config.instrumentParsPerBand,
218 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
219 'fitMirrorChromaticity': config.fitMirrorChromaticity,
220 'fitCCDChromaticityDict': dict(config.fitCcdChromaticityDict),
221 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
222 'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
223 'autoHighCutNSig': config.autoHighCutNSig,
224 'deltaAperInnerRadiusArcsec': config.deltaAperInnerRadiusArcsec,
225 'deltaAperOuterRadiusArcsec': config.deltaAperOuterRadiusArcsec,
226 'deltaAperFitMinNgoodObs': config.deltaAperFitMinNgoodObs,
227 'deltaAperFitPerCcdNx': config.deltaAperFitPerCcdNx,
228 'deltaAperFitPerCcdNy': config.deltaAperFitPerCcdNy,
229 'deltaAperFitSpatialNside': config.deltaAperFitSpatialNside,
230 'doComputeDeltaAperExposures': config.doComputeDeltaAperPerVisit,
231 'doComputeDeltaAperStars': config.doComputeDeltaAperPerStar,
232 'doComputeDeltaAperMap': config.doComputeDeltaAperMap,
233 'doComputeDeltaAperPerCcd': config.doComputeDeltaAperPerCcd,
234 'printOnly': False,
235 'quietMode': config.quietMode,
236 'randomSeed': config.randomSeed,
237 'outputStars': False,
238 'outputPath': os.path.abspath('.'),
239 'clobber': True,
240 'useSedLUT': False,
241 'resetParameters': resetFitParameters,
242 'doPlots': config.doPlots,
243 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
244 'outputZeropoints': outputZeropoints}
245
246 return configDict
247
248
249def translateFgcmLut(lutCat, physicalFilterMap):
250 """
251 Translate the FGCM look-up-table into an fgcm-compatible object
252
253 Parameters
254 ----------
255 lutCat: `lsst.afw.table.BaseCatalog`
256 Catalog describing the FGCM look-up table
257 physicalFilterMap: `dict`
258 Physical filter to band mapping
259
260 Returns
261 -------
262 fgcmLut: `lsst.fgcm.FgcmLut`
263 Lookup table for FGCM
264 lutIndexVals: `numpy.ndarray`
265 Numpy array with LUT index information for FGCM
266 lutStd: `numpy.ndarray`
267 Numpy array with LUT standard throughput values for FGCM
268
269 Notes
270 -----
271 After running this code, it is wise to `del lutCat` to clear the memory.
272 """
273
274 # first we need the lutIndexVals
275 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U')
276 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U')
277
278 # Note that any discrepancies between config values will raise relevant
279 # exceptions in the FGCM code.
280
281 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
282 lutFilterNames.size),
283 ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
284 lutStdFilterNames.size),
285 ('PMB', 'f8', lutCat[0]['pmb'].size),
286 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
287 ('PMBELEVATION', 'f8'),
288 ('LAMBDANORM', 'f8'),
289 ('PWV', 'f8', lutCat[0]['pwv'].size),
290 ('O3', 'f8', lutCat[0]['o3'].size),
291 ('TAU', 'f8', lutCat[0]['tau'].size),
292 ('ALPHA', 'f8', lutCat[0]['alpha'].size),
293 ('ZENITH', 'f8', lutCat[0]['zenith'].size),
294 ('NCCD', 'i4')])
295
296 lutIndexVals['FILTERNAMES'][:] = lutFilterNames
297 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
298 lutIndexVals['PMB'][:] = lutCat[0]['pmb']
299 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
300 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
301 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
302 lutIndexVals['PWV'][:] = lutCat[0]['pwv']
303 lutIndexVals['O3'][:] = lutCat[0]['o3']
304 lutIndexVals['TAU'][:] = lutCat[0]['tau']
305 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
306 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
307 lutIndexVals['NCCD'] = lutCat[0]['nCcd']
308
309 # now we need the Standard Values
310 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
311 ('PWVSTD', 'f8'),
312 ('O3STD', 'f8'),
313 ('TAUSTD', 'f8'),
314 ('ALPHASTD', 'f8'),
315 ('ZENITHSTD', 'f8'),
316 ('LAMBDARANGE', 'f8', 2),
317 ('LAMBDASTEP', 'f8'),
318 ('LAMBDASTD', 'f8', lutFilterNames.size),
319 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
320 ('I0STD', 'f8', lutFilterNames.size),
321 ('I1STD', 'f8', lutFilterNames.size),
322 ('I10STD', 'f8', lutFilterNames.size),
323 ('I2STD', 'f8', lutFilterNames.size),
324 ('LAMBDAB', 'f8', lutFilterNames.size),
325 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
326 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
327 lutStd['PMBSTD'] = lutCat[0]['pmbStd']
328 lutStd['PWVSTD'] = lutCat[0]['pwvStd']
329 lutStd['O3STD'] = lutCat[0]['o3Std']
330 lutStd['TAUSTD'] = lutCat[0]['tauStd']
331 lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
332 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
333 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
334 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
335 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
336 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
337 lutStd['I0STD'][:] = lutCat[0]['i0Std']
338 lutStd['I1STD'][:] = lutCat[0]['i1Std']
339 lutStd['I10STD'][:] = lutCat[0]['i10Std']
340 lutStd['I2STD'][:] = lutCat[0]['i2Std']
341 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
342 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
343 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
344
345 lutTypes = [row['luttype'] for row in lutCat]
346
347 # And the flattened look-up-table
348 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
349 ('I1', 'f4')])
350
351 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
352 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
353
354 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
355 ('D_O3', 'f4'),
356 ('D_LNTAU', 'f4'),
357 ('D_ALPHA', 'f4'),
358 ('D_SECZENITH', 'f4'),
359 ('D_LNPWV_I1', 'f4'),
360 ('D_O3_I1', 'f4'),
361 ('D_LNTAU_I1', 'f4'),
362 ('D_ALPHA_I1', 'f4'),
363 ('D_SECZENITH_I1', 'f4')])
364
365 for name in lutDerivFlat.dtype.names:
366 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
367
368 # The fgcm.FgcmLUT() class copies all the LUT information into special
369 # shared memory objects that will not blow up the memory usage when used
370 # with python multiprocessing. Once all the numbers are copied, the
371 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
372 # will fall out of scope and can be cleaned up by the garbage collector.
373 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
374 filterToBand=physicalFilterMap)
375
376 return fgcmLut, lutIndexVals, lutStd
377
378
380 """
381 Translate the FGCM visit catalog to an fgcm-compatible object
382
383 Parameters
384 ----------
385 visitCat: `lsst.afw.table.BaseCatalog`
386 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
387
388 Returns
389 -------
390 fgcmExpInfo: `numpy.ndarray`
391 Numpy array for visit information for FGCM
392
393 Notes
394 -----
395 After running this code, it is wise to `del visitCat` to clear the memory.
396 """
397
398 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
399 ('MJD', 'f8'),
400 ('EXPTIME', 'f8'),
401 ('PSFSIGMA', 'f8'),
402 ('DELTA_APER', 'f8'),
403 ('SKYBACKGROUND', 'f8'),
404 ('DEEPFLAG', 'i2'),
405 ('TELHA', 'f8'),
406 ('TELRA', 'f8'),
407 ('TELDEC', 'f8'),
408 ('TELROT', 'f8'),
409 ('PMB', 'f8'),
410 ('FILTERNAME', 'a50')])
411 fgcmExpInfo['VISIT'][:] = visitCat['visit']
412 fgcmExpInfo['MJD'][:] = visitCat['mjd']
413 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
414 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
415 fgcmExpInfo['TELHA'][:] = visitCat['telha']
416 fgcmExpInfo['TELRA'][:] = visitCat['telra']
417 fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
418 fgcmExpInfo['TELROT'][:] = visitCat['telrot']
419 fgcmExpInfo['PMB'][:] = visitCat['pmb']
420 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
421 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
422 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
423 # Note that we have to go through asAstropy() to get a string
424 # array out of an afwTable. This is faster than a row-by-row loop.
425 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter']
426
427 return fgcmExpInfo
428
429
431 """
432 Compute the median pixel scale in the camera
433
434 Returns
435 -------
436 pixelScale: `float`
437 Average pixel scale (arcsecond) over the camera
438 """
439
440 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
441 orientation = 0.0*geom.degrees
442 flipX = False
443
444 # Create a temporary visitInfo for input to createInitialSkyWcs
445 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
446 boresightRotAngle=orientation,
447 rotType=afwImage.RotType.SKY)
448
449 pixelScales = np.zeros(len(camera))
450 for i, detector in enumerate(camera):
451 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
452 pixelScales[i] = wcs.getPixelScale().asArcseconds()
453
454 ok, = np.where(pixelScales > 0.0)
455 return np.median(pixelScales[ok])
456
457
459 """
460 Compute the approximate pixel area bounded fields from the camera
461 geometry.
462
463 Parameters
464 ----------
465 camera: `lsst.afw.cameraGeom.Camera`
466
467 Returns
468 -------
469 approxPixelAreaFields: `dict`
470 Dictionary of approximate area fields, keyed with detector ID
471 """
472
473 areaScaling = 1. / computeReferencePixelScale(camera)**2.
474
475 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
476 # since we are looking for relative scales
477 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
478
479 flipX = False
480 # Create a temporary visitInfo for input to createInitialSkyWcs
481 # The orientation does not matter for the area computation
482 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
483 boresightRotAngle=0.0*geom.degrees,
484 rotType=afwImage.RotType.SKY)
485
486 approxPixelAreaFields = {}
487
488 for i, detector in enumerate(camera):
489 key = detector.getId()
490
491 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
492 bbox = detector.getBBox()
493
494 areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
495 unit=geom.arcseconds, scaling=areaScaling)
496 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
497
498 approxPixelAreaFields[key] = approxAreaField
499
500 return approxPixelAreaFields
501
502
503def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
504 """
505 Make the zeropoint schema
506
507 Parameters
508 ----------
509 superStarChebyshevSize: `int`
510 Length of the superstar chebyshev array
511 zptChebyshevSize: `int`
512 Length of the zeropoint chebyshev array
513
514 Returns
515 -------
516 zptSchema: `lsst.afw.table.schema`
517 """
518
519 zptSchema = afwTable.Schema()
520
521 zptSchema.addField('visit', type=np.int64, doc='Visit number')
522 zptSchema.addField('detector', type=np.int32, doc='Detector ID number')
523 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
524 '1: Photometric, used in fit; '
525 '2: Photometric, not used in fit; '
526 '4: Non-photometric, on partly photometric night; '
527 '8: Non-photometric, on non-photometric night; '
528 '16: No zeropoint could be determined; '
529 '32: Too few stars for reliable gray computation'))
530 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
531 zptSchema.addField('fgcmZptErr', type=np.float64,
532 doc='Error on zeropoint, estimated from repeatability + number of obs')
533 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
534 doc='maximum x/maximum y to scale to apply chebyshev parameters')
535 zptSchema.addField('fgcmfZptCheb', type='ArrayD',
536 size=zptChebyshevSize,
537 doc='Chebyshev parameters (flattened) for zeropoint')
538 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
539 size=superStarChebyshevSize,
540 doc='Chebyshev parameters (flattened) for superStarFlat')
541 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
542 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
543 zptSchema.addField('fgcmR0', type=np.float64,
544 doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
545 zptSchema.addField('fgcmR10', type=np.float64,
546 doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
547 zptSchema.addField('fgcmGry', type=np.float64,
548 doc='Estimated gray extinction relative to atmospheric solution; '
549 'only for fgcmFlag <= 4 (see fgcmFlag) ')
550 zptSchema.addField('fgcmDeltaChrom', type=np.float64,
551 doc='Mean chromatic correction for stars in this ccd; '
552 'only for fgcmFlag <= 4 (see fgcmFlag)')
553 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
554 zptSchema.addField('fgcmTilings', type=np.float64,
555 doc='Number of photometric tilings used for solution for ccd')
556 zptSchema.addField('fgcmFpGry', type=np.float64,
557 doc='Average gray extinction over the full focal plane '
558 '(same for all ccds in a visit)')
559 zptSchema.addField('fgcmFpGryBlue', type=np.float64,
560 doc='Average gray extinction over the full focal plane '
561 'for 25% bluest stars')
562 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
563 doc='Error on Average gray extinction over the full focal plane '
564 'for 25% bluest stars')
565 zptSchema.addField('fgcmFpGryRed', type=np.float64,
566 doc='Average gray extinction over the full focal plane '
567 'for 25% reddest stars')
568 zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
569 doc='Error on Average gray extinction over the full focal plane '
570 'for 25% reddest stars')
571 zptSchema.addField('fgcmFpVar', type=np.float64,
572 doc='Variance of gray extinction over the full focal plane '
573 '(same for all ccds in a visit)')
574 zptSchema.addField('fgcmDust', type=np.float64,
575 doc='Gray dust extinction from the primary/corrector'
576 'at the time of the exposure')
577 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
578 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
579 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
580 doc=('Local background correction from brightest percentile '
581 '(value set by deltaMagBkgOffsetPercentile) calibration '
582 'stars.'))
583 zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
584 zptSchema.addField('filtername', type=str, size=30, doc='Filter name')
585
586 return zptSchema
587
588
589def makeZptCat(zptSchema, zpStruct):
590 """
591 Make the zeropoint catalog for persistence
592
593 Parameters
594 ----------
595 zptSchema: `lsst.afw.table.Schema`
596 Zeropoint catalog schema
597 zpStruct: `numpy.ndarray`
598 Zeropoint structure from fgcm
599
600 Returns
601 -------
602 zptCat: `afwTable.BaseCatalog`
603 Zeropoint catalog for persistence
604 """
605
606 zptCat = afwTable.BaseCatalog(zptSchema)
607 zptCat.reserve(zpStruct.size)
608
609 for filterName in zpStruct['FILTERNAME']:
610 rec = zptCat.addNew()
611 rec['filtername'] = filterName.decode('utf-8')
612
613 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD]
614 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD]
615 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
616 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
617 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
618 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
619 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
620 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
621 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
622 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
623 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
624 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
625 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
626 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
627 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
628 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
629 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
630 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
631 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
632 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
633 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
634 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
635 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
636 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
637 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
638 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
639 zptCat['exptime'][:] = zpStruct['EXPTIME']
640
641 return zptCat
642
643
645 """
646 Make the atmosphere schema
647
648 Returns
649 -------
650 atmSchema: `lsst.afw.table.Schema`
651 """
652
653 atmSchema = afwTable.Schema()
654
655 atmSchema.addField('visit', type=np.int64, doc='Visit number')
656 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
657 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
658 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
659 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
660 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
661 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
662 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
663 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
664
665 return atmSchema
666
667
668def makeAtmCat(atmSchema, atmStruct):
669 """
670 Make the atmosphere catalog for persistence
671
672 Parameters
673 ----------
674 atmSchema: `lsst.afw.table.Schema`
675 Atmosphere catalog schema
676 atmStruct: `numpy.ndarray`
677 Atmosphere structure from fgcm
678
679 Returns
680 -------
681 atmCat: `lsst.afw.table.BaseCatalog`
682 Atmosphere catalog for persistence
683 """
684
685 atmCat = afwTable.BaseCatalog(atmSchema)
686 atmCat.resize(atmStruct.size)
687
688 atmCat['visit'][:] = atmStruct['VISIT']
689 atmCat['pmb'][:] = atmStruct['PMB']
690 atmCat['pwv'][:] = atmStruct['PWV']
691 atmCat['tau'][:] = atmStruct['TAU']
692 atmCat['alpha'][:] = atmStruct['ALPHA']
693 atmCat['o3'][:] = atmStruct['O3']
694 atmCat['secZenith'][:] = atmStruct['SECZENITH']
695 atmCat['cTrans'][:] = atmStruct['CTRANS']
696 atmCat['lamStd'][:] = atmStruct['LAMSTD']
697
698 return atmCat
699
700
701def makeStdSchema(nBands):
702 """
703 Make the standard star schema
704
705 Parameters
706 ----------
707 nBands: `int`
708 Number of bands in standard star catalog
709
710 Returns
711 -------
712 stdSchema: `lsst.afw.table.Schema`
713 """
714
715 stdSchema = afwTable.SimpleTable.makeMinimalSchema()
716 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
717 size=nBands)
718 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
719 size=nBands)
720 stdSchema.addField('mag_std_noabs', type='ArrayF',
721 doc='Standard magnitude (no absolute calibration)',
722 size=nBands)
723 stdSchema.addField('magErr_std', type='ArrayF',
724 doc='Standard magnitude error',
725 size=nBands)
726 stdSchema.addField('npsfcand', type='ArrayI',
727 doc='Number of observations flagged as psf candidates',
728 size=nBands)
729 stdSchema.addField('delta_aper', type='ArrayF',
730 doc='Delta mag (small - large aperture)',
731 size=nBands)
732
733 return stdSchema
734
735
736def makeStdCat(stdSchema, stdStruct, goodBands):
737 """
738 Make the standard star catalog for persistence
739
740 Parameters
741 ----------
742 stdSchema: `lsst.afw.table.Schema`
743 Standard star catalog schema
744 stdStruct: `numpy.ndarray`
745 Standard star structure in FGCM format
746 goodBands: `list`
747 List of good band names used in stdStruct
748
749 Returns
750 -------
751 stdCat: `lsst.afw.table.BaseCatalog`
752 Standard star catalog for persistence
753 """
754
755 stdCat = afwTable.SimpleCatalog(stdSchema)
756 stdCat.resize(stdStruct.size)
757
758 stdCat['id'][:] = stdStruct['FGCM_ID']
759 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
760 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
761 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
762 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
763 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
764 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
765 if 'NPSFCAND' in stdStruct.dtype.names:
766 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
767 stdCat['delta_aper'][:, :] = stdStruct['DELTA_APER'][:, :]
768
769 md = PropertyList()
770 md.set("BANDS", list(goodBands))
771 stdCat.setMetadata(md)
772
773 return stdCat
774
775
777 """
778 Compute the radius associated with a CircularApertureFlux or ApFlux field.
779
780 Parameters
781 ----------
782 fluxField : `str`
783 CircularApertureFlux or ApFlux
784
785 Returns
786 -------
787 apertureRadius : `float`
788 Radius of the aperture field, in pixels.
789
790 Raises
791 ------
792 RuntimeError: Raised if flux field is not a CircularApertureFlux,
793 ApFlux, or apFlux.
794 """
795 # TODO: Move this method to more general stack method in DM-25775
796 m = re.search(r'(CircularApertureFlux|ApFlux|apFlux)_(\d+)_(\d+)_', fluxField)
797
798 if m is None:
799 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
800
801 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
802
803 return apertureRadius
804
805
806def extractReferenceMags(refStars, bands, filterMap):
807 """
808 Extract reference magnitudes from refStars for given bands and
809 associated filterMap.
810
811 Parameters
812 ----------
813 refStars : `astropy.table.Table` or `lsst.afw.table.BaseCatalog`
814 FGCM reference star catalog.
815 bands : `list`
816 List of bands for calibration.
817 filterMap: `dict`
818 FGCM mapping of filter to band.
819
820 Returns
821 -------
822 refMag : `np.ndarray`
823 nstar x nband array of reference magnitudes.
824 refMagErr : `np.ndarray`
825 nstar x nband array of reference magnitude errors.
826 """
827 hasAstropyMeta = False
828 try:
829 meta = refStars.meta
830 hasAstropyMeta = True
831 except AttributeError:
832 meta = refStars.getMetadata()
833
834 if 'FILTERNAMES' in meta:
835 if hasAstropyMeta:
836 filternames = meta['FILTERNAMES']
837 else:
838 filternames = meta.getArray('FILTERNAMES')
839
840 # The reference catalog that fgcm wants has one entry per band
841 # in the config file
842 refMag = np.zeros((len(refStars), len(bands)),
843 dtype=refStars['refMag'].dtype) + 99.0
844 refMagErr = np.zeros_like(refMag) + 99.0
845 for i, filtername in enumerate(filternames):
846 # We are allowed to run the fit configured so that we do not
847 # use every column in the reference catalog.
848 try:
849 band = filterMap[filtername]
850 except KeyError:
851 continue
852 try:
853 ind = bands.index(band)
854 except ValueError:
855 continue
856
857 refMag[:, ind] = refStars['refMag'][:, i]
858 refMagErr[:, ind] = refStars['refMagErr'][:, i]
859 else:
860 raise RuntimeError("FGCM reference stars missing FILTERNAMES metadata.")
861
862 return refMag, refMagErr
863
864
865def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
866 # For static calibrations, we search with a timespan that has unbounded
867 # begin and end; we'll get an error if there's more than one match (because
868 # then it's not static).
869 timespan = Timespan(begin=None, end=None)
870 result = []
871 # First iterate over all of the data IDs for this dataset type that are
872 # consistent with the quantum data ID.
873 for dataId in registry.queryDataIds(datasetType.dimensions, dataId=quantumDataId):
874 # Find the dataset with this data ID using the unbounded timespan.
875 if ref := registry.findDataset(datasetType, dataId, collections=collections, timespan=timespan):
876 result.append(ref)
877 return result
translateVisitCatalog(visitCat)
Definition utilities.py:379
computeApertureRadiusFromName(fluxField)
Definition utilities.py:776
makeStdCat(stdSchema, stdStruct, goodBands)
Definition utilities.py:736
makeZptCat(zptSchema, zpStruct)
Definition utilities.py:589
lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition utilities.py:865
computeReferencePixelScale(camera)
Definition utilities.py:430
makeAtmCat(atmSchema, atmStruct)
Definition utilities.py:668
translateFgcmLut(lutCat, physicalFilterMap)
Definition utilities.py:249
makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None, nCore=1)
Definition utilities.py:49
computeApproxPixelAreaFields(camera)
Definition utilities.py:458
makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition utilities.py:503
extractReferenceMags(refStars, bands, filterMap)
Definition utilities.py:806