lsst.fgcmcal g2ab4ef6978+768cc448d9
Loading...
Searching...
No Matches
utilities.py
Go to the documentation of this file.
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Utility functions for fgcmcal.
22
23This file contains utility functions that are used by more than one task,
24and do not need to be part of a task.
25"""
26
27import numpy as np
28import os
29import re
30
31from lsst.daf.base import PropertyList
32from lsst.daf.butler import Timespan
33import lsst.afw.table as afwTable
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.geom as geom
37from lsst.obs.base import createInitialSkyWcs
38
39import fgcm
40
41
42FGCM_EXP_FIELD = 'VISIT'
43FGCM_CCD_FIELD = 'DETECTOR'
44FGCM_ILLEGAL_VALUE = -9999.0
45
46
47def makeConfigDict(config, log, camera, maxIter,
48 resetFitParameters, outputZeropoints,
49 lutFilterNames, tract=None):
50 """
51 Make the FGCM fit cycle configuration dict
52
53 Parameters
54 ----------
55 config: `lsst.fgcmcal.FgcmFitCycleConfig`
56 Configuration object
57 log: `lsst.log.Log`
58 LSST log object
59 camera: `lsst.afw.cameraGeom.Camera`
60 Camera from the butler
61 maxIter: `int`
62 Maximum number of iterations
63 resetFitParameters: `bool`
64 Reset fit parameters before fitting?
65 outputZeropoints: `bool`
66 Compute zeropoints for output?
67 lutFilterNames : array-like, `str`
68 Array of physical filter names in the LUT.
69 tract: `int`, optional
70 Tract number for extending the output file name for debugging.
71 Default is None.
72
73 Returns
74 -------
75 configDict: `dict`
76 Configuration dictionary for fgcm
77 """
78 # Extract the bands that are _not_ being fit for fgcm configuration
79 notFitBands = [b for b in config.bands if b not in config.fitBands]
80
81 # process the starColorCuts
82 starColorCutList = []
83 for ccut in config.starColorCuts:
84 if ccut == 'NO_DATA':
85 # No color cuts to apply.
86 break
87 parts = ccut.split(',')
88 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
89
90 # process the refStarColorCuts
91 refStarColorCutList = []
92 for ccut in config.refStarColorCuts:
93 if ccut == 'NO_DATA':
94 # No color cuts to apply.
95 break
96 parts = ccut.split(',')
97 refStarColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
98
99 # TODO: Having direct access to the mirror area from the camera would be
100 # useful. See DM-16489.
101 # Mirror area in cm**2
102 if config.mirrorArea is None:
103 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
104 else:
105 # Convert to square cm.
106 mirrorArea = config.mirrorArea * 100.**2.
107
108 # Get approximate average camera gain:
109 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
110 cameraGain = float(np.median(gains))
111
112 # Cut down the filter map to those that are in the LUT
113 filterToBand = {filterName: config.physicalFilterMap[filterName] for
114 filterName in lutFilterNames}
115
116 if tract is None:
117 outfileBase = config.outfileBase
118 else:
119 outfileBase = '%s-%06d' % (config.outfileBase, tract)
120
121 # create a configuration dictionary for fgcmFitCycle
122 configDict = {'outfileBase': outfileBase,
123 'logger': log,
124 'exposureFile': None,
125 'obsFile': None,
126 'indexFile': None,
127 'lutFile': None,
128 'mirrorArea': mirrorArea,
129 'cameraGain': cameraGain,
130 'ccdStartIndex': camera[0].getId(),
131 'expField': FGCM_EXP_FIELD,
132 'ccdField': FGCM_CCD_FIELD,
133 'seeingField': 'DELTA_APER',
134 'fwhmField': 'PSFSIGMA',
135 'skyBrightnessField': 'SKYBACKGROUND',
136 'deepFlag': 'DEEPFLAG', # unused
137 'bands': list(config.bands),
138 'fitBands': list(config.fitBands),
139 'notFitBands': notFitBands,
140 'requiredBands': list(config.requiredBands),
141 'filterToBand': filterToBand,
142 'logLevel': 'INFO',
143 'nCore': config.nCore,
144 'nStarPerRun': config.nStarPerRun,
145 'nExpPerRun': config.nExpPerRun,
146 'reserveFraction': config.reserveFraction,
147 'freezeStdAtmosphere': config.freezeStdAtmosphere,
148 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
149 'superStarSubCCDDict': dict(config.superStarSubCcdDict),
150 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
151 'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
152 'superStarSigmaClip': config.superStarSigmaClip,
153 'superStarPlotCCDResiduals': config.superStarPlotCcdResiduals,
154 'focalPlaneSigmaClip': config.focalPlaneSigmaClip,
155 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
156 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
157 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
158 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict),
159 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder,
160 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd,
161 'cycleNumber': config.cycleNumber,
162 'maxIter': maxIter,
163 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
164 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
165 'UTBoundary': config.utBoundary,
166 'washMJDs': config.washMjds,
167 'epochMJDs': config.epochMjds,
168 'coatingMJDs': config.coatingMjds,
169 'minObsPerBand': config.minObsPerBand,
170 'latitude': config.latitude,
171 'defaultCameraOrientation': config.defaultCameraOrientation,
172 'brightObsGrayMax': config.brightObsGrayMax,
173 'minStarPerCCD': config.minStarPerCcd,
174 'minCCDPerExp': config.minCcdPerExp,
175 'maxCCDGrayErr': config.maxCcdGrayErr,
176 'minStarPerExp': config.minStarPerExp,
177 'minExpPerNight': config.minExpPerNight,
178 'expGrayInitialCut': config.expGrayInitialCut,
179 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
180 'expGrayHighCutDict': dict(config.expGrayHighCutDict),
181 'expGrayRecoverCut': config.expGrayRecoverCut,
182 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
183 'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
184 'refStarSnMin': config.refStarSnMin,
185 'refStarOutlierNSig': config.refStarOutlierNSig,
186 'applyRefStarColorCuts': config.applyRefStarColorCuts,
187 'useExposureReferenceOffset': config.useExposureReferenceOffset,
188 'illegalValue': FGCM_ILLEGAL_VALUE, # internally used by fgcm.
189 'starColorCuts': starColorCutList,
190 'refStarColorCuts': refStarColorCutList,
191 'aperCorrFitNBins': config.aperCorrFitNBins,
192 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
193 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
194 'sedTermDict': config.sedterms.toDict()['data'],
195 'colorSplitBands': list(config.colorSplitBands),
196 'sigFgcmMaxErr': config.sigFgcmMaxErr,
197 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
198 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
199 'approxThroughputDict': dict(config.approxThroughputDict),
200 'sigmaCalRange': list(config.sigmaCalRange),
201 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
202 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
203 'sigma0Phot': config.sigma0Phot,
204 'mapLongitudeRef': config.mapLongitudeRef,
205 'mapNSide': config.mapNSide,
206 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
207 'varMinBand': 2,
208 'useRetrievedPwv': False,
209 'useNightlyRetrievedPwv': False,
210 'pwvRetrievalSmoothBlock': 25,
211 'useQuadraticPwv': config.useQuadraticPwv,
212 'useRetrievedTauInit': False,
213 'tauRetrievalMinCCDPerNight': 500,
214 'modelMagErrors': config.modelMagErrors,
215 'instrumentParsPerBand': config.instrumentParsPerBand,
216 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
217 'fitMirrorChromaticity': config.fitMirrorChromaticity,
218 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
219 'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
220 'autoHighCutNSig': config.autoHighCutNSig,
221 'deltaAperInnerRadiusArcsec': config.deltaAperInnerRadiusArcsec,
222 'deltaAperOuterRadiusArcsec': config.deltaAperOuterRadiusArcsec,
223 'deltaAperFitMinNgoodObs': config.deltaAperFitMinNgoodObs,
224 'deltaAperFitPerCcdNx': config.deltaAperFitPerCcdNx,
225 'deltaAperFitPerCcdNy': config.deltaAperFitPerCcdNy,
226 'deltaAperFitSpatialNside': config.deltaAperFitSpatialNside,
227 'doComputeDeltaAperExposures': config.doComputeDeltaAperPerVisit,
228 'doComputeDeltaAperStars': config.doComputeDeltaAperPerStar,
229 'doComputeDeltaAperMap': config.doComputeDeltaAperMap,
230 'doComputeDeltaAperPerCcd': config.doComputeDeltaAperPerCcd,
231 'printOnly': False,
232 'quietMode': config.quietMode,
233 'randomSeed': config.randomSeed,
234 'outputStars': False,
235 'outputPath': os.path.abspath('.'),
236 'clobber': True,
237 'useSedLUT': False,
238 'resetParameters': resetFitParameters,
239 'doPlots': config.doPlots,
240 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
241 'outputZeropoints': outputZeropoints}
242
243 return configDict
244
245
246def translateFgcmLut(lutCat, physicalFilterMap):
247 """
248 Translate the FGCM look-up-table into an fgcm-compatible object
249
250 Parameters
251 ----------
252 lutCat: `lsst.afw.table.BaseCatalog`
253 Catalog describing the FGCM look-up table
254 physicalFilterMap: `dict`
255 Physical filter to band mapping
256
257 Returns
258 -------
259 fgcmLut: `lsst.fgcm.FgcmLut`
260 Lookup table for FGCM
261 lutIndexVals: `numpy.ndarray`
262 Numpy array with LUT index information for FGCM
263 lutStd: `numpy.ndarray`
264 Numpy array with LUT standard throughput values for FGCM
265
266 Notes
267 -----
268 After running this code, it is wise to `del lutCat` to clear the memory.
269 """
270
271 # first we need the lutIndexVals
272 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U')
273 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U')
274
275 # Note that any discrepancies between config values will raise relevant
276 # exceptions in the FGCM code.
277
278 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
279 lutFilterNames.size),
280 ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
281 lutStdFilterNames.size),
282 ('PMB', 'f8', lutCat[0]['pmb'].size),
283 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
284 ('PMBELEVATION', 'f8'),
285 ('LAMBDANORM', 'f8'),
286 ('PWV', 'f8', lutCat[0]['pwv'].size),
287 ('O3', 'f8', lutCat[0]['o3'].size),
288 ('TAU', 'f8', lutCat[0]['tau'].size),
289 ('ALPHA', 'f8', lutCat[0]['alpha'].size),
290 ('ZENITH', 'f8', lutCat[0]['zenith'].size),
291 ('NCCD', 'i4')])
292
293 lutIndexVals['FILTERNAMES'][:] = lutFilterNames
294 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
295 lutIndexVals['PMB'][:] = lutCat[0]['pmb']
296 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
297 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
298 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
299 lutIndexVals['PWV'][:] = lutCat[0]['pwv']
300 lutIndexVals['O3'][:] = lutCat[0]['o3']
301 lutIndexVals['TAU'][:] = lutCat[0]['tau']
302 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
303 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
304 lutIndexVals['NCCD'] = lutCat[0]['nCcd']
305
306 # now we need the Standard Values
307 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
308 ('PWVSTD', 'f8'),
309 ('O3STD', 'f8'),
310 ('TAUSTD', 'f8'),
311 ('ALPHASTD', 'f8'),
312 ('ZENITHSTD', 'f8'),
313 ('LAMBDARANGE', 'f8', 2),
314 ('LAMBDASTEP', 'f8'),
315 ('LAMBDASTD', 'f8', lutFilterNames.size),
316 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
317 ('I0STD', 'f8', lutFilterNames.size),
318 ('I1STD', 'f8', lutFilterNames.size),
319 ('I10STD', 'f8', lutFilterNames.size),
320 ('I2STD', 'f8', lutFilterNames.size),
321 ('LAMBDAB', 'f8', lutFilterNames.size),
322 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
323 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
324 lutStd['PMBSTD'] = lutCat[0]['pmbStd']
325 lutStd['PWVSTD'] = lutCat[0]['pwvStd']
326 lutStd['O3STD'] = lutCat[0]['o3Std']
327 lutStd['TAUSTD'] = lutCat[0]['tauStd']
328 lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
329 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
330 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
331 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
332 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
333 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
334 lutStd['I0STD'][:] = lutCat[0]['i0Std']
335 lutStd['I1STD'][:] = lutCat[0]['i1Std']
336 lutStd['I10STD'][:] = lutCat[0]['i10Std']
337 lutStd['I2STD'][:] = lutCat[0]['i2Std']
338 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
339 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
340 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
341
342 lutTypes = [row['luttype'] for row in lutCat]
343
344 # And the flattened look-up-table
345 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
346 ('I1', 'f4')])
347
348 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
349 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
350
351 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
352 ('D_O3', 'f4'),
353 ('D_LNTAU', 'f4'),
354 ('D_ALPHA', 'f4'),
355 ('D_SECZENITH', 'f4'),
356 ('D_LNPWV_I1', 'f4'),
357 ('D_O3_I1', 'f4'),
358 ('D_LNTAU_I1', 'f4'),
359 ('D_ALPHA_I1', 'f4'),
360 ('D_SECZENITH_I1', 'f4')])
361
362 for name in lutDerivFlat.dtype.names:
363 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
364
365 # The fgcm.FgcmLUT() class copies all the LUT information into special
366 # shared memory objects that will not blow up the memory usage when used
367 # with python multiprocessing. Once all the numbers are copied, the
368 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
369 # will fall out of scope and can be cleaned up by the garbage collector.
370 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
371 filterToBand=physicalFilterMap)
372
373 return fgcmLut, lutIndexVals, lutStd
374
375
376def translateVisitCatalog(visitCat):
377 """
378 Translate the FGCM visit catalog to an fgcm-compatible object
379
380 Parameters
381 ----------
382 visitCat: `lsst.afw.table.BaseCatalog`
383 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
384
385 Returns
386 -------
387 fgcmExpInfo: `numpy.ndarray`
388 Numpy array for visit information for FGCM
389
390 Notes
391 -----
392 After running this code, it is wise to `del visitCat` to clear the memory.
393 """
394
395 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
396 ('MJD', 'f8'),
397 ('EXPTIME', 'f8'),
398 ('PSFSIGMA', 'f8'),
399 ('DELTA_APER', 'f8'),
400 ('SKYBACKGROUND', 'f8'),
401 ('DEEPFLAG', 'i2'),
402 ('TELHA', 'f8'),
403 ('TELRA', 'f8'),
404 ('TELDEC', 'f8'),
405 ('TELROT', 'f8'),
406 ('PMB', 'f8'),
407 ('FILTERNAME', 'a50')])
408 fgcmExpInfo['VISIT'][:] = visitCat['visit']
409 fgcmExpInfo['MJD'][:] = visitCat['mjd']
410 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
411 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
412 fgcmExpInfo['TELHA'][:] = visitCat['telha']
413 fgcmExpInfo['TELRA'][:] = visitCat['telra']
414 fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
415 fgcmExpInfo['TELROT'][:] = visitCat['telrot']
416 fgcmExpInfo['PMB'][:] = visitCat['pmb']
417 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
418 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
419 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
420 # Note that we have to go through asAstropy() to get a string
421 # array out of an afwTable. This is faster than a row-by-row loop.
422 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter']
423
424 return fgcmExpInfo
425
426
428 """
429 Compute the median pixel scale in the camera
430
431 Returns
432 -------
433 pixelScale: `float`
434 Average pixel scale (arcsecond) over the camera
435 """
436
437 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
438 orientation = 0.0*geom.degrees
439 flipX = False
440
441 # Create a temporary visitInfo for input to createInitialSkyWcs
442 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
443 boresightRotAngle=orientation,
444 rotType=afwImage.RotType.SKY)
445
446 pixelScales = np.zeros(len(camera))
447 for i, detector in enumerate(camera):
448 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
449 pixelScales[i] = wcs.getPixelScale().asArcseconds()
450
451 ok, = np.where(pixelScales > 0.0)
452 return np.median(pixelScales[ok])
453
454
455def computeApproxPixelAreaFields(camera):
456 """
457 Compute the approximate pixel area bounded fields from the camera
458 geometry.
459
460 Parameters
461 ----------
462 camera: `lsst.afw.cameraGeom.Camera`
463
464 Returns
465 -------
466 approxPixelAreaFields: `dict`
467 Dictionary of approximate area fields, keyed with detector ID
468 """
469
470 areaScaling = 1. / computeReferencePixelScale(camera)**2.
471
472 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
473 # since we are looking for relative scales
474 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
475
476 flipX = False
477 # Create a temporary visitInfo for input to createInitialSkyWcs
478 # The orientation does not matter for the area computation
479 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
480 boresightRotAngle=0.0*geom.degrees,
481 rotType=afwImage.RotType.SKY)
482
483 approxPixelAreaFields = {}
484
485 for i, detector in enumerate(camera):
486 key = detector.getId()
487
488 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
489 bbox = detector.getBBox()
490
491 areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
492 unit=geom.arcseconds, scaling=areaScaling)
493 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
494
495 approxPixelAreaFields[key] = approxAreaField
496
497 return approxPixelAreaFields
498
499
500def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
501 """
502 Make the zeropoint schema
503
504 Parameters
505 ----------
506 superStarChebyshevSize: `int`
507 Length of the superstar chebyshev array
508 zptChebyshevSize: `int`
509 Length of the zeropoint chebyshev array
510
511 Returns
512 -------
513 zptSchema: `lsst.afw.table.schema`
514 """
515
516 zptSchema = afwTable.Schema()
517
518 zptSchema.addField('visit', type=np.int64, doc='Visit number')
519 zptSchema.addField('detector', type=np.int32, doc='Detector ID number')
520 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
521 '1: Photometric, used in fit; '
522 '2: Photometric, not used in fit; '
523 '4: Non-photometric, on partly photometric night; '
524 '8: Non-photometric, on non-photometric night; '
525 '16: No zeropoint could be determined; '
526 '32: Too few stars for reliable gray computation'))
527 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
528 zptSchema.addField('fgcmZptErr', type=np.float64,
529 doc='Error on zeropoint, estimated from repeatability + number of obs')
530 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
531 doc='maximum x/maximum y to scale to apply chebyshev parameters')
532 zptSchema.addField('fgcmfZptCheb', type='ArrayD',
533 size=zptChebyshevSize,
534 doc='Chebyshev parameters (flattened) for zeropoint')
535 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
536 size=superStarChebyshevSize,
537 doc='Chebyshev parameters (flattened) for superStarFlat')
538 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
539 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
540 zptSchema.addField('fgcmR0', type=np.float64,
541 doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
542 zptSchema.addField('fgcmR10', type=np.float64,
543 doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
544 zptSchema.addField('fgcmGry', type=np.float64,
545 doc='Estimated gray extinction relative to atmospheric solution; '
546 'only for fgcmFlag <= 4 (see fgcmFlag) ')
547 zptSchema.addField('fgcmDeltaChrom', type=np.float64,
548 doc='Mean chromatic correction for stars in this ccd; '
549 'only for fgcmFlag <= 4 (see fgcmFlag)')
550 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
551 zptSchema.addField('fgcmTilings', type=np.float64,
552 doc='Number of photometric tilings used for solution for ccd')
553 zptSchema.addField('fgcmFpGry', type=np.float64,
554 doc='Average gray extinction over the full focal plane '
555 '(same for all ccds in a visit)')
556 zptSchema.addField('fgcmFpGryBlue', type=np.float64,
557 doc='Average gray extinction over the full focal plane '
558 'for 25% bluest stars')
559 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
560 doc='Error on Average gray extinction over the full focal plane '
561 'for 25% bluest stars')
562 zptSchema.addField('fgcmFpGryRed', type=np.float64,
563 doc='Average gray extinction over the full focal plane '
564 'for 25% reddest stars')
565 zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
566 doc='Error on Average gray extinction over the full focal plane '
567 'for 25% reddest stars')
568 zptSchema.addField('fgcmFpVar', type=np.float64,
569 doc='Variance of gray extinction over the full focal plane '
570 '(same for all ccds in a visit)')
571 zptSchema.addField('fgcmDust', type=np.float64,
572 doc='Gray dust extinction from the primary/corrector'
573 'at the time of the exposure')
574 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
575 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
576 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
577 doc=('Local background correction from brightest percentile '
578 '(value set by deltaMagBkgOffsetPercentile) calibration '
579 'stars.'))
580 zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
581 zptSchema.addField('filtername', type=str, size=30, doc='Filter name')
582
583 return zptSchema
584
585
586def makeZptCat(zptSchema, zpStruct):
587 """
588 Make the zeropoint catalog for persistence
589
590 Parameters
591 ----------
592 zptSchema: `lsst.afw.table.Schema`
593 Zeropoint catalog schema
594 zpStruct: `numpy.ndarray`
595 Zeropoint structure from fgcm
596
597 Returns
598 -------
599 zptCat: `afwTable.BaseCatalog`
600 Zeropoint catalog for persistence
601 """
602
603 zptCat = afwTable.BaseCatalog(zptSchema)
604 zptCat.reserve(zpStruct.size)
605
606 for filterName in zpStruct['FILTERNAME']:
607 rec = zptCat.addNew()
608 rec['filtername'] = filterName.decode('utf-8')
609
610 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD]
611 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD]
612 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
613 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
614 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
615 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
616 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
617 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
618 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
619 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
620 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
621 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
622 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
623 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
624 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
625 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
626 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
627 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
628 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
629 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
630 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
631 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
632 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
633 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
634 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
635 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
636 zptCat['exptime'][:] = zpStruct['EXPTIME']
637
638 return zptCat
639
640
642 """
643 Make the atmosphere schema
644
645 Returns
646 -------
647 atmSchema: `lsst.afw.table.Schema`
648 """
649
650 atmSchema = afwTable.Schema()
651
652 atmSchema.addField('visit', type=np.int64, doc='Visit number')
653 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
654 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
655 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
656 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
657 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
658 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
659 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
660 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
661
662 return atmSchema
663
664
665def makeAtmCat(atmSchema, atmStruct):
666 """
667 Make the atmosphere catalog for persistence
668
669 Parameters
670 ----------
671 atmSchema: `lsst.afw.table.Schema`
672 Atmosphere catalog schema
673 atmStruct: `numpy.ndarray`
674 Atmosphere structure from fgcm
675
676 Returns
677 -------
678 atmCat: `lsst.afw.table.BaseCatalog`
679 Atmosphere catalog for persistence
680 """
681
682 atmCat = afwTable.BaseCatalog(atmSchema)
683 atmCat.resize(atmStruct.size)
684
685 atmCat['visit'][:] = atmStruct['VISIT']
686 atmCat['pmb'][:] = atmStruct['PMB']
687 atmCat['pwv'][:] = atmStruct['PWV']
688 atmCat['tau'][:] = atmStruct['TAU']
689 atmCat['alpha'][:] = atmStruct['ALPHA']
690 atmCat['o3'][:] = atmStruct['O3']
691 atmCat['secZenith'][:] = atmStruct['SECZENITH']
692 atmCat['cTrans'][:] = atmStruct['CTRANS']
693 atmCat['lamStd'][:] = atmStruct['LAMSTD']
694
695 return atmCat
696
697
698def makeStdSchema(nBands):
699 """
700 Make the standard star schema
701
702 Parameters
703 ----------
704 nBands: `int`
705 Number of bands in standard star catalog
706
707 Returns
708 -------
709 stdSchema: `lsst.afw.table.Schema`
710 """
711
712 stdSchema = afwTable.SimpleTable.makeMinimalSchema()
713 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
714 size=nBands)
715 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
716 size=nBands)
717 stdSchema.addField('mag_std_noabs', type='ArrayF',
718 doc='Standard magnitude (no absolute calibration)',
719 size=nBands)
720 stdSchema.addField('magErr_std', type='ArrayF',
721 doc='Standard magnitude error',
722 size=nBands)
723 stdSchema.addField('npsfcand', type='ArrayI',
724 doc='Number of observations flagged as psf candidates',
725 size=nBands)
726 stdSchema.addField('delta_aper', type='ArrayF',
727 doc='Delta mag (small - large aperture)',
728 size=nBands)
729
730 return stdSchema
731
732
733def makeStdCat(stdSchema, stdStruct, goodBands):
734 """
735 Make the standard star catalog for persistence
736
737 Parameters
738 ----------
739 stdSchema: `lsst.afw.table.Schema`
740 Standard star catalog schema
741 stdStruct: `numpy.ndarray`
742 Standard star structure in FGCM format
743 goodBands: `list`
744 List of good band names used in stdStruct
745
746 Returns
747 -------
748 stdCat: `lsst.afw.table.BaseCatalog`
749 Standard star catalog for persistence
750 """
751
752 stdCat = afwTable.SimpleCatalog(stdSchema)
753 stdCat.resize(stdStruct.size)
754
755 stdCat['id'][:] = stdStruct['FGCM_ID']
756 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
757 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
758 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
759 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
760 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
761 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
762 if 'NPSFCAND' in stdStruct.dtype.names:
763 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
764 stdCat['delta_aper'][:, :] = stdStruct['DELTA_APER'][:, :]
765
766 md = PropertyList()
767 md.set("BANDS", list(goodBands))
768 stdCat.setMetadata(md)
769
770 return stdCat
771
772
773def computeApertureRadiusFromName(fluxField):
774 """
775 Compute the radius associated with a CircularApertureFlux or ApFlux field.
776
777 Parameters
778 ----------
779 fluxField : `str`
780 CircularApertureFlux or ApFlux
781
782 Returns
783 -------
784 apertureRadius : `float`
785 Radius of the aperture field, in pixels.
786
787 Raises
788 ------
789 RuntimeError: Raised if flux field is not a CircularApertureFlux,
790 ApFlux, or apFlux.
791 """
792 # TODO: Move this method to more general stack method in DM-25775
793 m = re.search(r'(CircularApertureFlux|ApFlux|apFlux)_(\d+)_(\d+)_', fluxField)
794
795 if m is None:
796 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
797
798 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
799
800 return apertureRadius
801
802
803def extractReferenceMags(refStars, bands, filterMap):
804 """
805 Extract reference magnitudes from refStars for given bands and
806 associated filterMap.
807
808 Parameters
809 ----------
810 refStars : `astropy.table.Table` or `lsst.afw.table.BaseCatalog`
811 FGCM reference star catalog.
812 bands : `list`
813 List of bands for calibration.
814 filterMap: `dict`
815 FGCM mapping of filter to band.
816
817 Returns
818 -------
819 refMag : `np.ndarray`
820 nstar x nband array of reference magnitudes.
821 refMagErr : `np.ndarray`
822 nstar x nband array of reference magnitude errors.
823 """
824 hasAstropyMeta = False
825 try:
826 meta = refStars.meta
827 hasAstropyMeta = True
828 except AttributeError:
829 meta = refStars.getMetadata()
830
831 if 'FILTERNAMES' in meta:
832 if hasAstropyMeta:
833 filternames = meta['FILTERNAMES']
834 else:
835 filternames = meta.getArray('FILTERNAMES')
836
837 # The reference catalog that fgcm wants has one entry per band
838 # in the config file
839 refMag = np.zeros((len(refStars), len(bands)),
840 dtype=refStars['refMag'].dtype) + 99.0
841 refMagErr = np.zeros_like(refMag) + 99.0
842 for i, filtername in enumerate(filternames):
843 # We are allowed to run the fit configured so that we do not
844 # use every column in the reference catalog.
845 try:
846 band = filterMap[filtername]
847 except KeyError:
848 continue
849 try:
850 ind = bands.index(band)
851 except ValueError:
852 continue
853
854 refMag[:, ind] = refStars['refMag'][:, i]
855 refMagErr[:, ind] = refStars['refMagErr'][:, i]
856 else:
857 raise RuntimeError("FGCM reference stars missing FILTERNAMES metadata.")
858
859 return refMag, refMagErr
860
861
862def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
863 # For static calibrations, we search with a timespan that has unbounded
864 # begin and end; we'll get an error if there's more than one match (because
865 # then it's not static).
866 timespan = Timespan(begin=None, end=None)
867 result = []
868 # First iterate over all of the data IDs for this dataset type that are
869 # consistent with the quantum data ID.
870 for dataId in registry.queryDataIds(datasetType.dimensions, dataId=quantumDataId):
871 # Find the dataset with this data ID using the unbounded timespan.
872 if ref := registry.findDataset(datasetType, dataId, collections=collections, timespan=timespan):
873 result.append(ref)
874 return result
computeReferencePixelScale(camera)
Definition utilities.py:427