lsst.fgcmcal g6a870b434d+84d954d6ed
utilities.py
Go to the documentation of this file.
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Utility functions for fgcmcal.
22
23This file contains utility functions that are used by more than one task,
24and do not need to be part of a task.
25"""
26
27import numpy as np
28import os
29import re
30from deprecated.sphinx import deprecated
31
32from lsst.daf.base import PropertyList
33import lsst.daf.persistence as dafPersist
34import lsst.afw.cameraGeom as afwCameraGeom
35import lsst.afw.table as afwTable
36import lsst.afw.image as afwImage
37import lsst.afw.math as afwMath
38import lsst.geom as geom
39from lsst.obs.base import createInitialSkyWcs
40from lsst.pipe.base import Instrument
41
42import fgcm
43
44
45FGCM_EXP_FIELD = 'VISIT'
46FGCM_CCD_FIELD = 'DETECTOR'
47FGCM_ILLEGAL_VALUE = -9999.0
48
49
50def makeConfigDict(config, log, camera, maxIter,
51 resetFitParameters, outputZeropoints,
52 lutFilterNames, tract=None):
53 """
54 Make the FGCM fit cycle configuration dict
55
56 Parameters
57 ----------
58 config: `lsst.fgcmcal.FgcmFitCycleConfig`
59 Configuration object
60 log: `lsst.log.Log`
61 LSST log object
62 camera: `lsst.afw.cameraGeom.Camera`
63 Camera from the butler
64 maxIter: `int`
65 Maximum number of iterations
66 resetFitParameters: `bool`
67 Reset fit parameters before fitting?
68 outputZeropoints: `bool`
69 Compute zeropoints for output?
70 lutFilterNames : array-like, `str`
71 Array of physical filter names in the LUT.
72 tract: `int`, optional
73 Tract number for extending the output file name for debugging.
74 Default is None.
75
76 Returns
77 -------
78 configDict: `dict`
79 Configuration dictionary for fgcm
80 """
81 # Extract the bands that are _not_ being fit for fgcm configuration
82 notFitBands = [b for b in config.bands if b not in config.fitBands]
83
84 # process the starColorCuts
85 starColorCutList = []
86 for ccut in config.starColorCuts:
87 parts = ccut.split(',')
88 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
89
90 # TODO: Having direct access to the mirror area from the camera would be
91 # useful. See DM-16489.
92 # Mirror area in cm**2
93 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
94
95 # Get approximate average camera gain:
96 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
97 cameraGain = float(np.median(gains))
98
99 # Cut down the filter map to those that are in the LUT
100 filterToBand = {filterName: config.physicalFilterMap[filterName] for
101 filterName in lutFilterNames}
102
103 if tract is None:
104 outfileBase = config.outfileBase
105 else:
106 outfileBase = '%s-%06d' % (config.outfileBase, tract)
107
108 # create a configuration dictionary for fgcmFitCycle
109 configDict = {'outfileBase': outfileBase,
110 'logger': log,
111 'exposureFile': None,
112 'obsFile': None,
113 'indexFile': None,
114 'lutFile': None,
115 'mirrorArea': mirrorArea,
116 'cameraGain': cameraGain,
117 'ccdStartIndex': camera[0].getId(),
118 'expField': FGCM_EXP_FIELD,
119 'ccdField': FGCM_CCD_FIELD,
120 'seeingField': 'DELTA_APER',
121 'fwhmField': 'PSFSIGMA',
122 'skyBrightnessField': 'SKYBACKGROUND',
123 'deepFlag': 'DEEPFLAG', # unused
124 'bands': list(config.bands),
125 'fitBands': list(config.fitBands),
126 'notFitBands': notFitBands,
127 'requiredBands': list(config.requiredBands),
128 'filterToBand': filterToBand,
129 'logLevel': 'INFO',
130 'nCore': config.nCore,
131 'nStarPerRun': config.nStarPerRun,
132 'nExpPerRun': config.nExpPerRun,
133 'reserveFraction': config.reserveFraction,
134 'freezeStdAtmosphere': config.freezeStdAtmosphere,
135 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
136 'superStarSubCCDDict': dict(config.superStarSubCcdDict),
137 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
138 'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
139 'superStarSigmaClip': config.superStarSigmaClip,
140 'focalPlaneSigmaClip': config.focalPlaneSigmaClip,
141 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
142 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
143 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
144 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict),
145 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder,
146 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd,
147 'cycleNumber': config.cycleNumber,
148 'maxIter': maxIter,
149 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
150 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
151 'UTBoundary': config.utBoundary,
152 'washMJDs': config.washMjds,
153 'epochMJDs': config.epochMjds,
154 'coatingMJDs': config.coatingMjds,
155 'minObsPerBand': config.minObsPerBand,
156 'latitude': config.latitude,
157 'defaultCameraOrientation': config.defaultCameraOrientation,
158 'brightObsGrayMax': config.brightObsGrayMax,
159 'minStarPerCCD': config.minStarPerCcd,
160 'minCCDPerExp': config.minCcdPerExp,
161 'maxCCDGrayErr': config.maxCcdGrayErr,
162 'minStarPerExp': config.minStarPerExp,
163 'minExpPerNight': config.minExpPerNight,
164 'expGrayInitialCut': config.expGrayInitialCut,
165 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
166 'expGrayHighCutDict': dict(config.expGrayHighCutDict),
167 'expGrayRecoverCut': config.expGrayRecoverCut,
168 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
169 'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
170 'refStarSnMin': config.refStarSnMin,
171 'refStarOutlierNSig': config.refStarOutlierNSig,
172 'applyRefStarColorCuts': config.applyRefStarColorCuts,
173 'illegalValue': FGCM_ILLEGAL_VALUE, # internally used by fgcm.
174 'starColorCuts': starColorCutList,
175 'aperCorrFitNBins': config.aperCorrFitNBins,
176 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
177 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
178 'sedTermDict': config.sedterms.toDict()['data'],
179 'colorSplitBands': list(config.colorSplitBands),
180 'sigFgcmMaxErr': config.sigFgcmMaxErr,
181 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
182 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
183 'approxThroughputDict': dict(config.approxThroughputDict),
184 'sigmaCalRange': list(config.sigmaCalRange),
185 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
186 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
187 'sigma0Phot': config.sigma0Phot,
188 'mapLongitudeRef': config.mapLongitudeRef,
189 'mapNSide': config.mapNSide,
190 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
191 'varMinBand': 2,
192 'useRetrievedPwv': False,
193 'useNightlyRetrievedPwv': False,
194 'pwvRetrievalSmoothBlock': 25,
195 'useQuadraticPwv': config.useQuadraticPwv,
196 'useRetrievedTauInit': False,
197 'tauRetrievalMinCCDPerNight': 500,
198 'modelMagErrors': config.modelMagErrors,
199 'instrumentParsPerBand': config.instrumentParsPerBand,
200 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
201 'fitMirrorChromaticity': config.fitMirrorChromaticity,
202 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
203 'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
204 'autoHighCutNSig': config.autoHighCutNSig,
205 'deltaAperInnerRadiusArcsec': config.deltaAperInnerRadiusArcsec,
206 'deltaAperOuterRadiusArcsec': config.deltaAperOuterRadiusArcsec,
207 'deltaAperFitMinNgoodObs': config.deltaAperFitMinNgoodObs,
208 'deltaAperFitPerCcdNx': config.deltaAperFitPerCcdNx,
209 'deltaAperFitPerCcdNy': config.deltaAperFitPerCcdNy,
210 'deltaAperFitSpatialNside': config.deltaAperFitSpatialNside,
211 'doComputeDeltaAperExposures': config.doComputeDeltaAperPerVisit,
212 'doComputeDeltaAperStars': config.doComputeDeltaAperPerStar,
213 'doComputeDeltaAperMap': config.doComputeDeltaAperMap,
214 'doComputeDeltaAperPerCcd': config.doComputeDeltaAperPerCcd,
215 'printOnly': False,
216 'quietMode': config.quietMode,
217 'randomSeed': config.randomSeed,
218 'outputStars': False,
219 'outputPath': os.path.abspath('.'),
220 'clobber': True,
221 'useSedLUT': False,
222 'resetParameters': resetFitParameters,
223 'doPlots': config.doPlots,
224 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
225 'outputZeropoints': outputZeropoints}
226
227 return configDict
228
229
230def translateFgcmLut(lutCat, physicalFilterMap):
231 """
232 Translate the FGCM look-up-table into an fgcm-compatible object
233
234 Parameters
235 ----------
236 lutCat: `lsst.afw.table.BaseCatalog`
237 Catalog describing the FGCM look-up table
238 physicalFilterMap: `dict`
239 Physical filter to band mapping
240
241 Returns
242 -------
243 fgcmLut: `lsst.fgcm.FgcmLut`
244 Lookup table for FGCM
245 lutIndexVals: `numpy.ndarray`
246 Numpy array with LUT index information for FGCM
247 lutStd: `numpy.ndarray`
248 Numpy array with LUT standard throughput values for FGCM
249
250 Notes
251 -----
252 After running this code, it is wise to `del lutCat` to clear the memory.
253 """
254
255 # first we need the lutIndexVals
256 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U')
257 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U')
258
259 # Note that any discrepancies between config values will raise relevant
260 # exceptions in the FGCM code.
261
262 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
263 lutFilterNames.size),
264 ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
265 lutStdFilterNames.size),
266 ('PMB', 'f8', lutCat[0]['pmb'].size),
267 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
268 ('PMBELEVATION', 'f8'),
269 ('LAMBDANORM', 'f8'),
270 ('PWV', 'f8', lutCat[0]['pwv'].size),
271 ('O3', 'f8', lutCat[0]['o3'].size),
272 ('TAU', 'f8', lutCat[0]['tau'].size),
273 ('ALPHA', 'f8', lutCat[0]['alpha'].size),
274 ('ZENITH', 'f8', lutCat[0]['zenith'].size),
275 ('NCCD', 'i4')])
276
277 lutIndexVals['FILTERNAMES'][:] = lutFilterNames
278 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
279 lutIndexVals['PMB'][:] = lutCat[0]['pmb']
280 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
281 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
282 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
283 lutIndexVals['PWV'][:] = lutCat[0]['pwv']
284 lutIndexVals['O3'][:] = lutCat[0]['o3']
285 lutIndexVals['TAU'][:] = lutCat[0]['tau']
286 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
287 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
288 lutIndexVals['NCCD'] = lutCat[0]['nCcd']
289
290 # now we need the Standard Values
291 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
292 ('PWVSTD', 'f8'),
293 ('O3STD', 'f8'),
294 ('TAUSTD', 'f8'),
295 ('ALPHASTD', 'f8'),
296 ('ZENITHSTD', 'f8'),
297 ('LAMBDARANGE', 'f8', 2),
298 ('LAMBDASTEP', 'f8'),
299 ('LAMBDASTD', 'f8', lutFilterNames.size),
300 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
301 ('I0STD', 'f8', lutFilterNames.size),
302 ('I1STD', 'f8', lutFilterNames.size),
303 ('I10STD', 'f8', lutFilterNames.size),
304 ('I2STD', 'f8', lutFilterNames.size),
305 ('LAMBDAB', 'f8', lutFilterNames.size),
306 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
307 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
308 lutStd['PMBSTD'] = lutCat[0]['pmbStd']
309 lutStd['PWVSTD'] = lutCat[0]['pwvStd']
310 lutStd['O3STD'] = lutCat[0]['o3Std']
311 lutStd['TAUSTD'] = lutCat[0]['tauStd']
312 lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
313 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
314 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
315 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
316 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
317 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
318 lutStd['I0STD'][:] = lutCat[0]['i0Std']
319 lutStd['I1STD'][:] = lutCat[0]['i1Std']
320 lutStd['I10STD'][:] = lutCat[0]['i10Std']
321 lutStd['I2STD'][:] = lutCat[0]['i2Std']
322 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
323 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
324 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
325
326 lutTypes = [row['luttype'] for row in lutCat]
327
328 # And the flattened look-up-table
329 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
330 ('I1', 'f4')])
331
332 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
333 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
334
335 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
336 ('D_O3', 'f4'),
337 ('D_LNTAU', 'f4'),
338 ('D_ALPHA', 'f4'),
339 ('D_SECZENITH', 'f4'),
340 ('D_LNPWV_I1', 'f4'),
341 ('D_O3_I1', 'f4'),
342 ('D_LNTAU_I1', 'f4'),
343 ('D_ALPHA_I1', 'f4'),
344 ('D_SECZENITH_I1', 'f4')])
345
346 for name in lutDerivFlat.dtype.names:
347 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
348
349 # The fgcm.FgcmLUT() class copies all the LUT information into special
350 # shared memory objects that will not blow up the memory usage when used
351 # with python multiprocessing. Once all the numbers are copied, the
352 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
353 # will fall out of scope and can be cleaned up by the garbage collector.
354 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
355 filterToBand=physicalFilterMap)
356
357 return fgcmLut, lutIndexVals, lutStd
358
359
361 """
362 Translate the FGCM visit catalog to an fgcm-compatible object
363
364 Parameters
365 ----------
366 visitCat: `lsst.afw.table.BaseCatalog`
367 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
368
369 Returns
370 -------
371 fgcmExpInfo: `numpy.ndarray`
372 Numpy array for visit information for FGCM
373
374 Notes
375 -----
376 After running this code, it is wise to `del visitCat` to clear the memory.
377 """
378
379 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
380 ('MJD', 'f8'),
381 ('EXPTIME', 'f8'),
382 ('PSFSIGMA', 'f8'),
383 ('DELTA_APER', 'f8'),
384 ('SKYBACKGROUND', 'f8'),
385 ('DEEPFLAG', 'i2'),
386 ('TELHA', 'f8'),
387 ('TELRA', 'f8'),
388 ('TELDEC', 'f8'),
389 ('TELROT', 'f8'),
390 ('PMB', 'f8'),
391 ('FILTERNAME', 'a50')])
392 fgcmExpInfo['VISIT'][:] = visitCat['visit']
393 fgcmExpInfo['MJD'][:] = visitCat['mjd']
394 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
395 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
396 fgcmExpInfo['TELHA'][:] = visitCat['telha']
397 fgcmExpInfo['TELRA'][:] = visitCat['telra']
398 fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
399 fgcmExpInfo['TELROT'][:] = visitCat['telrot']
400 fgcmExpInfo['PMB'][:] = visitCat['pmb']
401 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
402 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
403 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
404 # Note that we have to go through asAstropy() to get a string
405 # array out of an afwTable. This is faster than a row-by-row loop.
406 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter']
407
408 return fgcmExpInfo
409
410
411@deprecated(reason="This method is no longer used in fgcmcal. It will be removed after v23.",
412 version="v23.0", category=FutureWarning)
413def computeCcdOffsets(camera, defaultOrientation):
414 """
415 Compute the CCD offsets in ra/dec and x/y space
416
417 Parameters
418 ----------
419 camera: `lsst.afw.cameraGeom.Camera`
420 defaultOrientation: `float`
421 Default camera orientation (degrees)
422
423 Returns
424 -------
425 ccdOffsets: `numpy.ndarray`
426 Numpy array with ccd offset information for input to FGCM.
427 Angular units are degrees, and x/y units are pixels.
428 """
429 # TODO: DM-21215 will fully generalize to arbitrary camera orientations
430
431 # and we need to know the ccd offsets from the camera geometry
432 ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'),
433 ('DELTA_RA', 'f8'),
434 ('DELTA_DEC', 'f8'),
435 ('RA_SIZE', 'f8'),
436 ('DEC_SIZE', 'f8'),
437 ('X_SIZE', 'i4'),
438 ('Y_SIZE', 'i4')])
439
440 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
441 # since we are looking for relative positions
442 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
443
444 # TODO: DM-17597 will update testdata_jointcal so that the test data
445 # does not have nan as the boresight angle for HSC data. For the
446 # time being, there is this ungainly hack.
447 if camera.getName() == 'HSC' and np.isnan(defaultOrientation):
448 orientation = 270*geom.degrees
449 else:
450 orientation = defaultOrientation*geom.degrees
451 flipX = False
452
453 # Create a temporary visitInfo for input to createInitialSkyWcs
454 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
455 boresightRotAngle=orientation,
456 rotType=afwImage.RotType.SKY)
457
458 for i, detector in enumerate(camera):
459 ccdOffsets['CCDNUM'][i] = detector.getId()
460
461 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
462
463 detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS))
464 ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees()
465 ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees()
466
467 bbox = detector.getBBox()
468
469 detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin()))
470 detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax()))
471
472 ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees())
473 ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees())
474
475 ccdOffsets['X_SIZE'][i] = bbox.getMaxX()
476 ccdOffsets['Y_SIZE'][i] = bbox.getMaxY()
477
478 return ccdOffsets
479
480
482 """
483 Compute the median pixel scale in the camera
484
485 Returns
486 -------
487 pixelScale: `float`
488 Average pixel scale (arcsecond) over the camera
489 """
490
491 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
492 orientation = 0.0*geom.degrees
493 flipX = False
494
495 # Create a temporary visitInfo for input to createInitialSkyWcs
496 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
497 boresightRotAngle=orientation,
498 rotType=afwImage.RotType.SKY)
499
500 pixelScales = np.zeros(len(camera))
501 for i, detector in enumerate(camera):
502 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
503 pixelScales[i] = wcs.getPixelScale().asArcseconds()
504
505 ok, = np.where(pixelScales > 0.0)
506 return np.median(pixelScales[ok])
507
508
510 """
511 Compute the approximate pixel area bounded fields from the camera
512 geometry.
513
514 Parameters
515 ----------
516 camera: `lsst.afw.cameraGeom.Camera`
517
518 Returns
519 -------
520 approxPixelAreaFields: `dict`
521 Dictionary of approximate area fields, keyed with detector ID
522 """
523
524 areaScaling = 1. / computeReferencePixelScale(camera)**2.
525
526 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
527 # since we are looking for relative scales
528 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
529
530 flipX = False
531 # Create a temporary visitInfo for input to createInitialSkyWcs
532 # The orientation does not matter for the area computation
533 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
534 boresightRotAngle=0.0*geom.degrees,
535 rotType=afwImage.RotType.SKY)
536
537 approxPixelAreaFields = {}
538
539 for i, detector in enumerate(camera):
540 key = detector.getId()
541
542 wcs = createInitialSkyWcs(visitInfo, detector, flipX)
543 bbox = detector.getBBox()
544
545 areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
546 unit=geom.arcseconds, scaling=areaScaling)
547 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
548
549 approxPixelAreaFields[key] = approxAreaField
550
551 return approxPixelAreaFields
552
553
554def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
555 """
556 Make the zeropoint schema
557
558 Parameters
559 ----------
560 superStarChebyshevSize: `int`
561 Length of the superstar chebyshev array
562 zptChebyshevSize: `int`
563 Length of the zeropoint chebyshev array
564
565 Returns
566 -------
567 zptSchema: `lsst.afw.table.schema`
568 """
569
570 zptSchema = afwTable.Schema()
571
572 zptSchema.addField('visit', type=np.int32, doc='Visit number')
573 zptSchema.addField('detector', type=np.int32, doc='Detector ID number')
574 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
575 '1: Photometric, used in fit; '
576 '2: Photometric, not used in fit; '
577 '4: Non-photometric, on partly photometric night; '
578 '8: Non-photometric, on non-photometric night; '
579 '16: No zeropoint could be determined; '
580 '32: Too few stars for reliable gray computation'))
581 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
582 zptSchema.addField('fgcmZptErr', type=np.float64,
583 doc='Error on zeropoint, estimated from repeatability + number of obs')
584 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
585 doc='maximum x/maximum y to scale to apply chebyshev parameters')
586 zptSchema.addField('fgcmfZptCheb', type='ArrayD',
587 size=zptChebyshevSize,
588 doc='Chebyshev parameters (flattened) for zeropoint')
589 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
590 size=superStarChebyshevSize,
591 doc='Chebyshev parameters (flattened) for superStarFlat')
592 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
593 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
594 zptSchema.addField('fgcmR0', type=np.float64,
595 doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
596 zptSchema.addField('fgcmR10', type=np.float64,
597 doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
598 zptSchema.addField('fgcmGry', type=np.float64,
599 doc='Estimated gray extinction relative to atmospheric solution; '
600 'only for fgcmFlag <= 4 (see fgcmFlag) ')
601 zptSchema.addField('fgcmDeltaChrom', type=np.float64,
602 doc='Mean chromatic correction for stars in this ccd; '
603 'only for fgcmFlag <= 4 (see fgcmFlag)')
604 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
605 zptSchema.addField('fgcmTilings', type=np.float64,
606 doc='Number of photometric tilings used for solution for ccd')
607 zptSchema.addField('fgcmFpGry', type=np.float64,
608 doc='Average gray extinction over the full focal plane '
609 '(same for all ccds in a visit)')
610 zptSchema.addField('fgcmFpGryBlue', type=np.float64,
611 doc='Average gray extinction over the full focal plane '
612 'for 25% bluest stars')
613 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
614 doc='Error on Average gray extinction over the full focal plane '
615 'for 25% bluest stars')
616 zptSchema.addField('fgcmFpGryRed', type=np.float64,
617 doc='Average gray extinction over the full focal plane '
618 'for 25% reddest stars')
619 zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
620 doc='Error on Average gray extinction over the full focal plane '
621 'for 25% reddest stars')
622 zptSchema.addField('fgcmFpVar', type=np.float64,
623 doc='Variance of gray extinction over the full focal plane '
624 '(same for all ccds in a visit)')
625 zptSchema.addField('fgcmDust', type=np.float64,
626 doc='Gray dust extinction from the primary/corrector'
627 'at the time of the exposure')
628 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
629 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
630 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
631 doc=('Local background correction from brightest percentile '
632 '(value set by deltaMagBkgOffsetPercentile) calibration '
633 'stars.'))
634 zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
635 zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
636
637 return zptSchema
638
639
640def makeZptCat(zptSchema, zpStruct):
641 """
642 Make the zeropoint catalog for persistence
643
644 Parameters
645 ----------
646 zptSchema: `lsst.afw.table.Schema`
647 Zeropoint catalog schema
648 zpStruct: `numpy.ndarray`
649 Zeropoint structure from fgcm
650
651 Returns
652 -------
653 zptCat: `afwTable.BaseCatalog`
654 Zeropoint catalog for persistence
655 """
656
657 zptCat = afwTable.BaseCatalog(zptSchema)
658 zptCat.reserve(zpStruct.size)
659
660 for filterName in zpStruct['FILTERNAME']:
661 rec = zptCat.addNew()
662 rec['filtername'] = filterName.decode('utf-8')
663
664 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD]
665 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD]
666 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
667 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
668 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
669 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
670 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
671 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
672 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
673 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
674 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
675 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
676 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
677 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
678 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
679 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
680 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
681 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
682 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
683 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
684 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
685 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
686 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
687 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
688 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
689 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
690 zptCat['exptime'][:] = zpStruct['EXPTIME']
691
692 return zptCat
693
694
696 """
697 Make the atmosphere schema
698
699 Returns
700 -------
701 atmSchema: `lsst.afw.table.Schema`
702 """
703
704 atmSchema = afwTable.Schema()
705
706 atmSchema.addField('visit', type=np.int32, doc='Visit number')
707 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
708 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
709 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
710 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
711 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
712 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
713 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
714 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
715
716 return atmSchema
717
718
719def makeAtmCat(atmSchema, atmStruct):
720 """
721 Make the atmosphere catalog for persistence
722
723 Parameters
724 ----------
725 atmSchema: `lsst.afw.table.Schema`
726 Atmosphere catalog schema
727 atmStruct: `numpy.ndarray`
728 Atmosphere structure from fgcm
729
730 Returns
731 -------
732 atmCat: `lsst.afw.table.BaseCatalog`
733 Atmosphere catalog for persistence
734 """
735
736 atmCat = afwTable.BaseCatalog(atmSchema)
737 atmCat.resize(atmStruct.size)
738
739 atmCat['visit'][:] = atmStruct['VISIT']
740 atmCat['pmb'][:] = atmStruct['PMB']
741 atmCat['pwv'][:] = atmStruct['PWV']
742 atmCat['tau'][:] = atmStruct['TAU']
743 atmCat['alpha'][:] = atmStruct['ALPHA']
744 atmCat['o3'][:] = atmStruct['O3']
745 atmCat['secZenith'][:] = atmStruct['SECZENITH']
746 atmCat['cTrans'][:] = atmStruct['CTRANS']
747 atmCat['lamStd'][:] = atmStruct['LAMSTD']
748
749 return atmCat
750
751
752def makeStdSchema(nBands):
753 """
754 Make the standard star schema
755
756 Parameters
757 ----------
758 nBands: `int`
759 Number of bands in standard star catalog
760
761 Returns
762 -------
763 stdSchema: `lsst.afw.table.Schema`
764 """
765
766 stdSchema = afwTable.SimpleTable.makeMinimalSchema()
767 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
768 size=nBands)
769 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
770 size=nBands)
771 stdSchema.addField('mag_std_noabs', type='ArrayF',
772 doc='Standard magnitude (no absolute calibration)',
773 size=nBands)
774 stdSchema.addField('magErr_std', type='ArrayF',
775 doc='Standard magnitude error',
776 size=nBands)
777 stdSchema.addField('npsfcand', type='ArrayI',
778 doc='Number of observations flagged as psf candidates',
779 size=nBands)
780 stdSchema.addField('delta_aper', type='ArrayF',
781 doc='Delta mag (small - large aperture)',
782 size=nBands)
783
784 return stdSchema
785
786
787def makeStdCat(stdSchema, stdStruct, goodBands):
788 """
789 Make the standard star catalog for persistence
790
791 Parameters
792 ----------
793 stdSchema: `lsst.afw.table.Schema`
794 Standard star catalog schema
795 stdStruct: `numpy.ndarray`
796 Standard star structure in FGCM format
797 goodBands: `list`
798 List of good band names used in stdStruct
799
800 Returns
801 -------
802 stdCat: `lsst.afw.table.BaseCatalog`
803 Standard star catalog for persistence
804 """
805
806 stdCat = afwTable.SimpleCatalog(stdSchema)
807 stdCat.resize(stdStruct.size)
808
809 stdCat['id'][:] = stdStruct['FGCM_ID']
810 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
811 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
812 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
813 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
814 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
815 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
816 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
817 stdCat['delta_aper'][:, :] = stdStruct['DELTA_APER'][:, :]
818
819 md = PropertyList()
820 md.set("BANDS", list(goodBands))
821 stdCat.setMetadata(md)
822
823 return stdCat
824
825
826def computeApertureRadiusFromDataRef(dataRef, fluxField):
827 """
828 Compute the radius associated with a CircularApertureFlux field or
829 associated slot.
830
831 Parameters
832 ----------
833 dataRef : `lsst.daf.persistence.ButlerDataRef` or
834 `lsst.daf.butler.DeferredDatasetHandle`
835 fluxField : `str`
836 CircularApertureFlux or associated slot.
837
838 Returns
839 -------
840 apertureRadius : `float`
841 Radius of the aperture field, in pixels.
842
843 Raises
844 ------
845 RuntimeError: Raised if flux field is not a CircularApertureFlux, ApFlux,
846 apFlux, or associated slot.
847 """
848 # TODO: Move this method to more general stack method in DM-25775
849 if isinstance(dataRef, dafPersist.ButlerDataRef):
850 # Gen2 dataRef
851 datasetType = dataRef.butlerSubset.datasetType
852 else:
853 # Gen3 dataRef
854 datasetType = dataRef.ref.datasetType.name
855
856 if datasetType == 'src':
857 schema = dataRef.get(datasetType='src_schema').schema
858 try:
859 fluxFieldName = schema[fluxField].asField().getName()
860 except LookupError:
861 raise RuntimeError("Could not find %s or associated slot in schema." % (fluxField))
862 # This may also raise a RuntimeError
863 apertureRadius = computeApertureRadiusFromName(fluxFieldName)
864 else:
865 # This is a sourceTable_visit
866 apertureRadius = computeApertureRadiusFromName(fluxField)
867
868 return apertureRadius
869
870
872 """
873 Compute the radius associated with a CircularApertureFlux or ApFlux field.
874
875 Parameters
876 ----------
877 fluxField : `str`
878 CircularApertureFlux or ApFlux
879
880 Returns
881 -------
882 apertureRadius : `float`
883 Radius of the aperture field, in pixels.
884
885 Raises
886 ------
887 RuntimeError: Raised if flux field is not a CircularApertureFlux,
888 ApFlux, or apFlux.
889 """
890 # TODO: Move this method to more general stack method in DM-25775
891 m = re.search(r'(CircularApertureFlux|ApFlux|apFlux)_(\d+)_(\d+)_', fluxField)
892
893 if m is None:
894 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
895
896 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
897
898 return apertureRadius
899
900
901def extractReferenceMags(refStars, bands, filterMap):
902 """
903 Extract reference magnitudes from refStars for given bands and
904 associated filterMap.
905
906 Parameters
907 ----------
908 refStars : `lsst.afw.table.BaseCatalog`
909 FGCM reference star catalog
910 bands : `list`
911 List of bands for calibration
912 filterMap: `dict`
913 FGCM mapping of filter to band
914
915 Returns
916 -------
917 refMag : `np.ndarray`
918 nstar x nband array of reference magnitudes
919 refMagErr : `np.ndarray`
920 nstar x nband array of reference magnitude errors
921 """
922 # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent
923 # against index errors and allow more flexibility in fitting after
924 # the build stars step.
925
926 md = refStars.getMetadata()
927 if 'FILTERNAMES' in md:
928 filternames = md.getArray('FILTERNAMES')
929
930 # The reference catalog that fgcm wants has one entry per band
931 # in the config file
932 refMag = np.zeros((len(refStars), len(bands)),
933 dtype=refStars['refMag'].dtype) + 99.0
934 refMagErr = np.zeros_like(refMag) + 99.0
935 for i, filtername in enumerate(filternames):
936 # We are allowed to run the fit configured so that we do not
937 # use every column in the reference catalog.
938 try:
939 band = filterMap[filtername]
940 except KeyError:
941 continue
942 try:
943 ind = bands.index(band)
944 except ValueError:
945 continue
946
947 refMag[:, ind] = refStars['refMag'][:, i]
948 refMagErr[:, ind] = refStars['refMagErr'][:, i]
949
950 else:
951 # Continue to use old catalogs as before.
952 refMag = refStars['refMag'][:, :]
953 refMagErr = refStars['refMagErr'][:, :]
954
955 return refMag, refMagErr
956
957
958def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
959 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
960 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
961
962 return registry.queryDatasets(datasetType,
963 dataId=quantumDataId,
964 collections=[unboundedCollection])
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:901
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition: utilities.py:958
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:826
def makeStdSchema(nBands)
Definition: utilities.py:752
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:719
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
Definition: utilities.py:52
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:230
def computeReferencePixelScale(camera)
Definition: utilities.py:481
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:640
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:787
def computeApertureRadiusFromName(fluxField)
Definition: utilities.py:871
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:509
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:554
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:413
def translateVisitCatalog(visitCat)
Definition: utilities.py:360