lsst.fgcmcal  21.0.0-10-g8d1d15d+069596c267
utilities.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Utility functions for fgcmcal.
22 
23 This file contains utility functions that are used by more than one task,
24 and do not need to be part of a task.
25 """
26 
27 import numpy as np
28 import re
29 
30 from lsst.daf.base import PropertyList
31 import lsst.daf.persistence as dafPersist
32 import lsst.afw.cameraGeom as afwCameraGeom
33 import lsst.afw.table as afwTable
34 import lsst.afw.image as afwImage
35 import lsst.afw.math as afwMath
36 import lsst.geom as geom
37 from lsst.obs.base import createInitialSkyWcs
38 from lsst.obs.base import Instrument
39 
40 
41 import fgcm
42 
43 
44 FGCM_EXP_FIELD = 'VISIT'
45 FGCM_CCD_FIELD = 'DETECTOR'
46 
47 
48 def makeConfigDict(config, log, camera, maxIter,
49  resetFitParameters, outputZeropoints, tract=None):
50  """
51  Make the FGCM fit cycle configuration dict
52 
53  Parameters
54  ----------
55  config: `lsst.fgcmcal.FgcmFitCycleConfig`
56  Configuration object
57  log: `lsst.log.Log`
58  LSST log object
59  camera: `lsst.afw.cameraGeom.Camera`
60  Camera from the butler
61  maxIter: `int`
62  Maximum number of iterations
63  resetFitParameters: `bool`
64  Reset fit parameters before fitting?
65  outputZeropoints: `bool`
66  Compute zeropoints for output?
67  tract: `int`, optional
68  Tract number for extending the output file name for debugging.
69  Default is None.
70 
71  Returns
72  -------
73  configDict: `dict`
74  Configuration dictionary for fgcm
75  """
76  # Extract the bands that are _not_ being fit for fgcm configuration
77  notFitBands = [b for b in config.bands if b not in config.fitBands]
78 
79  # process the starColorCuts
80  starColorCutList = []
81  for ccut in config.starColorCuts:
82  parts = ccut.split(',')
83  starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
84 
85  # TODO: Having direct access to the mirror area from the camera would be
86  # useful. See DM-16489.
87  # Mirror area in cm**2
88  mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
89 
90  # Get approximate average camera gain:
91  gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
92  cameraGain = float(np.median(gains))
93 
94  if tract is None:
95  outfileBase = config.outfileBase
96  else:
97  outfileBase = '%s-%06d' % (config.outfileBase, tract)
98 
99  # create a configuration dictionary for fgcmFitCycle
100  configDict = {'outfileBase': outfileBase,
101  'logger': log,
102  'exposureFile': None,
103  'obsFile': None,
104  'indexFile': None,
105  'lutFile': None,
106  'mirrorArea': mirrorArea,
107  'cameraGain': cameraGain,
108  'ccdStartIndex': camera[0].getId(),
109  'expField': FGCM_EXP_FIELD,
110  'ccdField': FGCM_CCD_FIELD,
111  'seeingField': 'DELTA_APER',
112  'fwhmField': 'PSFSIGMA',
113  'skyBrightnessField': 'SKYBACKGROUND',
114  'deepFlag': 'DEEPFLAG', # unused
115  'bands': list(config.bands),
116  'fitBands': list(config.fitBands),
117  'notFitBands': notFitBands,
118  'requiredBands': list(config.requiredBands),
119  'filterToBand': dict(config.physicalFilterMap),
120  'logLevel': 'INFO',
121  'nCore': config.nCore,
122  'nStarPerRun': config.nStarPerRun,
123  'nExpPerRun': config.nExpPerRun,
124  'reserveFraction': config.reserveFraction,
125  'freezeStdAtmosphere': config.freezeStdAtmosphere,
126  'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
127  'superStarSubCCDDict': dict(config.superStarSubCcdDict),
128  'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
129  'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
130  'superStarSigmaClip': config.superStarSigmaClip,
131  'focalPlaneSigmaClip': config.focalPlaneSigmaClip,
132  'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
133  'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
134  'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
135  'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict),
136  'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder,
137  'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd,
138  'cycleNumber': config.cycleNumber,
139  'maxIter': maxIter,
140  'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
141  'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
142  'UTBoundary': config.utBoundary,
143  'washMJDs': config.washMjds,
144  'epochMJDs': config.epochMjds,
145  'coatingMJDs': config.coatingMjds,
146  'minObsPerBand': config.minObsPerBand,
147  'latitude': config.latitude,
148  'brightObsGrayMax': config.brightObsGrayMax,
149  'minStarPerCCD': config.minStarPerCcd,
150  'minCCDPerExp': config.minCcdPerExp,
151  'maxCCDGrayErr': config.maxCcdGrayErr,
152  'minStarPerExp': config.minStarPerExp,
153  'minExpPerNight': config.minExpPerNight,
154  'expGrayInitialCut': config.expGrayInitialCut,
155  'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
156  'expGrayHighCutDict': dict(config.expGrayHighCutDict),
157  'expGrayRecoverCut': config.expGrayRecoverCut,
158  'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
159  'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
160  'refStarSnMin': config.refStarSnMin,
161  'refStarOutlierNSig': config.refStarOutlierNSig,
162  'applyRefStarColorCuts': config.applyRefStarColorCuts,
163  'illegalValue': -9999.0, # internally used by fgcm.
164  'starColorCuts': starColorCutList,
165  'aperCorrFitNBins': config.aperCorrFitNBins,
166  'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
167  'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
168  'sedTermDict': config.sedterms.toDict()['data'],
169  'colorSplitBands': list(config.colorSplitBands),
170  'sigFgcmMaxErr': config.sigFgcmMaxErr,
171  'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
172  'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
173  'approxThroughputDict': dict(config.approxThroughputDict),
174  'sigmaCalRange': list(config.sigmaCalRange),
175  'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
176  'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
177  'sigma0Phot': config.sigma0Phot,
178  'mapLongitudeRef': config.mapLongitudeRef,
179  'mapNSide': config.mapNSide,
180  'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
181  'varMinBand': 2,
182  'useRetrievedPwv': False,
183  'useNightlyRetrievedPwv': False,
184  'pwvRetrievalSmoothBlock': 25,
185  'useQuadraticPwv': config.useQuadraticPwv,
186  'useRetrievedTauInit': False,
187  'tauRetrievalMinCCDPerNight': 500,
188  'modelMagErrors': config.modelMagErrors,
189  'instrumentParsPerBand': config.instrumentParsPerBand,
190  'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
191  'fitMirrorChromaticity': config.fitMirrorChromaticity,
192  'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
193  'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
194  'autoHighCutNSig': config.autoHighCutNSig,
195  'printOnly': False,
196  'quietMode': config.quietMode,
197  'randomSeed': config.randomSeed,
198  'outputStars': False,
199  'clobber': True,
200  'useSedLUT': False,
201  'resetParameters': resetFitParameters,
202  'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
203  'outputZeropoints': outputZeropoints}
204 
205  return configDict
206 
207 
208 def translateFgcmLut(lutCat, physicalFilterMap):
209  """
210  Translate the FGCM look-up-table into an fgcm-compatible object
211 
212  Parameters
213  ----------
214  lutCat: `lsst.afw.table.BaseCatalog`
215  Catalog describing the FGCM look-up table
216  physicalFilterMap: `dict`
217  Physical filter to band mapping
218 
219  Returns
220  -------
221  fgcmLut: `lsst.fgcm.FgcmLut`
222  Lookup table for FGCM
223  lutIndexVals: `numpy.ndarray`
224  Numpy array with LUT index information for FGCM
225  lutStd: `numpy.ndarray`
226  Numpy array with LUT standard throughput values for FGCM
227 
228  Notes
229  -----
230  After running this code, it is wise to `del lutCat` to clear the memory.
231  """
232 
233  # first we need the lutIndexVals
234  # dtype is set for py2/py3/fits/fgcm compatibility
235  lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='a')
236  lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='a')
237 
238  # Note that any discrepancies between config values will raise relevant
239  # exceptions in the FGCM code.
240 
241  lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
242  lutFilterNames.size),
243  ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
244  lutStdFilterNames.size),
245  ('PMB', 'f8', lutCat[0]['pmb'].size),
246  ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
247  ('PMBELEVATION', 'f8'),
248  ('LAMBDANORM', 'f8'),
249  ('PWV', 'f8', lutCat[0]['pwv'].size),
250  ('O3', 'f8', lutCat[0]['o3'].size),
251  ('TAU', 'f8', lutCat[0]['tau'].size),
252  ('ALPHA', 'f8', lutCat[0]['alpha'].size),
253  ('ZENITH', 'f8', lutCat[0]['zenith'].size),
254  ('NCCD', 'i4')])
255 
256  lutIndexVals['FILTERNAMES'][:] = lutFilterNames
257  lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
258  lutIndexVals['PMB'][:] = lutCat[0]['pmb']
259  lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
260  lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
261  lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
262  lutIndexVals['PWV'][:] = lutCat[0]['pwv']
263  lutIndexVals['O3'][:] = lutCat[0]['o3']
264  lutIndexVals['TAU'][:] = lutCat[0]['tau']
265  lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
266  lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
267  lutIndexVals['NCCD'] = lutCat[0]['nCcd']
268 
269  # now we need the Standard Values
270  lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
271  ('PWVSTD', 'f8'),
272  ('O3STD', 'f8'),
273  ('TAUSTD', 'f8'),
274  ('ALPHASTD', 'f8'),
275  ('ZENITHSTD', 'f8'),
276  ('LAMBDARANGE', 'f8', 2),
277  ('LAMBDASTEP', 'f8'),
278  ('LAMBDASTD', 'f8', lutFilterNames.size),
279  ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
280  ('I0STD', 'f8', lutFilterNames.size),
281  ('I1STD', 'f8', lutFilterNames.size),
282  ('I10STD', 'f8', lutFilterNames.size),
283  ('I2STD', 'f8', lutFilterNames.size),
284  ('LAMBDAB', 'f8', lutFilterNames.size),
285  ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
286  ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
287  lutStd['PMBSTD'] = lutCat[0]['pmbStd']
288  lutStd['PWVSTD'] = lutCat[0]['pwvStd']
289  lutStd['O3STD'] = lutCat[0]['o3Std']
290  lutStd['TAUSTD'] = lutCat[0]['tauStd']
291  lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
292  lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
293  lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
294  lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
295  lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
296  lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
297  lutStd['I0STD'][:] = lutCat[0]['i0Std']
298  lutStd['I1STD'][:] = lutCat[0]['i1Std']
299  lutStd['I10STD'][:] = lutCat[0]['i10Std']
300  lutStd['I2STD'][:] = lutCat[0]['i2Std']
301  lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
302  lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
303  lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
304 
305  lutTypes = [row['luttype'] for row in lutCat]
306 
307  # And the flattened look-up-table
308  lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
309  ('I1', 'f4')])
310 
311  lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
312  lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
313 
314  lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
315  ('D_O3', 'f4'),
316  ('D_LNTAU', 'f4'),
317  ('D_ALPHA', 'f4'),
318  ('D_SECZENITH', 'f4'),
319  ('D_LNPWV_I1', 'f4'),
320  ('D_O3_I1', 'f4'),
321  ('D_LNTAU_I1', 'f4'),
322  ('D_ALPHA_I1', 'f4'),
323  ('D_SECZENITH_I1', 'f4')])
324 
325  for name in lutDerivFlat.dtype.names:
326  lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
327 
328  # The fgcm.FgcmLUT() class copies all the LUT information into special
329  # shared memory objects that will not blow up the memory usage when used
330  # with python multiprocessing. Once all the numbers are copied, the
331  # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
332  # will fall out of scope and can be cleaned up by the garbage collector.
333  fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
334  filterToBand=physicalFilterMap)
335 
336  return fgcmLut, lutIndexVals, lutStd
337 
338 
339 def translateVisitCatalog(visitCat):
340  """
341  Translate the FGCM visit catalog to an fgcm-compatible object
342 
343  Parameters
344  ----------
345  visitCat: `lsst.afw.table.BaseCatalog`
346  FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
347 
348  Returns
349  -------
350  fgcmExpInfo: `numpy.ndarray`
351  Numpy array for visit information for FGCM
352 
353  Notes
354  -----
355  After running this code, it is wise to `del visitCat` to clear the memory.
356  """
357 
358  fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
359  ('MJD', 'f8'),
360  ('EXPTIME', 'f8'),
361  ('PSFSIGMA', 'f8'),
362  ('DELTA_APER', 'f8'),
363  ('SKYBACKGROUND', 'f8'),
364  ('DEEPFLAG', 'i2'),
365  ('TELHA', 'f8'),
366  ('TELRA', 'f8'),
367  ('TELDEC', 'f8'),
368  ('TELROT', 'f8'),
369  ('PMB', 'f8'),
370  ('FILTERNAME', 'a50')])
371  fgcmExpInfo['VISIT'][:] = visitCat['visit']
372  fgcmExpInfo['MJD'][:] = visitCat['mjd']
373  fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
374  fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
375  fgcmExpInfo['TELHA'][:] = visitCat['telha']
376  fgcmExpInfo['TELRA'][:] = visitCat['telra']
377  fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
378  fgcmExpInfo['TELROT'][:] = visitCat['telrot']
379  fgcmExpInfo['PMB'][:] = visitCat['pmb']
380  fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
381  fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
382  fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
383  # Note that we have to go through asAstropy() to get a string
384  # array out of an afwTable. This is faster than a row-by-row loop.
385  fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter']
386 
387  return fgcmExpInfo
388 
389 
390 def computeCcdOffsets(camera, defaultOrientation):
391  """
392  Compute the CCD offsets in ra/dec and x/y space
393 
394  Parameters
395  ----------
396  camera: `lsst.afw.cameraGeom.Camera`
397  defaultOrientation: `float`
398  Default camera orientation (degrees)
399 
400  Returns
401  -------
402  ccdOffsets: `numpy.ndarray`
403  Numpy array with ccd offset information for input to FGCM.
404  Angular units are degrees, and x/y units are pixels.
405  """
406  # TODO: DM-21215 will fully generalize to arbitrary camera orientations
407 
408  # and we need to know the ccd offsets from the camera geometry
409  ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'),
410  ('DELTA_RA', 'f8'),
411  ('DELTA_DEC', 'f8'),
412  ('RA_SIZE', 'f8'),
413  ('DEC_SIZE', 'f8'),
414  ('X_SIZE', 'i4'),
415  ('Y_SIZE', 'i4')])
416 
417  # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
418  # since we are looking for relative positions
419  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
420 
421  # TODO: DM-17597 will update testdata_jointcal so that the test data
422  # does not have nan as the boresight angle for HSC data. For the
423  # time being, there is this ungainly hack.
424  if camera.getName() == 'HSC' and np.isnan(defaultOrientation):
425  orientation = 270*geom.degrees
426  else:
427  orientation = defaultOrientation*geom.degrees
428  flipX = False
429 
430  # Create a temporary visitInfo for input to createInitialSkyWcs
431  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
432  boresightRotAngle=orientation,
433  rotType=afwImage.visitInfo.RotType.SKY)
434 
435  for i, detector in enumerate(camera):
436  ccdOffsets['CCDNUM'][i] = detector.getId()
437 
438  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
439 
440  detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS))
441  ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees()
442  ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees()
443 
444  bbox = detector.getBBox()
445 
446  detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin()))
447  detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax()))
448 
449  ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees())
450  ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees())
451 
452  ccdOffsets['X_SIZE'][i] = bbox.getMaxX()
453  ccdOffsets['Y_SIZE'][i] = bbox.getMaxY()
454 
455  return ccdOffsets
456 
457 
459  """
460  Compute the median pixel scale in the camera
461 
462  Returns
463  -------
464  pixelScale: `float`
465  Average pixel scale (arcsecond) over the camera
466  """
467 
468  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
469  orientation = 0.0*geom.degrees
470  flipX = False
471 
472  # Create a temporary visitInfo for input to createInitialSkyWcs
473  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
474  boresightRotAngle=orientation,
475  rotType=afwImage.visitInfo.RotType.SKY)
476 
477  pixelScales = np.zeros(len(camera))
478  for i, detector in enumerate(camera):
479  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
480  pixelScales[i] = wcs.getPixelScale().asArcseconds()
481 
482  ok, = np.where(pixelScales > 0.0)
483  return np.median(pixelScales[ok])
484 
485 
487  """
488  Compute the approximate pixel area bounded fields from the camera
489  geometry.
490 
491  Parameters
492  ----------
493  camera: `lsst.afw.cameraGeom.Camera`
494 
495  Returns
496  -------
497  approxPixelAreaFields: `dict`
498  Dictionary of approximate area fields, keyed with detector ID
499  """
500 
501  areaScaling = 1. / computeReferencePixelScale(camera)**2.
502 
503  # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
504  # since we are looking for relative scales
505  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
506 
507  flipX = False
508  # Create a temporary visitInfo for input to createInitialSkyWcs
509  # The orientation does not matter for the area computation
510  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
511  boresightRotAngle=0.0*geom.degrees,
512  rotType=afwImage.visitInfo.RotType.SKY)
513 
514  approxPixelAreaFields = {}
515 
516  for i, detector in enumerate(camera):
517  key = detector.getId()
518 
519  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
520  bbox = detector.getBBox()
521 
522  areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
523  unit=geom.arcseconds, scaling=areaScaling)
524  approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
525 
526  approxPixelAreaFields[key] = approxAreaField
527 
528  return approxPixelAreaFields
529 
530 
531 def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
532  """
533  Make the zeropoint schema
534 
535  Parameters
536  ----------
537  superStarChebyshevSize: `int`
538  Length of the superstar chebyshev array
539  zptChebyshevSize: `int`
540  Length of the zeropoint chebyshev array
541 
542  Returns
543  -------
544  zptSchema: `lsst.afw.table.schema`
545  """
546 
547  zptSchema = afwTable.Schema()
548 
549  zptSchema.addField('visit', type=np.int32, doc='Visit number')
550  zptSchema.addField('detector', type=np.int32, doc='Detector ID number')
551  zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
552  '1: Photometric, used in fit; '
553  '2: Photometric, not used in fit; '
554  '4: Non-photometric, on partly photometric night; '
555  '8: Non-photometric, on non-photometric night; '
556  '16: No zeropoint could be determined; '
557  '32: Too few stars for reliable gray computation'))
558  zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
559  zptSchema.addField('fgcmZptErr', type=np.float64,
560  doc='Error on zeropoint, estimated from repeatability + number of obs')
561  zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
562  doc='maximum x/maximum y to scale to apply chebyshev parameters')
563  zptSchema.addField('fgcmfZptCheb', type='ArrayD',
564  size=zptChebyshevSize,
565  doc='Chebyshev parameters (flattened) for zeropoint')
566  zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
567  size=superStarChebyshevSize,
568  doc='Chebyshev parameters (flattened) for superStarFlat')
569  zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
570  zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
571  zptSchema.addField('fgcmR0', type=np.float64,
572  doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
573  zptSchema.addField('fgcmR10', type=np.float64,
574  doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
575  zptSchema.addField('fgcmGry', type=np.float64,
576  doc='Estimated gray extinction relative to atmospheric solution; '
577  'only for fgcmFlag <= 4 (see fgcmFlag) ')
578  zptSchema.addField('fgcmDeltaChrom', type=np.float64,
579  doc='Mean chromatic correction for stars in this ccd; '
580  'only for fgcmFlag <= 4 (see fgcmFlag)')
581  zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
582  zptSchema.addField('fgcmTilings', type=np.float64,
583  doc='Number of photometric tilings used for solution for ccd')
584  zptSchema.addField('fgcmFpGry', type=np.float64,
585  doc='Average gray extinction over the full focal plane '
586  '(same for all ccds in a visit)')
587  zptSchema.addField('fgcmFpGryBlue', type=np.float64,
588  doc='Average gray extinction over the full focal plane '
589  'for 25% bluest stars')
590  zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
591  doc='Error on Average gray extinction over the full focal plane '
592  'for 25% bluest stars')
593  zptSchema.addField('fgcmFpGryRed', type=np.float64,
594  doc='Average gray extinction over the full focal plane '
595  'for 25% reddest stars')
596  zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
597  doc='Error on Average gray extinction over the full focal plane '
598  'for 25% reddest stars')
599  zptSchema.addField('fgcmFpVar', type=np.float64,
600  doc='Variance of gray extinction over the full focal plane '
601  '(same for all ccds in a visit)')
602  zptSchema.addField('fgcmDust', type=np.float64,
603  doc='Gray dust extinction from the primary/corrector'
604  'at the time of the exposure')
605  zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
606  zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
607  zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
608  doc=('Local background correction from brightest percentile '
609  '(value set by deltaMagBkgOffsetPercentile) calibration '
610  'stars.'))
611  zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
612  zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
613 
614  return zptSchema
615 
616 
617 def makeZptCat(zptSchema, zpStruct):
618  """
619  Make the zeropoint catalog for persistence
620 
621  Parameters
622  ----------
623  zptSchema: `lsst.afw.table.Schema`
624  Zeropoint catalog schema
625  zpStruct: `numpy.ndarray`
626  Zeropoint structure from fgcm
627 
628  Returns
629  -------
630  zptCat: `afwTable.BaseCatalog`
631  Zeropoint catalog for persistence
632  """
633 
634  zptCat = afwTable.BaseCatalog(zptSchema)
635  zptCat.reserve(zpStruct.size)
636 
637  for filterName in zpStruct['FILTERNAME']:
638  rec = zptCat.addNew()
639  rec['filtername'] = filterName.decode('utf-8')
640 
641  zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD]
642  zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD]
643  zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
644  zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
645  zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
646  zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
647  zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
648  zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
649  zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
650  zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
651  zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
652  zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
653  zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
654  zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
655  zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
656  zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
657  zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
658  zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
659  zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
660  zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
661  zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
662  zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
663  zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
664  zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
665  zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
666  zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
667  zptCat['exptime'][:] = zpStruct['EXPTIME']
668 
669  return zptCat
670 
671 
673  """
674  Make the atmosphere schema
675 
676  Returns
677  -------
678  atmSchema: `lsst.afw.table.Schema`
679  """
680 
681  atmSchema = afwTable.Schema()
682 
683  atmSchema.addField('visit', type=np.int32, doc='Visit number')
684  atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
685  atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
686  atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
687  atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
688  atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
689  atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
690  atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
691  atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
692 
693  return atmSchema
694 
695 
696 def makeAtmCat(atmSchema, atmStruct):
697  """
698  Make the atmosphere catalog for persistence
699 
700  Parameters
701  ----------
702  atmSchema: `lsst.afw.table.Schema`
703  Atmosphere catalog schema
704  atmStruct: `numpy.ndarray`
705  Atmosphere structure from fgcm
706 
707  Returns
708  -------
709  atmCat: `lsst.afw.table.BaseCatalog`
710  Atmosphere catalog for persistence
711  """
712 
713  atmCat = afwTable.BaseCatalog(atmSchema)
714  atmCat.resize(atmStruct.size)
715 
716  atmCat['visit'][:] = atmStruct['VISIT']
717  atmCat['pmb'][:] = atmStruct['PMB']
718  atmCat['pwv'][:] = atmStruct['PWV']
719  atmCat['tau'][:] = atmStruct['TAU']
720  atmCat['alpha'][:] = atmStruct['ALPHA']
721  atmCat['o3'][:] = atmStruct['O3']
722  atmCat['secZenith'][:] = atmStruct['SECZENITH']
723  atmCat['cTrans'][:] = atmStruct['CTRANS']
724  atmCat['lamStd'][:] = atmStruct['LAMSTD']
725 
726  return atmCat
727 
728 
729 def makeStdSchema(nBands):
730  """
731  Make the standard star schema
732 
733  Parameters
734  ----------
735  nBands: `int`
736  Number of bands in standard star catalog
737 
738  Returns
739  -------
740  stdSchema: `lsst.afw.table.Schema`
741  """
742 
743  stdSchema = afwTable.SimpleTable.makeMinimalSchema()
744  stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
745  size=nBands)
746  stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
747  size=nBands)
748  stdSchema.addField('mag_std_noabs', type='ArrayF',
749  doc='Standard magnitude (no absolute calibration)',
750  size=nBands)
751  stdSchema.addField('magErr_std', type='ArrayF',
752  doc='Standard magnitude error',
753  size=nBands)
754  stdSchema.addField('npsfcand', type='ArrayI',
755  doc='Number of observations flagged as psf candidates',
756  size=nBands)
757 
758  return stdSchema
759 
760 
761 def makeStdCat(stdSchema, stdStruct, goodBands):
762  """
763  Make the standard star catalog for persistence
764 
765  Parameters
766  ----------
767  stdSchema: `lsst.afw.table.Schema`
768  Standard star catalog schema
769  stdStruct: `numpy.ndarray`
770  Standard star structure in FGCM format
771  goodBands: `list`
772  List of good band names used in stdStruct
773 
774  Returns
775  -------
776  stdCat: `lsst.afw.table.BaseCatalog`
777  Standard star catalog for persistence
778  """
779 
780  stdCat = afwTable.SimpleCatalog(stdSchema)
781  stdCat.resize(stdStruct.size)
782 
783  stdCat['id'][:] = stdStruct['FGCM_ID']
784  stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
785  stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
786  stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
787  stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
788  stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
789  stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
790  stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
791 
792  md = PropertyList()
793  md.set("BANDS", list(goodBands))
794  stdCat.setMetadata(md)
795 
796  return stdCat
797 
798 
799 def computeApertureRadiusFromDataRef(dataRef, fluxField):
800  """
801  Compute the radius associated with a CircularApertureFlux field or
802  associated slot.
803 
804  Parameters
805  ----------
806  dataRef : `lsst.daf.persistence.ButlerDataRef` or
807  `lsst.daf.butler.DeferredDatasetHandle`
808  fluxField : `str`
809  CircularApertureFlux or associated slot.
810 
811  Returns
812  -------
813  apertureRadius : `float`
814  Radius of the aperture field, in pixels.
815 
816  Raises
817  ------
818  RuntimeError: Raised if flux field is not a CircularApertureFlux, ApFlux,
819  or associated slot.
820  """
821  # TODO: Move this method to more general stack method in DM-25775
822  if isinstance(dataRef, dafPersist.ButlerDataRef):
823  # Gen2 dataRef
824  datasetType = dataRef.butlerSubset.datasetType
825  else:
826  # Gen3 dataRef
827  datasetType = dataRef.ref.datasetType.name
828 
829  if datasetType == 'src':
830  schema = dataRef.get(datasetType='src_schema').schema
831  try:
832  fluxFieldName = schema[fluxField].asField().getName()
833  except LookupError:
834  raise RuntimeError("Could not find %s or associated slot in schema." % (fluxField))
835  # This may also raise a RuntimeError
836  apertureRadius = computeApertureRadiusFromName(fluxFieldName)
837  else:
838  # This is a sourceTable_visit
839  apertureRadius = computeApertureRadiusFromName(fluxField)
840 
841  return apertureRadius
842 
843 
845  """
846  Compute the radius associated with a CircularApertureFlux or ApFlux field.
847 
848  Parameters
849  ----------
850  fluxField : `str`
851  CircularApertureFlux or ApFlux
852 
853  Returns
854  -------
855  apertureRadius : `float`
856  Radius of the aperture field, in pixels.
857 
858  Raises
859  ------
860  RuntimeError: Raised if flux field is not a CircularApertureFlux
861  or ApFlux.
862  """
863  # TODO: Move this method to more general stack method in DM-25775
864  m = re.search(r'(CircularApertureFlux|ApFlux)_(\d+)_(\d+)_', fluxField)
865 
866  if m is None:
867  raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
868 
869  apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
870 
871  return apertureRadius
872 
873 
874 def extractReferenceMags(refStars, bands, filterMap):
875  """
876  Extract reference magnitudes from refStars for given bands and
877  associated filterMap.
878 
879  Parameters
880  ----------
881  refStars : `lsst.afw.table.BaseCatalog`
882  FGCM reference star catalog
883  bands : `list`
884  List of bands for calibration
885  filterMap: `dict`
886  FGCM mapping of filter to band
887 
888  Returns
889  -------
890  refMag : `np.ndarray`
891  nstar x nband array of reference magnitudes
892  refMagErr : `np.ndarray`
893  nstar x nband array of reference magnitude errors
894  """
895  # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent
896  # against index errors and allow more flexibility in fitting after
897  # the build stars step.
898 
899  md = refStars.getMetadata()
900  if 'FILTERNAMES' in md:
901  filternames = md.getArray('FILTERNAMES')
902 
903  # The reference catalog that fgcm wants has one entry per band
904  # in the config file
905  refMag = np.zeros((len(refStars), len(bands)),
906  dtype=refStars['refMag'].dtype) + 99.0
907  refMagErr = np.zeros_like(refMag) + 99.0
908  for i, filtername in enumerate(filternames):
909  # We are allowed to run the fit configured so that we do not
910  # use every column in the reference catalog.
911  try:
912  band = filterMap[filtername]
913  except KeyError:
914  continue
915  try:
916  ind = bands.index(band)
917  except ValueError:
918  continue
919 
920  refMag[:, ind] = refStars['refMag'][:, i]
921  refMagErr[:, ind] = refStars['refMagErr'][:, i]
922 
923  else:
924  # Continue to use old catalogs as before.
925  refMag = refStars['refMag'][:, :]
926  refMagErr = refStars['refMagErr'][:, :]
927 
928  return refMag, refMagErr
929 
930 
931 def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
932  instrument = Instrument.fromName(quantumDataId["instrument"], registry)
933  unboundedCollection = instrument.makeUnboundedCalibrationRunName()
934 
935  return registry.queryDatasets(datasetType,
936  dataId=quantumDataId,
937  collections=[unboundedCollection])
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:874
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition: utilities.py:931
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:799
def makeStdSchema(nBands)
Definition: utilities.py:729
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
Definition: utilities.py:49
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:696
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:208
def computeReferencePixelScale(camera)
Definition: utilities.py:458
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:617
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:761
def computeApertureRadiusFromName(fluxField)
Definition: utilities.py:844
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:486
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:531
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:390
def translateVisitCatalog(visitCat)
Definition: utilities.py:339