lsst.fgcmcal  21.0.0-15-g543e7c3+54360a7183
utilities.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Utility functions for fgcmcal.
22 
23 This file contains utility functions that are used by more than one task,
24 and do not need to be part of a task.
25 """
26 
27 import numpy as np
28 import os
29 import re
30 
31 from lsst.daf.base import PropertyList
32 import lsst.daf.persistence as dafPersist
33 import lsst.afw.cameraGeom as afwCameraGeom
34 import lsst.afw.table as afwTable
35 import lsst.afw.image as afwImage
36 import lsst.afw.math as afwMath
37 import lsst.geom as geom
38 from lsst.obs.base import createInitialSkyWcs
39 from lsst.obs.base import Instrument
40 
41 import fgcm
42 
43 
44 FGCM_EXP_FIELD = 'VISIT'
45 FGCM_CCD_FIELD = 'DETECTOR'
46 
47 
48 def makeConfigDict(config, log, camera, maxIter,
49  resetFitParameters, outputZeropoints,
50  lutFilterNames, tract=None):
51  """
52  Make the FGCM fit cycle configuration dict
53 
54  Parameters
55  ----------
56  config: `lsst.fgcmcal.FgcmFitCycleConfig`
57  Configuration object
58  log: `lsst.log.Log`
59  LSST log object
60  camera: `lsst.afw.cameraGeom.Camera`
61  Camera from the butler
62  maxIter: `int`
63  Maximum number of iterations
64  resetFitParameters: `bool`
65  Reset fit parameters before fitting?
66  outputZeropoints: `bool`
67  Compute zeropoints for output?
68  lutFilterNames : array-like, `str`
69  Array of physical filter names in the LUT.
70  tract: `int`, optional
71  Tract number for extending the output file name for debugging.
72  Default is None.
73 
74  Returns
75  -------
76  configDict: `dict`
77  Configuration dictionary for fgcm
78  """
79  # Extract the bands that are _not_ being fit for fgcm configuration
80  notFitBands = [b for b in config.bands if b not in config.fitBands]
81 
82  # process the starColorCuts
83  starColorCutList = []
84  for ccut in config.starColorCuts:
85  parts = ccut.split(',')
86  starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
87 
88  # TODO: Having direct access to the mirror area from the camera would be
89  # useful. See DM-16489.
90  # Mirror area in cm**2
91  mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
92 
93  # Get approximate average camera gain:
94  gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
95  cameraGain = float(np.median(gains))
96 
97  # Cut down the filter map to those that are in the LUT
98  filterToBand = {filterName: config.physicalFilterMap[filterName] for
99  filterName in lutFilterNames}
100 
101  if tract is None:
102  outfileBase = config.outfileBase
103  else:
104  outfileBase = '%s-%06d' % (config.outfileBase, tract)
105 
106  # create a configuration dictionary for fgcmFitCycle
107  configDict = {'outfileBase': outfileBase,
108  'logger': log,
109  'exposureFile': None,
110  'obsFile': None,
111  'indexFile': None,
112  'lutFile': None,
113  'mirrorArea': mirrorArea,
114  'cameraGain': cameraGain,
115  'ccdStartIndex': camera[0].getId(),
116  'expField': FGCM_EXP_FIELD,
117  'ccdField': FGCM_CCD_FIELD,
118  'seeingField': 'DELTA_APER',
119  'fwhmField': 'PSFSIGMA',
120  'skyBrightnessField': 'SKYBACKGROUND',
121  'deepFlag': 'DEEPFLAG', # unused
122  'bands': list(config.bands),
123  'fitBands': list(config.fitBands),
124  'notFitBands': notFitBands,
125  'requiredBands': list(config.requiredBands),
126  'filterToBand': filterToBand,
127  'logLevel': 'INFO',
128  'nCore': config.nCore,
129  'nStarPerRun': config.nStarPerRun,
130  'nExpPerRun': config.nExpPerRun,
131  'reserveFraction': config.reserveFraction,
132  'freezeStdAtmosphere': config.freezeStdAtmosphere,
133  'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
134  'superStarSubCCDDict': dict(config.superStarSubCcdDict),
135  'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
136  'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
137  'superStarSigmaClip': config.superStarSigmaClip,
138  'focalPlaneSigmaClip': config.focalPlaneSigmaClip,
139  'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
140  'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
141  'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
142  'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict),
143  'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder,
144  'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd,
145  'cycleNumber': config.cycleNumber,
146  'maxIter': maxIter,
147  'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
148  'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
149  'UTBoundary': config.utBoundary,
150  'washMJDs': config.washMjds,
151  'epochMJDs': config.epochMjds,
152  'coatingMJDs': config.coatingMjds,
153  'minObsPerBand': config.minObsPerBand,
154  'latitude': config.latitude,
155  'brightObsGrayMax': config.brightObsGrayMax,
156  'minStarPerCCD': config.minStarPerCcd,
157  'minCCDPerExp': config.minCcdPerExp,
158  'maxCCDGrayErr': config.maxCcdGrayErr,
159  'minStarPerExp': config.minStarPerExp,
160  'minExpPerNight': config.minExpPerNight,
161  'expGrayInitialCut': config.expGrayInitialCut,
162  'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
163  'expGrayHighCutDict': dict(config.expGrayHighCutDict),
164  'expGrayRecoverCut': config.expGrayRecoverCut,
165  'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
166  'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
167  'refStarSnMin': config.refStarSnMin,
168  'refStarOutlierNSig': config.refStarOutlierNSig,
169  'applyRefStarColorCuts': config.applyRefStarColorCuts,
170  'illegalValue': -9999.0, # internally used by fgcm.
171  'starColorCuts': starColorCutList,
172  'aperCorrFitNBins': config.aperCorrFitNBins,
173  'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
174  'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
175  'sedTermDict': config.sedterms.toDict()['data'],
176  'colorSplitBands': list(config.colorSplitBands),
177  'sigFgcmMaxErr': config.sigFgcmMaxErr,
178  'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
179  'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
180  'approxThroughputDict': dict(config.approxThroughputDict),
181  'sigmaCalRange': list(config.sigmaCalRange),
182  'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
183  'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
184  'sigma0Phot': config.sigma0Phot,
185  'mapLongitudeRef': config.mapLongitudeRef,
186  'mapNSide': config.mapNSide,
187  'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
188  'varMinBand': 2,
189  'useRetrievedPwv': False,
190  'useNightlyRetrievedPwv': False,
191  'pwvRetrievalSmoothBlock': 25,
192  'useQuadraticPwv': config.useQuadraticPwv,
193  'useRetrievedTauInit': False,
194  'tauRetrievalMinCCDPerNight': 500,
195  'modelMagErrors': config.modelMagErrors,
196  'instrumentParsPerBand': config.instrumentParsPerBand,
197  'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
198  'fitMirrorChromaticity': config.fitMirrorChromaticity,
199  'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
200  'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
201  'autoHighCutNSig': config.autoHighCutNSig,
202  'printOnly': False,
203  'quietMode': config.quietMode,
204  'randomSeed': config.randomSeed,
205  'outputStars': False,
206  'outputPath': os.path.abspath('.'),
207  'clobber': True,
208  'useSedLUT': False,
209  'resetParameters': resetFitParameters,
210  'doPlots': config.doPlots,
211  'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
212  'outputZeropoints': outputZeropoints}
213 
214  return configDict
215 
216 
217 def translateFgcmLut(lutCat, physicalFilterMap):
218  """
219  Translate the FGCM look-up-table into an fgcm-compatible object
220 
221  Parameters
222  ----------
223  lutCat: `lsst.afw.table.BaseCatalog`
224  Catalog describing the FGCM look-up table
225  physicalFilterMap: `dict`
226  Physical filter to band mapping
227 
228  Returns
229  -------
230  fgcmLut: `lsst.fgcm.FgcmLut`
231  Lookup table for FGCM
232  lutIndexVals: `numpy.ndarray`
233  Numpy array with LUT index information for FGCM
234  lutStd: `numpy.ndarray`
235  Numpy array with LUT standard throughput values for FGCM
236 
237  Notes
238  -----
239  After running this code, it is wise to `del lutCat` to clear the memory.
240  """
241 
242  # first we need the lutIndexVals
243  lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U')
244  lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U')
245 
246  # Note that any discrepancies between config values will raise relevant
247  # exceptions in the FGCM code.
248 
249  lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
250  lutFilterNames.size),
251  ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
252  lutStdFilterNames.size),
253  ('PMB', 'f8', lutCat[0]['pmb'].size),
254  ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
255  ('PMBELEVATION', 'f8'),
256  ('LAMBDANORM', 'f8'),
257  ('PWV', 'f8', lutCat[0]['pwv'].size),
258  ('O3', 'f8', lutCat[0]['o3'].size),
259  ('TAU', 'f8', lutCat[0]['tau'].size),
260  ('ALPHA', 'f8', lutCat[0]['alpha'].size),
261  ('ZENITH', 'f8', lutCat[0]['zenith'].size),
262  ('NCCD', 'i4')])
263 
264  lutIndexVals['FILTERNAMES'][:] = lutFilterNames
265  lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
266  lutIndexVals['PMB'][:] = lutCat[0]['pmb']
267  lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
268  lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
269  lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
270  lutIndexVals['PWV'][:] = lutCat[0]['pwv']
271  lutIndexVals['O3'][:] = lutCat[0]['o3']
272  lutIndexVals['TAU'][:] = lutCat[0]['tau']
273  lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
274  lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
275  lutIndexVals['NCCD'] = lutCat[0]['nCcd']
276 
277  # now we need the Standard Values
278  lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
279  ('PWVSTD', 'f8'),
280  ('O3STD', 'f8'),
281  ('TAUSTD', 'f8'),
282  ('ALPHASTD', 'f8'),
283  ('ZENITHSTD', 'f8'),
284  ('LAMBDARANGE', 'f8', 2),
285  ('LAMBDASTEP', 'f8'),
286  ('LAMBDASTD', 'f8', lutFilterNames.size),
287  ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
288  ('I0STD', 'f8', lutFilterNames.size),
289  ('I1STD', 'f8', lutFilterNames.size),
290  ('I10STD', 'f8', lutFilterNames.size),
291  ('I2STD', 'f8', lutFilterNames.size),
292  ('LAMBDAB', 'f8', lutFilterNames.size),
293  ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
294  ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
295  lutStd['PMBSTD'] = lutCat[0]['pmbStd']
296  lutStd['PWVSTD'] = lutCat[0]['pwvStd']
297  lutStd['O3STD'] = lutCat[0]['o3Std']
298  lutStd['TAUSTD'] = lutCat[0]['tauStd']
299  lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
300  lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
301  lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
302  lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
303  lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
304  lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
305  lutStd['I0STD'][:] = lutCat[0]['i0Std']
306  lutStd['I1STD'][:] = lutCat[0]['i1Std']
307  lutStd['I10STD'][:] = lutCat[0]['i10Std']
308  lutStd['I2STD'][:] = lutCat[0]['i2Std']
309  lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
310  lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
311  lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
312 
313  lutTypes = [row['luttype'] for row in lutCat]
314 
315  # And the flattened look-up-table
316  lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
317  ('I1', 'f4')])
318 
319  lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
320  lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
321 
322  lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
323  ('D_O3', 'f4'),
324  ('D_LNTAU', 'f4'),
325  ('D_ALPHA', 'f4'),
326  ('D_SECZENITH', 'f4'),
327  ('D_LNPWV_I1', 'f4'),
328  ('D_O3_I1', 'f4'),
329  ('D_LNTAU_I1', 'f4'),
330  ('D_ALPHA_I1', 'f4'),
331  ('D_SECZENITH_I1', 'f4')])
332 
333  for name in lutDerivFlat.dtype.names:
334  lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
335 
336  # The fgcm.FgcmLUT() class copies all the LUT information into special
337  # shared memory objects that will not blow up the memory usage when used
338  # with python multiprocessing. Once all the numbers are copied, the
339  # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
340  # will fall out of scope and can be cleaned up by the garbage collector.
341  fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
342  filterToBand=physicalFilterMap)
343 
344  return fgcmLut, lutIndexVals, lutStd
345 
346 
347 def translateVisitCatalog(visitCat):
348  """
349  Translate the FGCM visit catalog to an fgcm-compatible object
350 
351  Parameters
352  ----------
353  visitCat: `lsst.afw.table.BaseCatalog`
354  FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
355 
356  Returns
357  -------
358  fgcmExpInfo: `numpy.ndarray`
359  Numpy array for visit information for FGCM
360 
361  Notes
362  -----
363  After running this code, it is wise to `del visitCat` to clear the memory.
364  """
365 
366  fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
367  ('MJD', 'f8'),
368  ('EXPTIME', 'f8'),
369  ('PSFSIGMA', 'f8'),
370  ('DELTA_APER', 'f8'),
371  ('SKYBACKGROUND', 'f8'),
372  ('DEEPFLAG', 'i2'),
373  ('TELHA', 'f8'),
374  ('TELRA', 'f8'),
375  ('TELDEC', 'f8'),
376  ('TELROT', 'f8'),
377  ('PMB', 'f8'),
378  ('FILTERNAME', 'a50')])
379  fgcmExpInfo['VISIT'][:] = visitCat['visit']
380  fgcmExpInfo['MJD'][:] = visitCat['mjd']
381  fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
382  fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
383  fgcmExpInfo['TELHA'][:] = visitCat['telha']
384  fgcmExpInfo['TELRA'][:] = visitCat['telra']
385  fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
386  fgcmExpInfo['TELROT'][:] = visitCat['telrot']
387  fgcmExpInfo['PMB'][:] = visitCat['pmb']
388  fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
389  fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
390  fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
391  # Note that we have to go through asAstropy() to get a string
392  # array out of an afwTable. This is faster than a row-by-row loop.
393  fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter']
394 
395  return fgcmExpInfo
396 
397 
398 def computeCcdOffsets(camera, defaultOrientation):
399  """
400  Compute the CCD offsets in ra/dec and x/y space
401 
402  Parameters
403  ----------
404  camera: `lsst.afw.cameraGeom.Camera`
405  defaultOrientation: `float`
406  Default camera orientation (degrees)
407 
408  Returns
409  -------
410  ccdOffsets: `numpy.ndarray`
411  Numpy array with ccd offset information for input to FGCM.
412  Angular units are degrees, and x/y units are pixels.
413  """
414  # TODO: DM-21215 will fully generalize to arbitrary camera orientations
415 
416  # and we need to know the ccd offsets from the camera geometry
417  ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'),
418  ('DELTA_RA', 'f8'),
419  ('DELTA_DEC', 'f8'),
420  ('RA_SIZE', 'f8'),
421  ('DEC_SIZE', 'f8'),
422  ('X_SIZE', 'i4'),
423  ('Y_SIZE', 'i4')])
424 
425  # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
426  # since we are looking for relative positions
427  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
428 
429  # TODO: DM-17597 will update testdata_jointcal so that the test data
430  # does not have nan as the boresight angle for HSC data. For the
431  # time being, there is this ungainly hack.
432  if camera.getName() == 'HSC' and np.isnan(defaultOrientation):
433  orientation = 270*geom.degrees
434  else:
435  orientation = defaultOrientation*geom.degrees
436  flipX = False
437 
438  # Create a temporary visitInfo for input to createInitialSkyWcs
439  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
440  boresightRotAngle=orientation,
441  rotType=afwImage.RotType.SKY)
442 
443  for i, detector in enumerate(camera):
444  ccdOffsets['CCDNUM'][i] = detector.getId()
445 
446  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
447 
448  detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS))
449  ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees()
450  ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees()
451 
452  bbox = detector.getBBox()
453 
454  detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin()))
455  detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax()))
456 
457  ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees())
458  ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees())
459 
460  ccdOffsets['X_SIZE'][i] = bbox.getMaxX()
461  ccdOffsets['Y_SIZE'][i] = bbox.getMaxY()
462 
463  return ccdOffsets
464 
465 
467  """
468  Compute the median pixel scale in the camera
469 
470  Returns
471  -------
472  pixelScale: `float`
473  Average pixel scale (arcsecond) over the camera
474  """
475 
476  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
477  orientation = 0.0*geom.degrees
478  flipX = False
479 
480  # Create a temporary visitInfo for input to createInitialSkyWcs
481  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
482  boresightRotAngle=orientation,
483  rotType=afwImage.RotType.SKY)
484 
485  pixelScales = np.zeros(len(camera))
486  for i, detector in enumerate(camera):
487  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
488  pixelScales[i] = wcs.getPixelScale().asArcseconds()
489 
490  ok, = np.where(pixelScales > 0.0)
491  return np.median(pixelScales[ok])
492 
493 
495  """
496  Compute the approximate pixel area bounded fields from the camera
497  geometry.
498 
499  Parameters
500  ----------
501  camera: `lsst.afw.cameraGeom.Camera`
502 
503  Returns
504  -------
505  approxPixelAreaFields: `dict`
506  Dictionary of approximate area fields, keyed with detector ID
507  """
508 
509  areaScaling = 1. / computeReferencePixelScale(camera)**2.
510 
511  # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
512  # since we are looking for relative scales
513  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
514 
515  flipX = False
516  # Create a temporary visitInfo for input to createInitialSkyWcs
517  # The orientation does not matter for the area computation
518  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
519  boresightRotAngle=0.0*geom.degrees,
520  rotType=afwImage.RotType.SKY)
521 
522  approxPixelAreaFields = {}
523 
524  for i, detector in enumerate(camera):
525  key = detector.getId()
526 
527  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
528  bbox = detector.getBBox()
529 
530  areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
531  unit=geom.arcseconds, scaling=areaScaling)
532  approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
533 
534  approxPixelAreaFields[key] = approxAreaField
535 
536  return approxPixelAreaFields
537 
538 
539 def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
540  """
541  Make the zeropoint schema
542 
543  Parameters
544  ----------
545  superStarChebyshevSize: `int`
546  Length of the superstar chebyshev array
547  zptChebyshevSize: `int`
548  Length of the zeropoint chebyshev array
549 
550  Returns
551  -------
552  zptSchema: `lsst.afw.table.schema`
553  """
554 
555  zptSchema = afwTable.Schema()
556 
557  zptSchema.addField('visit', type=np.int32, doc='Visit number')
558  zptSchema.addField('detector', type=np.int32, doc='Detector ID number')
559  zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
560  '1: Photometric, used in fit; '
561  '2: Photometric, not used in fit; '
562  '4: Non-photometric, on partly photometric night; '
563  '8: Non-photometric, on non-photometric night; '
564  '16: No zeropoint could be determined; '
565  '32: Too few stars for reliable gray computation'))
566  zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
567  zptSchema.addField('fgcmZptErr', type=np.float64,
568  doc='Error on zeropoint, estimated from repeatability + number of obs')
569  zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
570  doc='maximum x/maximum y to scale to apply chebyshev parameters')
571  zptSchema.addField('fgcmfZptCheb', type='ArrayD',
572  size=zptChebyshevSize,
573  doc='Chebyshev parameters (flattened) for zeropoint')
574  zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
575  size=superStarChebyshevSize,
576  doc='Chebyshev parameters (flattened) for superStarFlat')
577  zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
578  zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
579  zptSchema.addField('fgcmR0', type=np.float64,
580  doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
581  zptSchema.addField('fgcmR10', type=np.float64,
582  doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
583  zptSchema.addField('fgcmGry', type=np.float64,
584  doc='Estimated gray extinction relative to atmospheric solution; '
585  'only for fgcmFlag <= 4 (see fgcmFlag) ')
586  zptSchema.addField('fgcmDeltaChrom', type=np.float64,
587  doc='Mean chromatic correction for stars in this ccd; '
588  'only for fgcmFlag <= 4 (see fgcmFlag)')
589  zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
590  zptSchema.addField('fgcmTilings', type=np.float64,
591  doc='Number of photometric tilings used for solution for ccd')
592  zptSchema.addField('fgcmFpGry', type=np.float64,
593  doc='Average gray extinction over the full focal plane '
594  '(same for all ccds in a visit)')
595  zptSchema.addField('fgcmFpGryBlue', type=np.float64,
596  doc='Average gray extinction over the full focal plane '
597  'for 25% bluest stars')
598  zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
599  doc='Error on Average gray extinction over the full focal plane '
600  'for 25% bluest stars')
601  zptSchema.addField('fgcmFpGryRed', type=np.float64,
602  doc='Average gray extinction over the full focal plane '
603  'for 25% reddest stars')
604  zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
605  doc='Error on Average gray extinction over the full focal plane '
606  'for 25% reddest stars')
607  zptSchema.addField('fgcmFpVar', type=np.float64,
608  doc='Variance of gray extinction over the full focal plane '
609  '(same for all ccds in a visit)')
610  zptSchema.addField('fgcmDust', type=np.float64,
611  doc='Gray dust extinction from the primary/corrector'
612  'at the time of the exposure')
613  zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
614  zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
615  zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
616  doc=('Local background correction from brightest percentile '
617  '(value set by deltaMagBkgOffsetPercentile) calibration '
618  'stars.'))
619  zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
620  zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
621 
622  return zptSchema
623 
624 
625 def makeZptCat(zptSchema, zpStruct):
626  """
627  Make the zeropoint catalog for persistence
628 
629  Parameters
630  ----------
631  zptSchema: `lsst.afw.table.Schema`
632  Zeropoint catalog schema
633  zpStruct: `numpy.ndarray`
634  Zeropoint structure from fgcm
635 
636  Returns
637  -------
638  zptCat: `afwTable.BaseCatalog`
639  Zeropoint catalog for persistence
640  """
641 
642  zptCat = afwTable.BaseCatalog(zptSchema)
643  zptCat.reserve(zpStruct.size)
644 
645  for filterName in zpStruct['FILTERNAME']:
646  rec = zptCat.addNew()
647  rec['filtername'] = filterName.decode('utf-8')
648 
649  zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD]
650  zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD]
651  zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
652  zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
653  zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
654  zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
655  zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
656  zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
657  zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
658  zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
659  zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
660  zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
661  zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
662  zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
663  zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
664  zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
665  zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
666  zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
667  zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
668  zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
669  zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
670  zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
671  zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
672  zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
673  zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
674  zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
675  zptCat['exptime'][:] = zpStruct['EXPTIME']
676 
677  return zptCat
678 
679 
681  """
682  Make the atmosphere schema
683 
684  Returns
685  -------
686  atmSchema: `lsst.afw.table.Schema`
687  """
688 
689  atmSchema = afwTable.Schema()
690 
691  atmSchema.addField('visit', type=np.int32, doc='Visit number')
692  atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
693  atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
694  atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
695  atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
696  atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
697  atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
698  atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
699  atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
700 
701  return atmSchema
702 
703 
704 def makeAtmCat(atmSchema, atmStruct):
705  """
706  Make the atmosphere catalog for persistence
707 
708  Parameters
709  ----------
710  atmSchema: `lsst.afw.table.Schema`
711  Atmosphere catalog schema
712  atmStruct: `numpy.ndarray`
713  Atmosphere structure from fgcm
714 
715  Returns
716  -------
717  atmCat: `lsst.afw.table.BaseCatalog`
718  Atmosphere catalog for persistence
719  """
720 
721  atmCat = afwTable.BaseCatalog(atmSchema)
722  atmCat.resize(atmStruct.size)
723 
724  atmCat['visit'][:] = atmStruct['VISIT']
725  atmCat['pmb'][:] = atmStruct['PMB']
726  atmCat['pwv'][:] = atmStruct['PWV']
727  atmCat['tau'][:] = atmStruct['TAU']
728  atmCat['alpha'][:] = atmStruct['ALPHA']
729  atmCat['o3'][:] = atmStruct['O3']
730  atmCat['secZenith'][:] = atmStruct['SECZENITH']
731  atmCat['cTrans'][:] = atmStruct['CTRANS']
732  atmCat['lamStd'][:] = atmStruct['LAMSTD']
733 
734  return atmCat
735 
736 
737 def makeStdSchema(nBands):
738  """
739  Make the standard star schema
740 
741  Parameters
742  ----------
743  nBands: `int`
744  Number of bands in standard star catalog
745 
746  Returns
747  -------
748  stdSchema: `lsst.afw.table.Schema`
749  """
750 
751  stdSchema = afwTable.SimpleTable.makeMinimalSchema()
752  stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
753  size=nBands)
754  stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
755  size=nBands)
756  stdSchema.addField('mag_std_noabs', type='ArrayF',
757  doc='Standard magnitude (no absolute calibration)',
758  size=nBands)
759  stdSchema.addField('magErr_std', type='ArrayF',
760  doc='Standard magnitude error',
761  size=nBands)
762  stdSchema.addField('npsfcand', type='ArrayI',
763  doc='Number of observations flagged as psf candidates',
764  size=nBands)
765 
766  return stdSchema
767 
768 
769 def makeStdCat(stdSchema, stdStruct, goodBands):
770  """
771  Make the standard star catalog for persistence
772 
773  Parameters
774  ----------
775  stdSchema: `lsst.afw.table.Schema`
776  Standard star catalog schema
777  stdStruct: `numpy.ndarray`
778  Standard star structure in FGCM format
779  goodBands: `list`
780  List of good band names used in stdStruct
781 
782  Returns
783  -------
784  stdCat: `lsst.afw.table.BaseCatalog`
785  Standard star catalog for persistence
786  """
787 
788  stdCat = afwTable.SimpleCatalog(stdSchema)
789  stdCat.resize(stdStruct.size)
790 
791  stdCat['id'][:] = stdStruct['FGCM_ID']
792  stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
793  stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
794  stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
795  stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
796  stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
797  stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
798  stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
799 
800  md = PropertyList()
801  md.set("BANDS", list(goodBands))
802  stdCat.setMetadata(md)
803 
804  return stdCat
805 
806 
807 def computeApertureRadiusFromDataRef(dataRef, fluxField):
808  """
809  Compute the radius associated with a CircularApertureFlux field or
810  associated slot.
811 
812  Parameters
813  ----------
814  dataRef : `lsst.daf.persistence.ButlerDataRef` or
815  `lsst.daf.butler.DeferredDatasetHandle`
816  fluxField : `str`
817  CircularApertureFlux or associated slot.
818 
819  Returns
820  -------
821  apertureRadius : `float`
822  Radius of the aperture field, in pixels.
823 
824  Raises
825  ------
826  RuntimeError: Raised if flux field is not a CircularApertureFlux, ApFlux,
827  or associated slot.
828  """
829  # TODO: Move this method to more general stack method in DM-25775
830  if isinstance(dataRef, dafPersist.ButlerDataRef):
831  # Gen2 dataRef
832  datasetType = dataRef.butlerSubset.datasetType
833  else:
834  # Gen3 dataRef
835  datasetType = dataRef.ref.datasetType.name
836 
837  if datasetType == 'src':
838  schema = dataRef.get(datasetType='src_schema').schema
839  try:
840  fluxFieldName = schema[fluxField].asField().getName()
841  except LookupError:
842  raise RuntimeError("Could not find %s or associated slot in schema." % (fluxField))
843  # This may also raise a RuntimeError
844  apertureRadius = computeApertureRadiusFromName(fluxFieldName)
845  else:
846  # This is a sourceTable_visit
847  apertureRadius = computeApertureRadiusFromName(fluxField)
848 
849  return apertureRadius
850 
851 
853  """
854  Compute the radius associated with a CircularApertureFlux or ApFlux field.
855 
856  Parameters
857  ----------
858  fluxField : `str`
859  CircularApertureFlux or ApFlux
860 
861  Returns
862  -------
863  apertureRadius : `float`
864  Radius of the aperture field, in pixels.
865 
866  Raises
867  ------
868  RuntimeError: Raised if flux field is not a CircularApertureFlux
869  or ApFlux.
870  """
871  # TODO: Move this method to more general stack method in DM-25775
872  m = re.search(r'(CircularApertureFlux|ApFlux)_(\d+)_(\d+)_', fluxField)
873 
874  if m is None:
875  raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
876 
877  apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
878 
879  return apertureRadius
880 
881 
882 def extractReferenceMags(refStars, bands, filterMap):
883  """
884  Extract reference magnitudes from refStars for given bands and
885  associated filterMap.
886 
887  Parameters
888  ----------
889  refStars : `lsst.afw.table.BaseCatalog`
890  FGCM reference star catalog
891  bands : `list`
892  List of bands for calibration
893  filterMap: `dict`
894  FGCM mapping of filter to band
895 
896  Returns
897  -------
898  refMag : `np.ndarray`
899  nstar x nband array of reference magnitudes
900  refMagErr : `np.ndarray`
901  nstar x nband array of reference magnitude errors
902  """
903  # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent
904  # against index errors and allow more flexibility in fitting after
905  # the build stars step.
906 
907  md = refStars.getMetadata()
908  if 'FILTERNAMES' in md:
909  filternames = md.getArray('FILTERNAMES')
910 
911  # The reference catalog that fgcm wants has one entry per band
912  # in the config file
913  refMag = np.zeros((len(refStars), len(bands)),
914  dtype=refStars['refMag'].dtype) + 99.0
915  refMagErr = np.zeros_like(refMag) + 99.0
916  for i, filtername in enumerate(filternames):
917  # We are allowed to run the fit configured so that we do not
918  # use every column in the reference catalog.
919  try:
920  band = filterMap[filtername]
921  except KeyError:
922  continue
923  try:
924  ind = bands.index(band)
925  except ValueError:
926  continue
927 
928  refMag[:, ind] = refStars['refMag'][:, i]
929  refMagErr[:, ind] = refStars['refMagErr'][:, i]
930 
931  else:
932  # Continue to use old catalogs as before.
933  refMag = refStars['refMag'][:, :]
934  refMagErr = refStars['refMagErr'][:, :]
935 
936  return refMag, refMagErr
937 
938 
939 def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
940  instrument = Instrument.fromName(quantumDataId["instrument"], registry)
941  unboundedCollection = instrument.makeUnboundedCalibrationRunName()
942 
943  return registry.queryDatasets(datasetType,
944  dataId=quantumDataId,
945  collections=[unboundedCollection])
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:882
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition: utilities.py:939
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:807
def makeStdSchema(nBands)
Definition: utilities.py:737
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:704
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
Definition: utilities.py:50
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:217
def computeReferencePixelScale(camera)
Definition: utilities.py:466
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:625
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:769
def computeApertureRadiusFromName(fluxField)
Definition: utilities.py:852
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:494
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:539
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:398
def translateVisitCatalog(visitCat)
Definition: utilities.py:347