lsst.fgcmcal  20.0.0-4-ge48a6ca+10
utilities.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Utility functions for fgcmcal.
22 
23 This file contains utility functions that are used by more than one task,
24 and do not need to be part of a task.
25 """
26 
27 import numpy as np
28 import re
29 
30 from lsst.daf.base import PropertyList
31 import lsst.afw.cameraGeom as afwCameraGeom
32 import lsst.afw.table as afwTable
33 import lsst.afw.image as afwImage
34 import lsst.afw.math as afwMath
35 import lsst.geom as geom
36 from lsst.obs.base import createInitialSkyWcs
37 
38 import fgcm
39 
40 
41 def makeConfigDict(config, log, camera, maxIter,
42  resetFitParameters, outputZeropoints, tract=None):
43  """
44  Make the FGCM fit cycle configuration dict
45 
46  Parameters
47  ----------
48  config: `lsst.fgcmcal.FgcmFitCycleConfig`
49  Configuration object
50  log: `lsst.log.Log`
51  LSST log object
52  camera: `lsst.afw.cameraGeom.Camera`
53  Camera from the butler
54  maxIter: `int`
55  Maximum number of iterations
56  resetFitParameters: `bool`
57  Reset fit parameters before fitting?
58  outputZeropoints: `bool`
59  Compute zeropoints for output?
60  tract: `int`, optional
61  Tract number for extending the output file name for debugging.
62  Default is None.
63 
64  Returns
65  -------
66  configDict: `dict`
67  Configuration dictionary for fgcm
68  """
69  # Extract the bands that are _not_ being fit for fgcm configuration
70  notFitBands = [b for b in config.bands if b not in config.fitBands]
71 
72  # process the starColorCuts
73  starColorCutList = []
74  for ccut in config.starColorCuts:
75  parts = ccut.split(',')
76  starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
77 
78  # TODO: Having direct access to the mirror area from the camera would be
79  # useful. See DM-16489.
80  # Mirror area in cm**2
81  mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
82 
83  # Get approximate average camera gain:
84  gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
85  cameraGain = float(np.median(gains))
86 
87  if tract is None:
88  outfileBase = config.outfileBase
89  else:
90  outfileBase = '%s-%06d' % (config.outfileBase, tract)
91 
92  # create a configuration dictionary for fgcmFitCycle
93  configDict = {'outfileBase': outfileBase,
94  'logger': log,
95  'exposureFile': None,
96  'obsFile': None,
97  'indexFile': None,
98  'lutFile': None,
99  'mirrorArea': mirrorArea,
100  'cameraGain': cameraGain,
101  'ccdStartIndex': camera[0].getId(),
102  'expField': 'VISIT',
103  'ccdField': 'CCD',
104  'seeingField': 'DELTA_APER',
105  'fwhmField': 'PSFSIGMA',
106  'skyBrightnessField': 'SKYBACKGROUND',
107  'deepFlag': 'DEEPFLAG', # unused
108  'bands': list(config.bands),
109  'fitBands': list(config.fitBands),
110  'notFitBands': notFitBands,
111  'requiredBands': list(config.requiredBands),
112  'filterToBand': dict(config.filterMap),
113  'logLevel': 'INFO', # FIXME
114  'nCore': config.nCore,
115  'nStarPerRun': config.nStarPerRun,
116  'nExpPerRun': config.nExpPerRun,
117  'reserveFraction': config.reserveFraction,
118  'freezeStdAtmosphere': config.freezeStdAtmosphere,
119  'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
120  'superStarSubCCDDict': dict(config.superStarSubCcdDict),
121  'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
122  'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
123  'superStarSigmaClip': config.superStarSigmaClip,
124  'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
125  'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
126  'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
127  'cycleNumber': config.cycleNumber,
128  'maxIter': maxIter,
129  'UTBoundary': config.utBoundary,
130  'washMJDs': config.washMjds,
131  'epochMJDs': config.epochMjds,
132  'coatingMJDs': config.coatingMjds,
133  'minObsPerBand': config.minObsPerBand,
134  'latitude': config.latitude,
135  'brightObsGrayMax': config.brightObsGrayMax,
136  'minStarPerCCD': config.minStarPerCcd,
137  'minCCDPerExp': config.minCcdPerExp,
138  'maxCCDGrayErr': config.maxCcdGrayErr,
139  'minStarPerExp': config.minStarPerExp,
140  'minExpPerNight': config.minExpPerNight,
141  'expGrayInitialCut': config.expGrayInitialCut,
142  'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
143  'expGrayHighCutDict': dict(config.expGrayHighCutDict),
144  'expGrayRecoverCut': config.expGrayRecoverCut,
145  'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
146  'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
147  'refStarSnMin': config.refStarSnMin,
148  'refStarOutlierNSig': config.refStarOutlierNSig,
149  'applyRefStarColorCuts': config.applyRefStarColorCuts,
150  'illegalValue': -9999.0, # internally used by fgcm.
151  'starColorCuts': starColorCutList,
152  'aperCorrFitNBins': config.aperCorrFitNBins,
153  'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
154  'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
155  'sedTermDict': config.sedterms.toDict()['data'],
156  'colorSplitBands': list(config.colorSplitBands),
157  'sigFgcmMaxErr': config.sigFgcmMaxErr,
158  'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
159  'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
160  'approxThroughputDict': dict(config.approxThroughputDict),
161  'sigmaCalRange': list(config.sigmaCalRange),
162  'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
163  'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
164  'sigma0Phot': config.sigma0Phot,
165  'mapLongitudeRef': config.mapLongitudeRef,
166  'mapNSide': config.mapNSide,
167  'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
168  'varMinBand': 2,
169  'useRetrievedPwv': False,
170  'useNightlyRetrievedPwv': False,
171  'pwvRetrievalSmoothBlock': 25,
172  'useQuadraticPwv': config.useQuadraticPwv,
173  'useRetrievedTauInit': False,
174  'tauRetrievalMinCCDPerNight': 500,
175  'modelMagErrors': config.modelMagErrors,
176  'instrumentParsPerBand': config.instrumentParsPerBand,
177  'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
178  'fitMirrorChromaticity': config.fitMirrorChromaticity,
179  'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
180  'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
181  'autoHighCutNSig': config.autoHighCutNSig,
182  'printOnly': False,
183  'quietMode': config.quietMode,
184  'outputStars': False,
185  'clobber': True,
186  'useSedLUT': False,
187  'resetParameters': resetFitParameters,
188  'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
189  'outputZeropoints': outputZeropoints}
190 
191  return configDict
192 
193 
194 def translateFgcmLut(lutCat, filterMap):
195  """
196  Translate the FGCM look-up-table into an fgcm-compatible object
197 
198  Parameters
199  ----------
200  lutCat: `lsst.afw.table.BaseCatalog`
201  Catalog describing the FGCM look-up table
202  filterMap: `dict`
203  Filter to band mapping
204 
205  Returns
206  -------
207  fgcmLut: `lsst.fgcm.FgcmLut`
208  Lookup table for FGCM
209  lutIndexVals: `numpy.ndarray`
210  Numpy array with LUT index information for FGCM
211  lutStd: `numpy.ndarray`
212  Numpy array with LUT standard throughput values for FGCM
213 
214  Notes
215  -----
216  After running this code, it is wise to `del lutCat` to clear the memory.
217  """
218 
219  # first we need the lutIndexVals
220  # dtype is set for py2/py3/fits/fgcm compatibility
221  lutFilterNames = np.array(lutCat[0]['filterNames'].split(','), dtype='a')
222  lutStdFilterNames = np.array(lutCat[0]['stdFilterNames'].split(','), dtype='a')
223 
224  # Note that any discrepancies between config values will raise relevant
225  # exceptions in the FGCM code.
226 
227  lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
228  lutFilterNames.size),
229  ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
230  lutStdFilterNames.size),
231  ('PMB', 'f8', lutCat[0]['pmb'].size),
232  ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
233  ('PMBELEVATION', 'f8'),
234  ('LAMBDANORM', 'f8'),
235  ('PWV', 'f8', lutCat[0]['pwv'].size),
236  ('O3', 'f8', lutCat[0]['o3'].size),
237  ('TAU', 'f8', lutCat[0]['tau'].size),
238  ('ALPHA', 'f8', lutCat[0]['alpha'].size),
239  ('ZENITH', 'f8', lutCat[0]['zenith'].size),
240  ('NCCD', 'i4')])
241 
242  lutIndexVals['FILTERNAMES'][:] = lutFilterNames
243  lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
244  lutIndexVals['PMB'][:] = lutCat[0]['pmb']
245  lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
246  lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
247  lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
248  lutIndexVals['PWV'][:] = lutCat[0]['pwv']
249  lutIndexVals['O3'][:] = lutCat[0]['o3']
250  lutIndexVals['TAU'][:] = lutCat[0]['tau']
251  lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
252  lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
253  lutIndexVals['NCCD'] = lutCat[0]['nCcd']
254 
255  # now we need the Standard Values
256  lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
257  ('PWVSTD', 'f8'),
258  ('O3STD', 'f8'),
259  ('TAUSTD', 'f8'),
260  ('ALPHASTD', 'f8'),
261  ('ZENITHSTD', 'f8'),
262  ('LAMBDARANGE', 'f8', 2),
263  ('LAMBDASTEP', 'f8'),
264  ('LAMBDASTD', 'f8', lutFilterNames.size),
265  ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
266  ('I0STD', 'f8', lutFilterNames.size),
267  ('I1STD', 'f8', lutFilterNames.size),
268  ('I10STD', 'f8', lutFilterNames.size),
269  ('I2STD', 'f8', lutFilterNames.size),
270  ('LAMBDAB', 'f8', lutFilterNames.size),
271  ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
272  ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
273  lutStd['PMBSTD'] = lutCat[0]['pmbStd']
274  lutStd['PWVSTD'] = lutCat[0]['pwvStd']
275  lutStd['O3STD'] = lutCat[0]['o3Std']
276  lutStd['TAUSTD'] = lutCat[0]['tauStd']
277  lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
278  lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
279  lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
280  lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
281  lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
282  lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
283  lutStd['I0STD'][:] = lutCat[0]['i0Std']
284  lutStd['I1STD'][:] = lutCat[0]['i1Std']
285  lutStd['I10STD'][:] = lutCat[0]['i10Std']
286  lutStd['I2STD'][:] = lutCat[0]['i2Std']
287  lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
288  lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
289  lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
290 
291  lutTypes = [row['luttype'] for row in lutCat]
292 
293  # And the flattened look-up-table
294  lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
295  ('I1', 'f4')])
296 
297  lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
298  lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
299 
300  lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
301  ('D_O3', 'f4'),
302  ('D_LNTAU', 'f4'),
303  ('D_ALPHA', 'f4'),
304  ('D_SECZENITH', 'f4'),
305  ('D_LNPWV_I1', 'f4'),
306  ('D_O3_I1', 'f4'),
307  ('D_LNTAU_I1', 'f4'),
308  ('D_ALPHA_I1', 'f4'),
309  ('D_SECZENITH_I1', 'f4')])
310 
311  for name in lutDerivFlat.dtype.names:
312  lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
313 
314  # The fgcm.FgcmLUT() class copies all the LUT information into special
315  # shared memory objects that will not blow up the memory usage when used
316  # with python multiprocessing. Once all the numbers are copied, the
317  # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
318  # will fall out of scope and can be cleaned up by the garbage collector.
319  fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
320  filterToBand=filterMap)
321 
322  return fgcmLut, lutIndexVals, lutStd
323 
324 
325 def translateVisitCatalog(visitCat):
326  """
327  Translate the FGCM visit catalog to an fgcm-compatible object
328 
329  Parameters
330  ----------
331  visitCat: `lsst.afw.table.BaseCatalog`
332  FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
333 
334  Returns
335  -------
336  fgcmExpInfo: `numpy.ndarray`
337  Numpy array for visit information for FGCM
338 
339  Notes
340  -----
341  After running this code, it is wise to `del visitCat` to clear the memory.
342  """
343 
344  fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
345  ('MJD', 'f8'),
346  ('EXPTIME', 'f8'),
347  ('PSFSIGMA', 'f8'),
348  ('DELTA_APER', 'f8'),
349  ('SKYBACKGROUND', 'f8'),
350  ('DEEPFLAG', 'i2'),
351  ('TELHA', 'f8'),
352  ('TELRA', 'f8'),
353  ('TELDEC', 'f8'),
354  ('TELROT', 'f8'),
355  ('PMB', 'f8'),
356  ('FILTERNAME', 'a10')])
357  fgcmExpInfo['VISIT'][:] = visitCat['visit']
358  fgcmExpInfo['MJD'][:] = visitCat['mjd']
359  fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
360  fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
361  fgcmExpInfo['TELHA'][:] = visitCat['telha']
362  fgcmExpInfo['TELRA'][:] = visitCat['telra']
363  fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
364  fgcmExpInfo['TELROT'][:] = visitCat['telrot']
365  fgcmExpInfo['PMB'][:] = visitCat['pmb']
366  fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
367  fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
368  fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
369  # Note that we have to go through asAstropy() to get a string
370  # array out of an afwTable. This is faster than a row-by-row loop.
371  fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['filtername']
372 
373  return fgcmExpInfo
374 
375 
376 def computeCcdOffsets(camera, defaultOrientation):
377  """
378  Compute the CCD offsets in ra/dec and x/y space
379 
380  Parameters
381  ----------
382  camera: `lsst.afw.cameraGeom.Camera`
383  defaultOrientation: `float`
384  Default camera orientation (degrees)
385 
386  Returns
387  -------
388  ccdOffsets: `numpy.ndarray`
389  Numpy array with ccd offset information for input to FGCM.
390  Angular units are degrees, and x/y units are pixels.
391  """
392  # TODO: DM-21215 will fully generalize to arbitrary camera orientations
393 
394  # and we need to know the ccd offsets from the camera geometry
395  ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'),
396  ('DELTA_RA', 'f8'),
397  ('DELTA_DEC', 'f8'),
398  ('RA_SIZE', 'f8'),
399  ('DEC_SIZE', 'f8'),
400  ('X_SIZE', 'i4'),
401  ('Y_SIZE', 'i4')])
402 
403  # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
404  # since we are looking for relative positions
405  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
406 
407  # TODO: DM-17597 will update testdata_jointcal so that the test data
408  # does not have nan as the boresight angle for HSC data. For the
409  # time being, there is this ungainly hack.
410  if camera.getName() == 'HSC' and np.isnan(defaultOrientation):
411  orientation = 270*geom.degrees
412  else:
413  orientation = defaultOrientation*geom.degrees
414  flipX = False
415 
416  # Create a temporary visitInfo for input to createInitialSkyWcs
417  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
418  boresightRotAngle=orientation,
419  rotType=afwImage.visitInfo.RotType.SKY)
420 
421  for i, detector in enumerate(camera):
422  ccdOffsets['CCDNUM'][i] = detector.getId()
423 
424  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
425 
426  detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS))
427  ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees()
428  ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees()
429 
430  bbox = detector.getBBox()
431 
432  detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin()))
433  detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax()))
434 
435  ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees())
436  ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees())
437 
438  ccdOffsets['X_SIZE'][i] = bbox.getMaxX()
439  ccdOffsets['Y_SIZE'][i] = bbox.getMaxY()
440 
441  return ccdOffsets
442 
443 
445  """
446  Compute the median pixel scale in the camera
447 
448  Returns
449  -------
450  pixelScale: `float`
451  Average pixel scale (arcsecond) over the camera
452  """
453 
454  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
455  orientation = 0.0*geom.degrees
456  flipX = False
457 
458  # Create a temporary visitInfo for input to createInitialSkyWcs
459  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
460  boresightRotAngle=orientation,
461  rotType=afwImage.visitInfo.RotType.SKY)
462 
463  pixelScales = np.zeros(len(camera))
464  for i, detector in enumerate(camera):
465  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
466  pixelScales[i] = wcs.getPixelScale().asArcseconds()
467 
468  ok, = np.where(pixelScales > 0.0)
469  return np.median(pixelScales[ok])
470 
471 
473  """
474  Compute the approximate pixel area bounded fields from the camera
475  geometry.
476 
477  Parameters
478  ----------
479  camera: `lsst.afw.cameraGeom.Camera`
480 
481  Returns
482  -------
483  approxPixelAreaFields: `dict`
484  Dictionary of approximate area fields, keyed with detector ID
485  """
486 
487  areaScaling = 1. / computeReferencePixelScale(camera)**2.
488 
489  # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
490  # since we are looking for relative scales
491  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
492 
493  flipX = False
494  # Create a temporary visitInfo for input to createInitialSkyWcs
495  # The orientation does not matter for the area computation
496  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
497  boresightRotAngle=0.0*geom.degrees,
498  rotType=afwImage.visitInfo.RotType.SKY)
499 
500  approxPixelAreaFields = {}
501 
502  for i, detector in enumerate(camera):
503  key = detector.getId()
504 
505  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
506  bbox = detector.getBBox()
507 
508  areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
509  unit=geom.arcseconds, scaling=areaScaling)
510  approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
511 
512  approxPixelAreaFields[key] = approxAreaField
513 
514  return approxPixelAreaFields
515 
516 
517 def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
518  """
519  Make the zeropoint schema
520 
521  Parameters
522  ----------
523  superStarChebyshevSize: `int`
524  Length of the superstar chebyshev array
525  zptChebyshevSize: `int`
526  Length of the zeropoint chebyshev array
527 
528  Returns
529  -------
530  zptSchema: `lsst.afw.table.schema`
531  """
532 
533  zptSchema = afwTable.Schema()
534 
535  zptSchema.addField('visit', type=np.int32, doc='Visit number')
536  zptSchema.addField('ccd', type=np.int32, doc='CCD number')
537  zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
538  '1: Photometric, used in fit; '
539  '2: Photometric, not used in fit; '
540  '4: Non-photometric, on partly photometric night; '
541  '8: Non-photometric, on non-photometric night; '
542  '16: No zeropoint could be determined; '
543  '32: Too few stars for reliable gray computation'))
544  zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
545  zptSchema.addField('fgcmZptErr', type=np.float64,
546  doc='Error on zeropoint, estimated from repeatability + number of obs')
547  zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
548  doc='maximum x/maximum y to scale to apply chebyshev parameters')
549  zptSchema.addField('fgcmfZptCheb', type='ArrayD',
550  size=zptChebyshevSize,
551  doc='Chebyshev parameters (flattened) for zeropoint')
552  zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
553  size=superStarChebyshevSize,
554  doc='Chebyshev parameters (flattened) for superStarFlat')
555  zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
556  zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
557  zptSchema.addField('fgcmR0', type=np.float64,
558  doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
559  zptSchema.addField('fgcmR10', type=np.float64,
560  doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
561  zptSchema.addField('fgcmGry', type=np.float64,
562  doc='Estimated gray extinction relative to atmospheric solution; '
563  'only for flag <= 4')
564  zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
565  zptSchema.addField('fgcmTilings', type=np.float64,
566  doc='Number of photometric tilings used for solution for ccd')
567  zptSchema.addField('fgcmFpGry', type=np.float64,
568  doc='Average gray extinction over the full focal plane '
569  '(same for all ccds in a visit)')
570  zptSchema.addField('fgcmFpGryBlue', type=np.float64,
571  doc='Average gray extinction over the full focal plane '
572  'for 25% bluest stars')
573  zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
574  doc='Error on Average gray extinction over the full focal plane '
575  'for 25% bluest stars')
576  zptSchema.addField('fgcmFpGryRed', type=np.float64,
577  doc='Average gray extinction over the full focal plane '
578  'for 25% reddest stars')
579  zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
580  doc='Error on Average gray extinction over the full focal plane '
581  'for 25% reddest stars')
582  zptSchema.addField('fgcmFpVar', type=np.float64,
583  doc='Variance of gray extinction over the full focal plane '
584  '(same for all ccds in a visit)')
585  zptSchema.addField('fgcmDust', type=np.float64,
586  doc='Gray dust extinction from the primary/corrector'
587  'at the time of the exposure')
588  zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
589  zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
590  zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
591  zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
592 
593  return zptSchema
594 
595 
596 def makeZptCat(zptSchema, zpStruct):
597  """
598  Make the zeropoint catalog for persistence
599 
600  Parameters
601  ----------
602  zptSchema: `lsst.afw.table.Schema`
603  Zeropoint catalog schema
604  zpStruct: `numpy.ndarray`
605  Zeropoint structure from fgcm
606 
607  Returns
608  -------
609  zptCat: `afwTable.BaseCatalog`
610  Zeropoint catalog for persistence
611  """
612 
613  zptCat = afwTable.BaseCatalog(zptSchema)
614  zptCat.reserve(zpStruct.size)
615 
616  for filterName in zpStruct['FILTERNAME']:
617  rec = zptCat.addNew()
618  rec['filtername'] = filterName.decode('utf-8')
619 
620  zptCat['visit'][:] = zpStruct['VISIT']
621  zptCat['ccd'][:] = zpStruct['CCD']
622  zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
623  zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
624  zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
625  zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
626  zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
627  zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
628  zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
629  zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
630  zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
631  zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
632  zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
633  zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
634  zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
635  zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
636  zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
637  zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
638  zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
639  zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
640  zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
641  zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
642  zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
643  zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
644  zptCat['exptime'][:] = zpStruct['EXPTIME']
645 
646  return zptCat
647 
648 
650  """
651  Make the atmosphere schema
652 
653  Returns
654  -------
655  atmSchema: `lsst.afw.table.Schema`
656  """
657 
658  atmSchema = afwTable.Schema()
659 
660  atmSchema.addField('visit', type=np.int32, doc='Visit number')
661  atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
662  atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
663  atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
664  atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
665  atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
666  atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
667  atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
668  atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
669 
670  return atmSchema
671 
672 
673 def makeAtmCat(atmSchema, atmStruct):
674  """
675  Make the atmosphere catalog for persistence
676 
677  Parameters
678  ----------
679  atmSchema: `lsst.afw.table.Schema`
680  Atmosphere catalog schema
681  atmStruct: `numpy.ndarray`
682  Atmosphere structure from fgcm
683 
684  Returns
685  -------
686  atmCat: `lsst.afw.table.BaseCatalog`
687  Atmosphere catalog for persistence
688  """
689 
690  atmCat = afwTable.BaseCatalog(atmSchema)
691  atmCat.resize(atmStruct.size)
692 
693  atmCat['visit'][:] = atmStruct['VISIT']
694  atmCat['pmb'][:] = atmStruct['PMB']
695  atmCat['pwv'][:] = atmStruct['PWV']
696  atmCat['tau'][:] = atmStruct['TAU']
697  atmCat['alpha'][:] = atmStruct['ALPHA']
698  atmCat['o3'][:] = atmStruct['O3']
699  atmCat['secZenith'][:] = atmStruct['SECZENITH']
700  atmCat['cTrans'][:] = atmStruct['CTRANS']
701  atmCat['lamStd'][:] = atmStruct['LAMSTD']
702 
703  return atmCat
704 
705 
706 def makeStdSchema(nBands):
707  """
708  Make the standard star schema
709 
710  Parameters
711  ----------
712  nBands: `int`
713  Number of bands in standard star catalog
714 
715  Returns
716  -------
717  stdSchema: `lsst.afw.table.Schema`
718  """
719 
720  stdSchema = afwTable.SimpleTable.makeMinimalSchema()
721  stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
722  size=nBands)
723  stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
724  size=nBands)
725  stdSchema.addField('mag_std_noabs', type='ArrayF',
726  doc='Standard magnitude (no absolute calibration)',
727  size=nBands)
728  stdSchema.addField('magErr_std', type='ArrayF',
729  doc='Standard magnitude error',
730  size=nBands)
731  stdSchema.addField('npsfcand', type='ArrayI',
732  doc='Number of observations flagged as psf candidates',
733  size=nBands)
734 
735  return stdSchema
736 
737 
738 def makeStdCat(stdSchema, stdStruct, goodBands):
739  """
740  Make the standard star catalog for persistence
741 
742  Parameters
743  ----------
744  stdSchema: `lsst.afw.table.Schema`
745  Standard star catalog schema
746  stdStruct: `numpy.ndarray`
747  Standard star structure in FGCM format
748  goodBands: `list`
749  List of good band names used in stdStruct
750 
751  Returns
752  -------
753  stdCat: `lsst.afw.table.BaseCatalog`
754  Standard star catalog for persistence
755  """
756 
757  stdCat = afwTable.SimpleCatalog(stdSchema)
758  stdCat.resize(stdStruct.size)
759 
760  stdCat['id'][:] = stdStruct['FGCM_ID']
761  stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
762  stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
763  stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
764  stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
765  stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
766  stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
767  stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
768 
769  md = PropertyList()
770  md.set("BANDS", list(goodBands))
771  stdCat.setMetadata(md)
772 
773  return stdCat
774 
775 
776 def computeApertureRadiusFromDataRef(dataRef, fluxField):
777  """
778  Compute the radius associated with a CircularApertureFlux field or
779  associated slot.
780 
781  Parameters
782  ----------
783  dataRef : `lsst.daf.persistence.ButlerDataRef`
784  fluxField : `str`
785  CircularApertureFlux or associated slot.
786 
787  Returns
788  -------
789  apertureRadius : `float`
790  Radius of the aperture field, in pixels.
791 
792  Raises
793  ------
794  RuntimeError: Raised if flux field is not a CircularApertureFlux, ApFlux,
795  or associated slot.
796  """
797  # TODO: Move this method to more general stack method in DM-25775
798  datasetType = dataRef.butlerSubset.datasetType
799 
800  if datasetType == 'src':
801  schema = dataRef.get(datasetType='src_schema').schema
802  try:
803  fluxFieldName = schema[fluxField].asField().getName()
804  except LookupError:
805  raise RuntimeError("Could not find %s or associated slot in schema." % (fluxField))
806  # This may also raise a RuntimeError
807  apertureRadius = computeApertureRadiusFromName(fluxFieldName)
808  else:
809  # This is a sourceTable_visit
810  apertureRadius = computeApertureRadiusFromName(fluxField)
811 
812  return apertureRadius
813 
814 
816  """
817  Compute the radius associated with a CircularApertureFlux or ApFlux field.
818 
819  Parameters
820  ----------
821  fluxField : `str`
822  CircularApertureFlux or ApFlux
823 
824  Returns
825  -------
826  apertureRadius : `float`
827  Radius of the aperture field, in pixels.
828 
829  Raises
830  ------
831  RuntimeError: Raised if flux field is not a CircularApertureFlux
832  or ApFlux.
833  """
834  # TODO: Move this method to more general stack method in DM-25775
835  m = re.search(r'(CircularApertureFlux|ApFlux)_(\d+)_(\d+)_', fluxField)
836 
837  if m is None:
838  raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
839 
840  apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
841 
842  return apertureRadius
843 
844 
845 def extractReferenceMags(refStars, bands, filterMap):
846  """
847  Extract reference magnitudes from refStars for given bands and
848  associated filterMap.
849 
850  Parameters
851  ----------
852  refStars : `lsst.afw.table.BaseCatalog`
853  FGCM reference star catalog
854  bands : `list`
855  List of bands for calibration
856  filterMap: `dict`
857  FGCM mapping of filter to band
858 
859  Returns
860  -------
861  refMag : `np.ndarray`
862  nstar x nband array of reference magnitudes
863  refMagErr : `np.ndarray`
864  nstar x nband array of reference magnitude errors
865  """
866  # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent
867  # against index errors and allow more flexibility in fitting after
868  # the build stars step.
869 
870  md = refStars.getMetadata()
871  if 'FILTERNAMES' in md:
872  filternames = md.getArray('FILTERNAMES')
873 
874  # The reference catalog that fgcm wants has one entry per band
875  # in the config file
876  refMag = np.zeros((len(refStars), len(bands)),
877  dtype=refStars['refMag'].dtype) + 99.0
878  refMagErr = np.zeros_like(refMag) + 99.0
879  for i, filtername in enumerate(filternames):
880  # We are allowed to run the fit configured so that we do not
881  # use every column in the reference catalog.
882  try:
883  band = filterMap[filtername]
884  except KeyError:
885  continue
886  try:
887  ind = bands.index(band)
888  except ValueError:
889  continue
890 
891  refMag[:, ind] = refStars['refMag'][:, i]
892  refMagErr[:, ind] = refStars['refMagErr'][:, i]
893 
894  else:
895  # Continue to use old catalogs as before.
896  refMag = refStars['refMag'][:, :]
897  refMagErr = refStars['refMagErr'][:, :]
898 
899  return refMag, refMagErr
lsst.fgcmcal.utilities.computeCcdOffsets
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:376
lsst.fgcmcal.utilities.makeZptCat
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:596
lsst.fgcmcal.utilities.computeReferencePixelScale
def computeReferencePixelScale(camera)
Definition: utilities.py:444
lsst.fgcmcal.utilities.computeApertureRadiusFromName
def computeApertureRadiusFromName(fluxField)
Definition: utilities.py:815
lsst.fgcmcal.utilities.makeStdCat
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:738
lsst.fgcmcal.utilities.makeConfigDict
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
Definition: utilities.py:41
lsst.fgcmcal.utilities.computeApproxPixelAreaFields
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:472
lsst.fgcmcal.utilities.translateFgcmLut
def translateFgcmLut(lutCat, filterMap)
Definition: utilities.py:194
lsst.fgcmcal.utilities.translateVisitCatalog
def translateVisitCatalog(visitCat)
Definition: utilities.py:325
lsst.fgcmcal.utilities.makeStdSchema
def makeStdSchema(nBands)
Definition: utilities.py:706
lsst.fgcmcal.utilities.computeApertureRadiusFromDataRef
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:776
lsst.fgcmcal.utilities.makeAtmSchema
def makeAtmSchema()
Definition: utilities.py:649
lsst.fgcmcal.utilities.makeZptSchema
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:517
lsst.fgcmcal.utilities.makeAtmCat
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:673
lsst.fgcmcal.utilities.extractReferenceMags
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:845