lsst.fgcmcal  20.0.0-5-gfcebe35+da3b173188
utilities.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Utility functions for fgcmcal.
22 
23 This file contains utility functions that are used by more than one task,
24 and do not need to be part of a task.
25 """
26 
27 import numpy as np
28 import re
29 
30 from lsst.daf.base import PropertyList
31 import lsst.afw.cameraGeom as afwCameraGeom
32 import lsst.afw.table as afwTable
33 import lsst.afw.image as afwImage
34 import lsst.afw.math as afwMath
35 import lsst.geom as geom
36 from lsst.obs.base import createInitialSkyWcs
37 
38 import fgcm
39 
40 
41 def makeConfigDict(config, log, camera, maxIter,
42  resetFitParameters, outputZeropoints, tract=None):
43  """
44  Make the FGCM fit cycle configuration dict
45 
46  Parameters
47  ----------
48  config: `lsst.fgcmcal.FgcmFitCycleConfig`
49  Configuration object
50  log: `lsst.log.Log`
51  LSST log object
52  camera: `lsst.afw.cameraGeom.Camera`
53  Camera from the butler
54  maxIter: `int`
55  Maximum number of iterations
56  resetFitParameters: `bool`
57  Reset fit parameters before fitting?
58  outputZeropoints: `bool`
59  Compute zeropoints for output?
60  tract: `int`, optional
61  Tract number for extending the output file name for debugging.
62  Default is None.
63 
64  Returns
65  -------
66  configDict: `dict`
67  Configuration dictionary for fgcm
68  """
69  # Extract the bands that are _not_ being fit for fgcm configuration
70  notFitBands = [b for b in config.bands if b not in config.fitBands]
71 
72  # process the starColorCuts
73  starColorCutList = []
74  for ccut in config.starColorCuts:
75  parts = ccut.split(',')
76  starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])])
77 
78  # TODO: Having direct access to the mirror area from the camera would be
79  # useful. See DM-16489.
80  # Mirror area in cm**2
81  mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2.
82 
83  # Get approximate average camera gain:
84  gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()]
85  cameraGain = float(np.median(gains))
86 
87  if tract is None:
88  outfileBase = config.outfileBase
89  else:
90  outfileBase = '%s-%06d' % (config.outfileBase, tract)
91 
92  # create a configuration dictionary for fgcmFitCycle
93  configDict = {'outfileBase': outfileBase,
94  'logger': log,
95  'exposureFile': None,
96  'obsFile': None,
97  'indexFile': None,
98  'lutFile': None,
99  'mirrorArea': mirrorArea,
100  'cameraGain': cameraGain,
101  'ccdStartIndex': camera[0].getId(),
102  'expField': 'VISIT',
103  'ccdField': 'CCD',
104  'seeingField': 'DELTA_APER',
105  'fwhmField': 'PSFSIGMA',
106  'skyBrightnessField': 'SKYBACKGROUND',
107  'deepFlag': 'DEEPFLAG', # unused
108  'bands': list(config.bands),
109  'fitBands': list(config.fitBands),
110  'notFitBands': notFitBands,
111  'requiredBands': list(config.requiredBands),
112  'filterToBand': dict(config.filterMap),
113  'logLevel': 'INFO', # FIXME
114  'nCore': config.nCore,
115  'nStarPerRun': config.nStarPerRun,
116  'nExpPerRun': config.nExpPerRun,
117  'reserveFraction': config.reserveFraction,
118  'freezeStdAtmosphere': config.freezeStdAtmosphere,
119  'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
120  'superStarSubCCDDict': dict(config.superStarSubCcdDict),
121  'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
122  'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
123  'superStarSigmaClip': config.superStarSigmaClip,
124  'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
125  'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
126  'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
127  'cycleNumber': config.cycleNumber,
128  'maxIter': maxIter,
129  'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile,
130  'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd,
131  'UTBoundary': config.utBoundary,
132  'washMJDs': config.washMjds,
133  'epochMJDs': config.epochMjds,
134  'coatingMJDs': config.coatingMjds,
135  'minObsPerBand': config.minObsPerBand,
136  'latitude': config.latitude,
137  'brightObsGrayMax': config.brightObsGrayMax,
138  'minStarPerCCD': config.minStarPerCcd,
139  'minCCDPerExp': config.minCcdPerExp,
140  'maxCCDGrayErr': config.maxCcdGrayErr,
141  'minStarPerExp': config.minStarPerExp,
142  'minExpPerNight': config.minExpPerNight,
143  'expGrayInitialCut': config.expGrayInitialCut,
144  'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
145  'expGrayHighCutDict': dict(config.expGrayHighCutDict),
146  'expGrayRecoverCut': config.expGrayRecoverCut,
147  'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
148  'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
149  'refStarSnMin': config.refStarSnMin,
150  'refStarOutlierNSig': config.refStarOutlierNSig,
151  'applyRefStarColorCuts': config.applyRefStarColorCuts,
152  'illegalValue': -9999.0, # internally used by fgcm.
153  'starColorCuts': starColorCutList,
154  'aperCorrFitNBins': config.aperCorrFitNBins,
155  'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
156  'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
157  'sedTermDict': config.sedterms.toDict()['data'],
158  'colorSplitBands': list(config.colorSplitBands),
159  'sigFgcmMaxErr': config.sigFgcmMaxErr,
160  'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
161  'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
162  'approxThroughputDict': dict(config.approxThroughputDict),
163  'sigmaCalRange': list(config.sigmaCalRange),
164  'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
165  'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
166  'sigma0Phot': config.sigma0Phot,
167  'mapLongitudeRef': config.mapLongitudeRef,
168  'mapNSide': config.mapNSide,
169  'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet
170  'varMinBand': 2,
171  'useRetrievedPwv': False,
172  'useNightlyRetrievedPwv': False,
173  'pwvRetrievalSmoothBlock': 25,
174  'useQuadraticPwv': config.useQuadraticPwv,
175  'useRetrievedTauInit': False,
176  'tauRetrievalMinCCDPerNight': 500,
177  'modelMagErrors': config.modelMagErrors,
178  'instrumentParsPerBand': config.instrumentParsPerBand,
179  'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
180  'fitMirrorChromaticity': config.fitMirrorChromaticity,
181  'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
182  'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
183  'autoHighCutNSig': config.autoHighCutNSig,
184  'printOnly': False,
185  'quietMode': config.quietMode,
186  'outputStars': False,
187  'clobber': True,
188  'useSedLUT': False,
189  'resetParameters': resetFitParameters,
190  'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format
191  'outputZeropoints': outputZeropoints}
192 
193  return configDict
194 
195 
196 def translateFgcmLut(lutCat, filterMap):
197  """
198  Translate the FGCM look-up-table into an fgcm-compatible object
199 
200  Parameters
201  ----------
202  lutCat: `lsst.afw.table.BaseCatalog`
203  Catalog describing the FGCM look-up table
204  filterMap: `dict`
205  Filter to band mapping
206 
207  Returns
208  -------
209  fgcmLut: `lsst.fgcm.FgcmLut`
210  Lookup table for FGCM
211  lutIndexVals: `numpy.ndarray`
212  Numpy array with LUT index information for FGCM
213  lutStd: `numpy.ndarray`
214  Numpy array with LUT standard throughput values for FGCM
215 
216  Notes
217  -----
218  After running this code, it is wise to `del lutCat` to clear the memory.
219  """
220 
221  # first we need the lutIndexVals
222  # dtype is set for py2/py3/fits/fgcm compatibility
223  lutFilterNames = np.array(lutCat[0]['filterNames'].split(','), dtype='a')
224  lutStdFilterNames = np.array(lutCat[0]['stdFilterNames'].split(','), dtype='a')
225 
226  # Note that any discrepancies between config values will raise relevant
227  # exceptions in the FGCM code.
228 
229  lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str,
230  lutFilterNames.size),
231  ('STDFILTERNAMES', lutStdFilterNames.dtype.str,
232  lutStdFilterNames.size),
233  ('PMB', 'f8', lutCat[0]['pmb'].size),
234  ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size),
235  ('PMBELEVATION', 'f8'),
236  ('LAMBDANORM', 'f8'),
237  ('PWV', 'f8', lutCat[0]['pwv'].size),
238  ('O3', 'f8', lutCat[0]['o3'].size),
239  ('TAU', 'f8', lutCat[0]['tau'].size),
240  ('ALPHA', 'f8', lutCat[0]['alpha'].size),
241  ('ZENITH', 'f8', lutCat[0]['zenith'].size),
242  ('NCCD', 'i4')])
243 
244  lutIndexVals['FILTERNAMES'][:] = lutFilterNames
245  lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames
246  lutIndexVals['PMB'][:] = lutCat[0]['pmb']
247  lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor']
248  lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation']
249  lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm']
250  lutIndexVals['PWV'][:] = lutCat[0]['pwv']
251  lutIndexVals['O3'][:] = lutCat[0]['o3']
252  lutIndexVals['TAU'][:] = lutCat[0]['tau']
253  lutIndexVals['ALPHA'][:] = lutCat[0]['alpha']
254  lutIndexVals['ZENITH'][:] = lutCat[0]['zenith']
255  lutIndexVals['NCCD'] = lutCat[0]['nCcd']
256 
257  # now we need the Standard Values
258  lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'),
259  ('PWVSTD', 'f8'),
260  ('O3STD', 'f8'),
261  ('TAUSTD', 'f8'),
262  ('ALPHASTD', 'f8'),
263  ('ZENITHSTD', 'f8'),
264  ('LAMBDARANGE', 'f8', 2),
265  ('LAMBDASTEP', 'f8'),
266  ('LAMBDASTD', 'f8', lutFilterNames.size),
267  ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size),
268  ('I0STD', 'f8', lutFilterNames.size),
269  ('I1STD', 'f8', lutFilterNames.size),
270  ('I10STD', 'f8', lutFilterNames.size),
271  ('I2STD', 'f8', lutFilterNames.size),
272  ('LAMBDAB', 'f8', lutFilterNames.size),
273  ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size),
274  ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)])
275  lutStd['PMBSTD'] = lutCat[0]['pmbStd']
276  lutStd['PWVSTD'] = lutCat[0]['pwvStd']
277  lutStd['O3STD'] = lutCat[0]['o3Std']
278  lutStd['TAUSTD'] = lutCat[0]['tauStd']
279  lutStd['ALPHASTD'] = lutCat[0]['alphaStd']
280  lutStd['ZENITHSTD'] = lutCat[0]['zenithStd']
281  lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:]
282  lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep']
283  lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd']
284  lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter']
285  lutStd['I0STD'][:] = lutCat[0]['i0Std']
286  lutStd['I1STD'][:] = lutCat[0]['i1Std']
287  lutStd['I10STD'][:] = lutCat[0]['i10Std']
288  lutStd['I2STD'][:] = lutCat[0]['i2Std']
289  lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB']
290  lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:]
291  lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:]
292 
293  lutTypes = [row['luttype'] for row in lutCat]
294 
295  # And the flattened look-up-table
296  lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'),
297  ('I1', 'f4')])
298 
299  lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:]
300  lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:]
301 
302  lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'),
303  ('D_O3', 'f4'),
304  ('D_LNTAU', 'f4'),
305  ('D_ALPHA', 'f4'),
306  ('D_SECZENITH', 'f4'),
307  ('D_LNPWV_I1', 'f4'),
308  ('D_O3_I1', 'f4'),
309  ('D_LNTAU_I1', 'f4'),
310  ('D_ALPHA_I1', 'f4'),
311  ('D_SECZENITH_I1', 'f4')])
312 
313  for name in lutDerivFlat.dtype.names:
314  lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:]
315 
316  # The fgcm.FgcmLUT() class copies all the LUT information into special
317  # shared memory objects that will not blow up the memory usage when used
318  # with python multiprocessing. Once all the numbers are copied, the
319  # references to the temporary objects (lutCat, lutFlat, lutDerivFlat)
320  # will fall out of scope and can be cleaned up by the garbage collector.
321  fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd,
322  filterToBand=filterMap)
323 
324  return fgcmLut, lutIndexVals, lutStd
325 
326 
327 def translateVisitCatalog(visitCat):
328  """
329  Translate the FGCM visit catalog to an fgcm-compatible object
330 
331  Parameters
332  ----------
333  visitCat: `lsst.afw.table.BaseCatalog`
334  FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask`
335 
336  Returns
337  -------
338  fgcmExpInfo: `numpy.ndarray`
339  Numpy array for visit information for FGCM
340 
341  Notes
342  -----
343  After running this code, it is wise to `del visitCat` to clear the memory.
344  """
345 
346  fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'),
347  ('MJD', 'f8'),
348  ('EXPTIME', 'f8'),
349  ('PSFSIGMA', 'f8'),
350  ('DELTA_APER', 'f8'),
351  ('SKYBACKGROUND', 'f8'),
352  ('DEEPFLAG', 'i2'),
353  ('TELHA', 'f8'),
354  ('TELRA', 'f8'),
355  ('TELDEC', 'f8'),
356  ('TELROT', 'f8'),
357  ('PMB', 'f8'),
358  ('FILTERNAME', 'a10')])
359  fgcmExpInfo['VISIT'][:] = visitCat['visit']
360  fgcmExpInfo['MJD'][:] = visitCat['mjd']
361  fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']
362  fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag']
363  fgcmExpInfo['TELHA'][:] = visitCat['telha']
364  fgcmExpInfo['TELRA'][:] = visitCat['telra']
365  fgcmExpInfo['TELDEC'][:] = visitCat['teldec']
366  fgcmExpInfo['TELROT'][:] = visitCat['telrot']
367  fgcmExpInfo['PMB'][:] = visitCat['pmb']
368  fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma']
369  fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper']
370  fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground']
371  # Note that we have to go through asAstropy() to get a string
372  # array out of an afwTable. This is faster than a row-by-row loop.
373  fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['filtername']
374 
375  return fgcmExpInfo
376 
377 
378 def computeCcdOffsets(camera, defaultOrientation):
379  """
380  Compute the CCD offsets in ra/dec and x/y space
381 
382  Parameters
383  ----------
384  camera: `lsst.afw.cameraGeom.Camera`
385  defaultOrientation: `float`
386  Default camera orientation (degrees)
387 
388  Returns
389  -------
390  ccdOffsets: `numpy.ndarray`
391  Numpy array with ccd offset information for input to FGCM.
392  Angular units are degrees, and x/y units are pixels.
393  """
394  # TODO: DM-21215 will fully generalize to arbitrary camera orientations
395 
396  # and we need to know the ccd offsets from the camera geometry
397  ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'),
398  ('DELTA_RA', 'f8'),
399  ('DELTA_DEC', 'f8'),
400  ('RA_SIZE', 'f8'),
401  ('DEC_SIZE', 'f8'),
402  ('X_SIZE', 'i4'),
403  ('Y_SIZE', 'i4')])
404 
405  # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
406  # since we are looking for relative positions
407  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
408 
409  # TODO: DM-17597 will update testdata_jointcal so that the test data
410  # does not have nan as the boresight angle for HSC data. For the
411  # time being, there is this ungainly hack.
412  if camera.getName() == 'HSC' and np.isnan(defaultOrientation):
413  orientation = 270*geom.degrees
414  else:
415  orientation = defaultOrientation*geom.degrees
416  flipX = False
417 
418  # Create a temporary visitInfo for input to createInitialSkyWcs
419  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
420  boresightRotAngle=orientation,
421  rotType=afwImage.visitInfo.RotType.SKY)
422 
423  for i, detector in enumerate(camera):
424  ccdOffsets['CCDNUM'][i] = detector.getId()
425 
426  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
427 
428  detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS))
429  ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees()
430  ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees()
431 
432  bbox = detector.getBBox()
433 
434  detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin()))
435  detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax()))
436 
437  ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees())
438  ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees())
439 
440  ccdOffsets['X_SIZE'][i] = bbox.getMaxX()
441  ccdOffsets['Y_SIZE'][i] = bbox.getMaxY()
442 
443  return ccdOffsets
444 
445 
447  """
448  Compute the median pixel scale in the camera
449 
450  Returns
451  -------
452  pixelScale: `float`
453  Average pixel scale (arcsecond) over the camera
454  """
455 
456  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
457  orientation = 0.0*geom.degrees
458  flipX = False
459 
460  # Create a temporary visitInfo for input to createInitialSkyWcs
461  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
462  boresightRotAngle=orientation,
463  rotType=afwImage.visitInfo.RotType.SKY)
464 
465  pixelScales = np.zeros(len(camera))
466  for i, detector in enumerate(camera):
467  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
468  pixelScales[i] = wcs.getPixelScale().asArcseconds()
469 
470  ok, = np.where(pixelScales > 0.0)
471  return np.median(pixelScales[ok])
472 
473 
475  """
476  Compute the approximate pixel area bounded fields from the camera
477  geometry.
478 
479  Parameters
480  ----------
481  camera: `lsst.afw.cameraGeom.Camera`
482 
483  Returns
484  -------
485  approxPixelAreaFields: `dict`
486  Dictionary of approximate area fields, keyed with detector ID
487  """
488 
489  areaScaling = 1. / computeReferencePixelScale(camera)**2.
490 
491  # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem,
492  # since we are looking for relative scales
493  boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees)
494 
495  flipX = False
496  # Create a temporary visitInfo for input to createInitialSkyWcs
497  # The orientation does not matter for the area computation
498  visitInfo = afwImage.VisitInfo(boresightRaDec=boresight,
499  boresightRotAngle=0.0*geom.degrees,
500  rotType=afwImage.visitInfo.RotType.SKY)
501 
502  approxPixelAreaFields = {}
503 
504  for i, detector in enumerate(camera):
505  key = detector.getId()
506 
507  wcs = createInitialSkyWcs(visitInfo, detector, flipX)
508  bbox = detector.getBBox()
509 
510  areaField = afwMath.PixelAreaBoundedField(bbox, wcs,
511  unit=geom.arcseconds, scaling=areaScaling)
512  approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField)
513 
514  approxPixelAreaFields[key] = approxAreaField
515 
516  return approxPixelAreaFields
517 
518 
519 def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
520  """
521  Make the zeropoint schema
522 
523  Parameters
524  ----------
525  superStarChebyshevSize: `int`
526  Length of the superstar chebyshev array
527  zptChebyshevSize: `int`
528  Length of the zeropoint chebyshev array
529 
530  Returns
531  -------
532  zptSchema: `lsst.afw.table.schema`
533  """
534 
535  zptSchema = afwTable.Schema()
536 
537  zptSchema.addField('visit', type=np.int32, doc='Visit number')
538  zptSchema.addField('ccd', type=np.int32, doc='CCD number')
539  zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
540  '1: Photometric, used in fit; '
541  '2: Photometric, not used in fit; '
542  '4: Non-photometric, on partly photometric night; '
543  '8: Non-photometric, on non-photometric night; '
544  '16: No zeropoint could be determined; '
545  '32: Too few stars for reliable gray computation'))
546  zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
547  zptSchema.addField('fgcmZptErr', type=np.float64,
548  doc='Error on zeropoint, estimated from repeatability + number of obs')
549  zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
550  doc='maximum x/maximum y to scale to apply chebyshev parameters')
551  zptSchema.addField('fgcmfZptCheb', type='ArrayD',
552  size=zptChebyshevSize,
553  doc='Chebyshev parameters (flattened) for zeropoint')
554  zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
555  size=superStarChebyshevSize,
556  doc='Chebyshev parameters (flattened) for superStarFlat')
557  zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
558  zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
559  zptSchema.addField('fgcmR0', type=np.float64,
560  doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
561  zptSchema.addField('fgcmR10', type=np.float64,
562  doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
563  zptSchema.addField('fgcmGry', type=np.float64,
564  doc='Estimated gray extinction relative to atmospheric solution; '
565  'only for fgcmFlag <= 4 (see fgcmFlag) ')
566  zptSchema.addField('fgcmDeltaChrom', type=np.float64,
567  doc='Mean chromatic correction for stars in this ccd; '
568  'only for fgcmFlag <= 4 (see fgcmFlag)')
569  zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
570  zptSchema.addField('fgcmTilings', type=np.float64,
571  doc='Number of photometric tilings used for solution for ccd')
572  zptSchema.addField('fgcmFpGry', type=np.float64,
573  doc='Average gray extinction over the full focal plane '
574  '(same for all ccds in a visit)')
575  zptSchema.addField('fgcmFpGryBlue', type=np.float64,
576  doc='Average gray extinction over the full focal plane '
577  'for 25% bluest stars')
578  zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
579  doc='Error on Average gray extinction over the full focal plane '
580  'for 25% bluest stars')
581  zptSchema.addField('fgcmFpGryRed', type=np.float64,
582  doc='Average gray extinction over the full focal plane '
583  'for 25% reddest stars')
584  zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
585  doc='Error on Average gray extinction over the full focal plane '
586  'for 25% reddest stars')
587  zptSchema.addField('fgcmFpVar', type=np.float64,
588  doc='Variance of gray extinction over the full focal plane '
589  '(same for all ccds in a visit)')
590  zptSchema.addField('fgcmDust', type=np.float64,
591  doc='Gray dust extinction from the primary/corrector'
592  'at the time of the exposure')
593  zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
594  zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
595  zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
596  doc=('Local background correction from brightest percentile '
597  '(value set by deltaMagBkgOffsetPercentile) calibration '
598  'stars.'))
599  zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
600  zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
601 
602  return zptSchema
603 
604 
605 def makeZptCat(zptSchema, zpStruct):
606  """
607  Make the zeropoint catalog for persistence
608 
609  Parameters
610  ----------
611  zptSchema: `lsst.afw.table.Schema`
612  Zeropoint catalog schema
613  zpStruct: `numpy.ndarray`
614  Zeropoint structure from fgcm
615 
616  Returns
617  -------
618  zptCat: `afwTable.BaseCatalog`
619  Zeropoint catalog for persistence
620  """
621 
622  zptCat = afwTable.BaseCatalog(zptSchema)
623  zptCat.reserve(zpStruct.size)
624 
625  for filterName in zpStruct['FILTERNAME']:
626  rec = zptCat.addNew()
627  rec['filtername'] = filterName.decode('utf-8')
628 
629  zptCat['visit'][:] = zpStruct['VISIT']
630  zptCat['ccd'][:] = zpStruct['CCD']
631  zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
632  zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
633  zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
634  zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
635  zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
636  zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
637  zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
638  zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
639  zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
640  zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
641  zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
642  zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
643  zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
644  zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
645  zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
646  zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
647  zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
648  zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
649  zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
650  zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
651  zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
652  zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
653  zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
654  zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
655  zptCat['exptime'][:] = zpStruct['EXPTIME']
656 
657  return zptCat
658 
659 
661  """
662  Make the atmosphere schema
663 
664  Returns
665  -------
666  atmSchema: `lsst.afw.table.Schema`
667  """
668 
669  atmSchema = afwTable.Schema()
670 
671  atmSchema.addField('visit', type=np.int32, doc='Visit number')
672  atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
673  atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
674  atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
675  atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
676  atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
677  atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
678  atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
679  atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
680 
681  return atmSchema
682 
683 
684 def makeAtmCat(atmSchema, atmStruct):
685  """
686  Make the atmosphere catalog for persistence
687 
688  Parameters
689  ----------
690  atmSchema: `lsst.afw.table.Schema`
691  Atmosphere catalog schema
692  atmStruct: `numpy.ndarray`
693  Atmosphere structure from fgcm
694 
695  Returns
696  -------
697  atmCat: `lsst.afw.table.BaseCatalog`
698  Atmosphere catalog for persistence
699  """
700 
701  atmCat = afwTable.BaseCatalog(atmSchema)
702  atmCat.resize(atmStruct.size)
703 
704  atmCat['visit'][:] = atmStruct['VISIT']
705  atmCat['pmb'][:] = atmStruct['PMB']
706  atmCat['pwv'][:] = atmStruct['PWV']
707  atmCat['tau'][:] = atmStruct['TAU']
708  atmCat['alpha'][:] = atmStruct['ALPHA']
709  atmCat['o3'][:] = atmStruct['O3']
710  atmCat['secZenith'][:] = atmStruct['SECZENITH']
711  atmCat['cTrans'][:] = atmStruct['CTRANS']
712  atmCat['lamStd'][:] = atmStruct['LAMSTD']
713 
714  return atmCat
715 
716 
717 def makeStdSchema(nBands):
718  """
719  Make the standard star schema
720 
721  Parameters
722  ----------
723  nBands: `int`
724  Number of bands in standard star catalog
725 
726  Returns
727  -------
728  stdSchema: `lsst.afw.table.Schema`
729  """
730 
731  stdSchema = afwTable.SimpleTable.makeMinimalSchema()
732  stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations',
733  size=nBands)
734  stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations',
735  size=nBands)
736  stdSchema.addField('mag_std_noabs', type='ArrayF',
737  doc='Standard magnitude (no absolute calibration)',
738  size=nBands)
739  stdSchema.addField('magErr_std', type='ArrayF',
740  doc='Standard magnitude error',
741  size=nBands)
742  stdSchema.addField('npsfcand', type='ArrayI',
743  doc='Number of observations flagged as psf candidates',
744  size=nBands)
745 
746  return stdSchema
747 
748 
749 def makeStdCat(stdSchema, stdStruct, goodBands):
750  """
751  Make the standard star catalog for persistence
752 
753  Parameters
754  ----------
755  stdSchema: `lsst.afw.table.Schema`
756  Standard star catalog schema
757  stdStruct: `numpy.ndarray`
758  Standard star structure in FGCM format
759  goodBands: `list`
760  List of good band names used in stdStruct
761 
762  Returns
763  -------
764  stdCat: `lsst.afw.table.BaseCatalog`
765  Standard star catalog for persistence
766  """
767 
768  stdCat = afwTable.SimpleCatalog(stdSchema)
769  stdCat.resize(stdStruct.size)
770 
771  stdCat['id'][:] = stdStruct['FGCM_ID']
772  stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
773  stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
774  stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
775  stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
776  stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
777  stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
778  stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
779 
780  md = PropertyList()
781  md.set("BANDS", list(goodBands))
782  stdCat.setMetadata(md)
783 
784  return stdCat
785 
786 
787 def computeApertureRadiusFromDataRef(dataRef, fluxField):
788  """
789  Compute the radius associated with a CircularApertureFlux field or
790  associated slot.
791 
792  Parameters
793  ----------
794  dataRef : `lsst.daf.persistence.ButlerDataRef`
795  fluxField : `str`
796  CircularApertureFlux or associated slot.
797 
798  Returns
799  -------
800  apertureRadius : `float`
801  Radius of the aperture field, in pixels.
802 
803  Raises
804  ------
805  RuntimeError: Raised if flux field is not a CircularApertureFlux, ApFlux,
806  or associated slot.
807  """
808  # TODO: Move this method to more general stack method in DM-25775
809  datasetType = dataRef.butlerSubset.datasetType
810 
811  if datasetType == 'src':
812  schema = dataRef.get(datasetType='src_schema').schema
813  try:
814  fluxFieldName = schema[fluxField].asField().getName()
815  except LookupError:
816  raise RuntimeError("Could not find %s or associated slot in schema." % (fluxField))
817  # This may also raise a RuntimeError
818  apertureRadius = computeApertureRadiusFromName(fluxFieldName)
819  else:
820  # This is a sourceTable_visit
821  apertureRadius = computeApertureRadiusFromName(fluxField)
822 
823  return apertureRadius
824 
825 
827  """
828  Compute the radius associated with a CircularApertureFlux or ApFlux field.
829 
830  Parameters
831  ----------
832  fluxField : `str`
833  CircularApertureFlux or ApFlux
834 
835  Returns
836  -------
837  apertureRadius : `float`
838  Radius of the aperture field, in pixels.
839 
840  Raises
841  ------
842  RuntimeError: Raised if flux field is not a CircularApertureFlux
843  or ApFlux.
844  """
845  # TODO: Move this method to more general stack method in DM-25775
846  m = re.search(r'(CircularApertureFlux|ApFlux)_(\d+)_(\d+)_', fluxField)
847 
848  if m is None:
849  raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux")
850 
851  apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10.
852 
853  return apertureRadius
854 
855 
856 def extractReferenceMags(refStars, bands, filterMap):
857  """
858  Extract reference magnitudes from refStars for given bands and
859  associated filterMap.
860 
861  Parameters
862  ----------
863  refStars : `lsst.afw.table.BaseCatalog`
864  FGCM reference star catalog
865  bands : `list`
866  List of bands for calibration
867  filterMap: `dict`
868  FGCM mapping of filter to band
869 
870  Returns
871  -------
872  refMag : `np.ndarray`
873  nstar x nband array of reference magnitudes
874  refMagErr : `np.ndarray`
875  nstar x nband array of reference magnitude errors
876  """
877  # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent
878  # against index errors and allow more flexibility in fitting after
879  # the build stars step.
880 
881  md = refStars.getMetadata()
882  if 'FILTERNAMES' in md:
883  filternames = md.getArray('FILTERNAMES')
884 
885  # The reference catalog that fgcm wants has one entry per band
886  # in the config file
887  refMag = np.zeros((len(refStars), len(bands)),
888  dtype=refStars['refMag'].dtype) + 99.0
889  refMagErr = np.zeros_like(refMag) + 99.0
890  for i, filtername in enumerate(filternames):
891  # We are allowed to run the fit configured so that we do not
892  # use every column in the reference catalog.
893  try:
894  band = filterMap[filtername]
895  except KeyError:
896  continue
897  try:
898  ind = bands.index(band)
899  except ValueError:
900  continue
901 
902  refMag[:, ind] = refStars['refMag'][:, i]
903  refMagErr[:, ind] = refStars['refMagErr'][:, i]
904 
905  else:
906  # Continue to use old catalogs as before.
907  refMag = refStars['refMag'][:, :]
908  refMagErr = refStars['refMagErr'][:, :]
909 
910  return refMag, refMagErr
lsst.fgcmcal.utilities.computeCcdOffsets
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:378
lsst.fgcmcal.utilities.makeZptCat
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:605
lsst.fgcmcal.utilities.computeReferencePixelScale
def computeReferencePixelScale(camera)
Definition: utilities.py:446
lsst.fgcmcal.utilities.computeApertureRadiusFromName
def computeApertureRadiusFromName(fluxField)
Definition: utilities.py:826
lsst.fgcmcal.utilities.makeStdCat
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:749
lsst.fgcmcal.utilities.makeConfigDict
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
Definition: utilities.py:41
lsst.fgcmcal.utilities.computeApproxPixelAreaFields
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:474
lsst.fgcmcal.utilities.translateFgcmLut
def translateFgcmLut(lutCat, filterMap)
Definition: utilities.py:196
lsst.fgcmcal.utilities.translateVisitCatalog
def translateVisitCatalog(visitCat)
Definition: utilities.py:327
lsst.fgcmcal.utilities.makeStdSchema
def makeStdSchema(nBands)
Definition: utilities.py:717
lsst.fgcmcal.utilities.computeApertureRadiusFromDataRef
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:787
lsst.fgcmcal.utilities.makeAtmSchema
def makeAtmSchema()
Definition: utilities.py:660
lsst.fgcmcal.utilities.makeZptSchema
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:519
lsst.fgcmcal.utilities.makeAtmCat
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:684
lsst.fgcmcal.utilities.extractReferenceMags
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:856