lsst.fgcmcal  20.0.0-4-ge48a6ca+3
fgcmCalibrateTractBase.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Base class for running fgcmcal on a single tract using src tables
22 or sourceTable_visit tables.
23 """
24 
25 import sys
26 import traceback
27 
28 import numpy as np
29 
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 from .fgcmBuildStars import FgcmBuildStarsTask, FgcmBuildStarsConfig
34 from .fgcmFitCycle import FgcmFitCycleConfig
35 from .fgcmOutputProducts import FgcmOutputProductsTask
36 from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
37 from .utilities import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
38 from .utilities import makeZptSchema, makeZptCat
39 from .utilities import makeAtmSchema, makeAtmCat
40 from .utilities import makeStdSchema, makeStdCat
41 
42 import fgcm
43 
44 __all__ = ['FgcmCalibrateTractConfigBase', 'FgcmCalibrateTractBaseTask', 'FgcmCalibrateTractRunner']
45 
46 
47 class FgcmCalibrateTractConfigBase(pexConfig.Config):
48  """Config for FgcmCalibrateTract"""
49 
50  fgcmBuildStars = pexConfig.ConfigurableField(
51  target=FgcmBuildStarsTask,
52  doc="Task to load and match stars for fgcm",
53  )
54  fgcmFitCycle = pexConfig.ConfigField(
55  dtype=FgcmFitCycleConfig,
56  doc="Config to run a single fgcm fit cycle",
57  )
58  fgcmOutputProducts = pexConfig.ConfigurableField(
59  target=FgcmOutputProductsTask,
60  doc="Task to output fgcm products",
61  )
62  convergenceTolerance = pexConfig.Field(
63  doc="Tolerance on repeatability convergence (per band)",
64  dtype=float,
65  default=0.005,
66  )
67  maxFitCycles = pexConfig.Field(
68  doc="Maximum number of fit cycles",
69  dtype=int,
70  default=5,
71  )
72  doDebuggingPlots = pexConfig.Field(
73  doc="Make plots for debugging purposes?",
74  dtype=bool,
75  default=False,
76  )
77 
78  def setDefaults(self):
79  pexConfig.Config.setDefaults(self)
80 
81  self.fgcmFitCycle.quietMode = True
82  self.fgcmOutputProducts.doReferenceCalibration = False
83  self.fgcmOutputProducts.doRefcatOutput = False
84  self.fgcmOutputProducts.cycleNumber = 0
85  self.fgcmOutputProducts.photoCal.applyColorTerms = False
86 
87  def validate(self):
88  super().validate()
89 
90  for band in self.fgcmFitCycle.bands:
91  if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
92  msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
93  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
94  self, msg)
95 
96 
97 class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner):
98  """Subclass of TaskRunner for FgcmCalibrateTractTask
99 
100  fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
101  the butler (for persistence and mapper data), and a list of dataRefs
102  extracted from the command line. This task runs on a constrained set
103  of dataRefs, typically a single tract.
104  This class transforms the process arguments generated by the ArgumentParser
105  into the arguments expected by FgcmCalibrateTractTask.run().
106  This runner does not use any parallelization.
107  """
108 
109  @staticmethod
110  def getTargetList(parsedCmd):
111  """
112  Return a list with one element: a tuple with the butler and
113  list of dataRefs.
114  """
115  return [(parsedCmd.butler, parsedCmd.id.refList)]
116 
117  def __call__(self, args):
118  """
119  Parameters
120  ----------
121  args: `tuple` with (butler, dataRefList)
122 
123  Returns
124  -------
125  exitStatus: `list` with `lsst.pipe.base.Struct`
126  exitStatus (0: success; 1: failure)
127  May also contain results if `self.doReturnResults` is `True`.
128  """
129  butler, dataRefList = args
130 
131  task = self.TaskClass(config=self.config, log=self.log)
132 
133  exitStatus = 0
134  if self.doRaise:
135  results = task.runDataRef(butler, dataRefList)
136  else:
137  try:
138  results = task.runDataRef(butler, dataRefList)
139  except Exception as e:
140  exitStatus = 1
141  task.log.fatal("Failed: %s" % e)
142  if not isinstance(e, pipeBase.TaskError):
143  traceback.print_exc(file=sys.stderr)
144 
145  task.writeMetadata(butler)
146 
147  if self.doReturnResults:
148  return [pipeBase.Struct(exitStatus=exitStatus,
149  results=results)]
150  else:
151  return [pipeBase.Struct(exitStatus=exitStatus)]
152 
153  def run(self, parsedCmd):
154  """
155  Run the task, with no multiprocessing
156 
157  Parameters
158  ----------
159  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
160  """
161 
162  resultList = []
163 
164  if self.precall(parsedCmd):
165  targetList = self.getTargetList(parsedCmd)
166  resultList = self(targetList[0])
167 
168  return resultList
169 
170 
171 class FgcmCalibrateTractBaseTask(pipeBase.CmdLineTask):
172  """
173  Base class to calibrate a single tract using fgcmcal
174  """
175  def __init__(self, butler=None, **kwargs):
176  """
177  Instantiate an `FgcmCalibrateTractTask`.
178 
179  Parameters
180  ----------
181  butler : `lsst.daf.persistence.Butler`
182  """
183 
184  pipeBase.CmdLineTask.__init__(self, **kwargs)
185 
186  # no saving of metadata for now
187  def _getMetadataName(self):
188  return None
189 
190  @pipeBase.timeMethod
191  def runDataRef(self, butler, dataRefs):
192  """
193  Run full FGCM calibration on a single tract, including building star list,
194  fitting multiple cycles, and making outputs.
195 
196  Parameters
197  ----------
198  butler: `lsst.daf.persistence.Butler`
199  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
200  Data references for the input visits.
201  These may be either per-ccd "src" or per-visit"sourceTable_visit"
202  references.
203 
204  Raises
205  ------
206  RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
207  not True, or if fgcmLookUpTable is not available, or if
208  doSubtractLocalBackground is True and aperture radius cannot be
209  determined.
210  """
211  datasetType = dataRefs[0].butlerSubset.datasetType
212  self.log.info("Running with %d %s dataRefs" % (len(dataRefs), datasetType))
213 
214  if not butler.datasetExists('fgcmLookUpTable'):
215  raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable")
216 
217  if not self.config.fgcmBuildStars.doReferenceMatches:
218  raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
219  if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
220  if self.config.fgcmBuildStars.checkAllCcds:
221  raise RuntimeError("Cannot run FgcmCalibrateTract with "
222  "fgcmBuildStars.checkAllCcds set to True")
223 
224  self.makeSubtask("fgcmBuildStars", butler=butler)
225  self.makeSubtask("fgcmOutputProducts", butler=butler)
226 
227  # Compute the aperture radius if necessary. This is useful to do now before
228  # any heavy lifting has happened (fail early).
229  calibFluxApertureRadius = None
230  if self.config.fgcmBuildStars.doSubtractLocalBackground:
231  try:
232  field = self.config.fgcmBuildStars.instFluxField
233  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
234  field)
235  except RuntimeError:
236  raise RuntimeError("Could not determine aperture radius from %s. "
237  "Cannot use doSubtractLocalBackground." %
238  (field))
239 
240  # Run the build stars tasks
241  tract = int(dataRefs[0].dataId['tract'])
242  self.log.info("Running on tract %d" % (tract))
243 
244  # Note that we will need visitCat at the end of the procedure for the outputs
245  groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs)
246  camera = butler.get('camera')
247  visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs)
248  rad = calibFluxApertureRadius
249  fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
250  visitCat,
251  calibFluxApertureRadius=rad)
252 
253  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
254  self.fgcmBuildStars.fgcmMatchStars(butler,
255  visitCat,
256  fgcmStarObservationCat)
257 
258  # Load the LUT
259  lutCat = butler.get('fgcmLookUpTable')
260  fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
261  dict(self.config.fgcmFitCycle.filterMap))
262  del lutCat
263 
264  # Translate the visit catalog into fgcm format
265  fgcmExpInfo = translateVisitCatalog(visitCat)
266 
267  camera = butler.get('camera')
268  configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera,
269  self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
270  True, False, tract=tract)
271  # Turn off plotting in tract mode
272  configDict['doPlots'] = False
273 
274  # Use the first orientation.
275  # TODO: DM-21215 will generalize to arbitrary camera orientations
276  ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
277  del camera
278 
279  # Set up the fit cycle task
280 
281  noFitsDict = {'lutIndex': lutIndexVals,
282  'lutStd': lutStd,
283  'expInfo': fgcmExpInfo,
284  'ccdOffsets': ccdOffsets}
285 
286  fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
287  noFitsDict=noFitsDict, noOutput=True)
288 
289  # We determine the conversion from the native units (typically radians) to
290  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
291  # numpy arrays rather than Angles, which would we approximately 600x slower.
292  conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra'])
293 
294  # To load the stars, we need an initial parameter object
295  fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
296  fgcmLut,
297  fgcmExpInfo)
298 
299  # Match star observations to visits
300  # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
301  # actually be transferred into fgcm using the indexing below.
302 
303  obsIndex = fgcmStarIndicesCat['obsIndex']
304  visitIndex = np.searchsorted(fgcmExpInfo['VISIT'],
305  fgcmStarObservationCat['visit'][obsIndex])
306 
307  refMag, refMagErr = extractReferenceMags(fgcmRefCat,
308  self.config.fgcmFitCycle.bands,
309  self.config.fgcmFitCycle.filterMap)
310  refId = fgcmRefCat['fgcm_id'][:]
311 
312  fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
313  fgcmStars.loadStars(fgcmPars,
314  fgcmStarObservationCat['visit'][obsIndex],
315  fgcmStarObservationCat['ccd'][obsIndex],
316  fgcmStarObservationCat['ra'][obsIndex] * conv,
317  fgcmStarObservationCat['dec'][obsIndex] * conv,
318  fgcmStarObservationCat['instMag'][obsIndex],
319  fgcmStarObservationCat['instMagErr'][obsIndex],
320  fgcmExpInfo['FILTERNAME'][visitIndex],
321  fgcmStarIdCat['fgcm_id'][:],
322  fgcmStarIdCat['ra'][:],
323  fgcmStarIdCat['dec'][:],
324  fgcmStarIdCat['obsArrIndex'][:],
325  fgcmStarIdCat['nObs'][:],
326  obsX=fgcmStarObservationCat['x'][obsIndex],
327  obsY=fgcmStarObservationCat['y'][obsIndex],
328  psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex],
329  refID=refId,
330  refMag=refMag,
331  refMagErr=refMagErr,
332  flagID=None,
333  flagFlag=None,
334  computeNobs=True)
335 
336  # Clear out some memory
337  del fgcmStarIdCat
338  del fgcmStarIndicesCat
339  del fgcmRefCat
340 
341  fgcmFitCycle.setLUT(fgcmLut)
342  fgcmFitCycle.setStars(fgcmStars, fgcmPars)
343 
344  converged = False
345  cycleNumber = 0
346 
347  previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
348  previousParInfo = None
349  previousParams = None
350  previousSuperStar = None
351 
352  while (not converged and cycleNumber < self.config.maxFitCycles):
353 
354  fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
355 
356  if cycleNumber > 0:
357  # Use parameters from previous cycle
358  fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
359  fgcmExpInfo,
360  previousParInfo,
361  previousParams,
362  previousSuperStar)
363  # We need to reset the star magnitudes and errors for the next
364  # cycle
365  fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
366  fgcmStarObservationCat['instMagErr'][obsIndex])
367  fgcmFitCycle.initialCycle = False
368 
369  fgcmFitCycle.setPars(fgcmPars)
370  fgcmFitCycle.finishSetup()
371 
372  fgcmFitCycle.run()
373 
374  # Grab the parameters for the next cycle
375  previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
376  previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
377 
378  self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber))
379  for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
380  if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
381  continue
382  rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
383  self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
384 
385  # Check for convergence
386  if np.alltrue((previousReservedRawRepeatability -
387  fgcmFitCycle.fgcmPars.compReservedRawRepeatability) <
388  self.config.convergenceTolerance):
389  self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber))
390  converged = True
391  else:
392  fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
393  fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
394  fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False
395  fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
396  previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
397  self.log.info("Setting exposure gray photometricity cuts to:")
398  for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
399  if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
400  continue
401  cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
402  self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut))
403 
404  cycleNumber += 1
405 
406  # Log warning if not converged
407  if not converged:
408  self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
409 
410  # Do final clean-up iteration
411  fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
412  fgcmFitCycle.fgcmConfig.resetParameters = False
413  fgcmFitCycle.fgcmConfig.maxIter = 0
414  fgcmFitCycle.fgcmConfig.outputZeropoints = True
415  fgcmFitCycle.fgcmConfig.outputStandards = True
416  fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
417  fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
418  fgcmFitCycle.initialCycle = False
419 
420  fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
421  fgcmExpInfo,
422  previousParInfo,
423  previousParams,
424  previousSuperStar)
425  fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
426  fgcmStarObservationCat['instMagErr'][obsIndex])
427  fgcmFitCycle.setPars(fgcmPars)
428  fgcmFitCycle.finishSetup()
429 
430  self.log.info("Running final clean-up fit cycle...")
431  fgcmFitCycle.run()
432 
433  self.log.info("Raw repeatability after clean-up cycle is:")
434  for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
435  if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
436  continue
437  rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
438  self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
439 
440  # Do the outputs. Need to keep track of tract.
441 
442  superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
443  zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
444 
445  zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
446  zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
447 
448  atmSchema = makeAtmSchema()
449  atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
450 
451  stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
452  stdSchema = makeStdSchema(len(goodBands))
453  stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
454 
455  outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract,
456  visitCat,
457  zptCat, atmCat, stdCat,
458  self.config.fgcmBuildStars,
459  self.config.fgcmFitCycle)
460  outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
461 
462  return outStruct
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractBaseTask.__init__
def __init__(self, butler=None, **kwargs)
Definition: fgcmCalibrateTractBase.py:175
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractRunner
Definition: fgcmCalibrateTractBase.py:97
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractConfigBase.validate
def validate(self)
Definition: fgcmCalibrateTractBase.py:87
lsst.fgcmcal.utilities.computeCcdOffsets
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:376
lsst.fgcmcal.utilities.makeZptCat
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:596
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractConfigBase
Definition: fgcmCalibrateTractBase.py:47
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractConfigBase.fgcmFitCycle
fgcmFitCycle
Definition: fgcmCalibrateTractBase.py:54
lsst.fgcmcal.utilities.makeStdCat
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:738
lsst.fgcmcal.utilities.makeConfigDict
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
Definition: utilities.py:41
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractBaseTask.runDataRef
def runDataRef(self, butler, dataRefs)
Definition: fgcmCalibrateTractBase.py:191
lsst.fgcmcal.utilities.translateFgcmLut
def translateFgcmLut(lutCat, filterMap)
Definition: utilities.py:194
lsst.fgcmcal.utilities.translateVisitCatalog
def translateVisitCatalog(visitCat)
Definition: utilities.py:325
lsst.fgcmcal.utilities.makeStdSchema
def makeStdSchema(nBands)
Definition: utilities.py:706
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractRunner.__call__
def __call__(self, args)
Definition: fgcmCalibrateTractBase.py:117
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractConfigBase.fgcmOutputProducts
fgcmOutputProducts
Definition: fgcmCalibrateTractBase.py:58
lsst.fgcmcal.utilities.computeApertureRadiusFromDataRef
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:776
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractBaseTask
Definition: fgcmCalibrateTractBase.py:171
lsst.fgcmcal.utilities.makeAtmSchema
def makeAtmSchema()
Definition: utilities.py:649
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractConfigBase.setDefaults
def setDefaults(self)
Definition: fgcmCalibrateTractBase.py:78
lsst.fgcmcal.utilities.makeZptSchema
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:517
lsst.fgcmcal.utilities.makeAtmCat
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:673
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractRunner.getTargetList
def getTargetList(parsedCmd)
Definition: fgcmCalibrateTractBase.py:110
lsst.fgcmcal.fgcmCalibrateTractBase.FgcmCalibrateTractRunner.run
def run(self, parsedCmd)
Definition: fgcmCalibrateTractBase.py:153
lsst.fgcmcal.utilities.extractReferenceMags
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:845