Coverage for python/lsst/fgcmcal/fgcmCalibrateTractBase.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Base class for running fgcmcal on a single tract using src tables
22or sourceTable_visit tables.
23"""
25import sys
26import traceback
28import numpy as np
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
33from .fgcmBuildStars import FgcmBuildStarsTask, FgcmBuildStarsConfig
34from .fgcmFitCycle import FgcmFitCycleConfig
35from .fgcmOutputProducts import FgcmOutputProductsTask
36from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
37from .utilities import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
38from .utilities import makeZptSchema, makeZptCat
39from .utilities import makeAtmSchema, makeAtmCat
40from .utilities import makeStdSchema, makeStdCat
42import fgcm
44__all__ = ['FgcmCalibrateTractConfigBase', 'FgcmCalibrateTractBaseTask', 'FgcmCalibrateTractRunner']
47class FgcmCalibrateTractConfigBase(pexConfig.Config):
48 """Config for FgcmCalibrateTract"""
50 fgcmBuildStars = pexConfig.ConfigurableField(
51 target=FgcmBuildStarsTask,
52 doc="Task to load and match stars for fgcm",
53 )
54 fgcmFitCycle = pexConfig.ConfigField(
55 dtype=FgcmFitCycleConfig,
56 doc="Config to run a single fgcm fit cycle",
57 )
58 fgcmOutputProducts = pexConfig.ConfigurableField(
59 target=FgcmOutputProductsTask,
60 doc="Task to output fgcm products",
61 )
62 convergenceTolerance = pexConfig.Field(
63 doc="Tolerance on repeatability convergence (per band)",
64 dtype=float,
65 default=0.005,
66 )
67 maxFitCycles = pexConfig.Field(
68 doc="Maximum number of fit cycles",
69 dtype=int,
70 default=5,
71 )
72 doDebuggingPlots = pexConfig.Field(
73 doc="Make plots for debugging purposes?",
74 dtype=bool,
75 default=False,
76 )
78 def setDefaults(self):
79 pexConfig.Config.setDefaults(self)
81 self.fgcmFitCycle.quietMode = True
82 self.fgcmOutputProducts.doReferenceCalibration = False
83 self.fgcmOutputProducts.doRefcatOutput = False
84 self.fgcmOutputProducts.cycleNumber = 0
85 self.fgcmOutputProducts.photoCal.applyColorTerms = False
87 def validate(self):
88 super().validate()
90 for band in self.fgcmFitCycle.bands:
91 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
92 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
93 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
94 self, msg)
97class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner):
98 """Subclass of TaskRunner for FgcmCalibrateTractTask
100 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
101 the butler (for persistence and mapper data), and a list of dataRefs
102 extracted from the command line. This task runs on a constrained set
103 of dataRefs, typically a single tract.
104 This class transforms the process arguments generated by the ArgumentParser
105 into the arguments expected by FgcmCalibrateTractTask.run().
106 This runner does not use any parallelization.
107 """
109 @staticmethod
110 def getTargetList(parsedCmd):
111 """
112 Return a list with one element: a tuple with the butler and
113 list of dataRefs.
114 """
115 return [(parsedCmd.butler, parsedCmd.id.refList)]
117 def __call__(self, args):
118 """
119 Parameters
120 ----------
121 args: `tuple` with (butler, dataRefList)
123 Returns
124 -------
125 exitStatus: `list` with `lsst.pipe.base.Struct`
126 exitStatus (0: success; 1: failure)
127 May also contain results if `self.doReturnResults` is `True`.
128 """
129 butler, dataRefList = args
131 task = self.TaskClass(config=self.config, log=self.log)
133 exitStatus = 0
134 if self.doRaise:
135 results = task.runDataRef(butler, dataRefList)
136 else:
137 try:
138 results = task.runDataRef(butler, dataRefList)
139 except Exception as e:
140 exitStatus = 1
141 task.log.fatal("Failed: %s" % e)
142 if not isinstance(e, pipeBase.TaskError):
143 traceback.print_exc(file=sys.stderr)
145 task.writeMetadata(butler)
147 if self.doReturnResults:
148 return [pipeBase.Struct(exitStatus=exitStatus,
149 results=results)]
150 else:
151 return [pipeBase.Struct(exitStatus=exitStatus)]
153 def run(self, parsedCmd):
154 """
155 Run the task, with no multiprocessing
157 Parameters
158 ----------
159 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
160 """
162 resultList = []
164 if self.precall(parsedCmd):
165 targetList = self.getTargetList(parsedCmd)
166 resultList = self(targetList[0])
168 return resultList
171class FgcmCalibrateTractBaseTask(pipeBase.CmdLineTask):
172 """
173 Base class to calibrate a single tract using fgcmcal
174 """
175 def __init__(self, butler=None, **kwargs):
176 """
177 Instantiate an `FgcmCalibrateTractTask`.
179 Parameters
180 ----------
181 butler : `lsst.daf.persistence.Butler`
182 """
184 pipeBase.CmdLineTask.__init__(self, **kwargs)
186 # no saving of metadata for now
187 def _getMetadataName(self):
188 return None
190 @pipeBase.timeMethod
191 def runDataRef(self, butler, dataRefs):
192 """
193 Run full FGCM calibration on a single tract, including building star list,
194 fitting multiple cycles, and making outputs.
196 Parameters
197 ----------
198 butler: `lsst.daf.persistence.Butler`
199 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
200 Data references for the input visits.
201 These may be either per-ccd "src" or per-visit"sourceTable_visit"
202 references.
204 Raises
205 ------
206 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
207 not True, or if fgcmLookUpTable is not available, or if
208 doSubtractLocalBackground is True and aperture radius cannot be
209 determined.
210 """
211 datasetType = dataRefs[0].butlerSubset.datasetType
212 self.log.info("Running with %d %s dataRefs" % (len(dataRefs), datasetType))
214 if not butler.datasetExists('fgcmLookUpTable'):
215 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable")
217 if not self.config.fgcmBuildStars.doReferenceMatches:
218 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
219 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
220 if self.config.fgcmBuildStars.checkAllCcds:
221 raise RuntimeError("Cannot run FgcmCalibrateTract with "
222 "fgcmBuildStars.checkAllCcds set to True")
224 self.makeSubtask("fgcmBuildStars", butler=butler)
225 self.makeSubtask("fgcmOutputProducts", butler=butler)
227 # Compute the aperture radius if necessary. This is useful to do now before
228 # any heavy lifting has happened (fail early).
229 calibFluxApertureRadius = None
230 if self.config.fgcmBuildStars.doSubtractLocalBackground:
231 try:
232 field = self.config.fgcmBuildStars.instFluxField
233 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
234 field)
235 except RuntimeError:
236 raise RuntimeError("Could not determine aperture radius from %s. "
237 "Cannot use doSubtractLocalBackground." %
238 (field))
240 # Run the build stars tasks
241 tract = int(dataRefs[0].dataId['tract'])
242 self.log.info("Running on tract %d" % (tract))
244 # Note that we will need visitCat at the end of the procedure for the outputs
245 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs)
246 camera = butler.get('camera')
247 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs)
248 rad = calibFluxApertureRadius
249 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
250 visitCat,
251 calibFluxApertureRadius=rad)
253 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
254 self.fgcmBuildStars.fgcmMatchStars(butler,
255 visitCat,
256 fgcmStarObservationCat)
258 # Load the LUT
259 lutCat = butler.get('fgcmLookUpTable')
260 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
261 dict(self.config.fgcmFitCycle.filterMap))
262 del lutCat
264 # Translate the visit catalog into fgcm format
265 fgcmExpInfo = translateVisitCatalog(visitCat)
267 camera = butler.get('camera')
268 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera,
269 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
270 True, False, tract=tract)
271 # Turn off plotting in tract mode
272 configDict['doPlots'] = False
274 # Use the first orientation.
275 # TODO: DM-21215 will generalize to arbitrary camera orientations
276 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
277 del camera
279 # Set up the fit cycle task
281 noFitsDict = {'lutIndex': lutIndexVals,
282 'lutStd': lutStd,
283 'expInfo': fgcmExpInfo,
284 'ccdOffsets': ccdOffsets}
286 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
287 noFitsDict=noFitsDict, noOutput=True)
289 # We determine the conversion from the native units (typically radians) to
290 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
291 # numpy arrays rather than Angles, which would we approximately 600x slower.
292 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra'])
294 # To load the stars, we need an initial parameter object
295 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
296 fgcmLut,
297 fgcmExpInfo)
299 # Match star observations to visits
300 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
301 # actually be transferred into fgcm using the indexing below.
303 obsIndex = fgcmStarIndicesCat['obsIndex']
304 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'],
305 fgcmStarObservationCat['visit'][obsIndex])
307 refMag, refMagErr = extractReferenceMags(fgcmRefCat,
308 self.config.fgcmFitCycle.bands,
309 self.config.fgcmFitCycle.filterMap)
310 refId = fgcmRefCat['fgcm_id'][:]
312 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
313 fgcmStars.loadStars(fgcmPars,
314 fgcmStarObservationCat['visit'][obsIndex],
315 fgcmStarObservationCat['ccd'][obsIndex],
316 fgcmStarObservationCat['ra'][obsIndex] * conv,
317 fgcmStarObservationCat['dec'][obsIndex] * conv,
318 fgcmStarObservationCat['instMag'][obsIndex],
319 fgcmStarObservationCat['instMagErr'][obsIndex],
320 fgcmExpInfo['FILTERNAME'][visitIndex],
321 fgcmStarIdCat['fgcm_id'][:],
322 fgcmStarIdCat['ra'][:],
323 fgcmStarIdCat['dec'][:],
324 fgcmStarIdCat['obsArrIndex'][:],
325 fgcmStarIdCat['nObs'][:],
326 obsX=fgcmStarObservationCat['x'][obsIndex],
327 obsY=fgcmStarObservationCat['y'][obsIndex],
328 obsDeltaMagBkg=fgcmStarObservationCat['deltaMagBkg'][obsIndex],
329 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex],
330 refID=refId,
331 refMag=refMag,
332 refMagErr=refMagErr,
333 flagID=None,
334 flagFlag=None,
335 computeNobs=True)
337 # Clear out some memory
338 del fgcmStarIdCat
339 del fgcmStarIndicesCat
340 del fgcmRefCat
342 fgcmFitCycle.setLUT(fgcmLut)
343 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
345 converged = False
346 cycleNumber = 0
348 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
349 previousParInfo = None
350 previousParams = None
351 previousSuperStar = None
353 while (not converged and cycleNumber < self.config.maxFitCycles):
355 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
357 if cycleNumber > 0:
358 # Use parameters from previous cycle
359 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
360 fgcmExpInfo,
361 previousParInfo,
362 previousParams,
363 previousSuperStar)
364 # We need to reset the star magnitudes and errors for the next
365 # cycle
366 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
367 fgcmStarObservationCat['instMagErr'][obsIndex])
368 fgcmFitCycle.initialCycle = False
370 fgcmFitCycle.setPars(fgcmPars)
371 fgcmFitCycle.finishSetup()
373 fgcmFitCycle.run()
375 # Grab the parameters for the next cycle
376 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
377 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
379 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber))
380 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
381 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
382 continue
383 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
384 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
386 # Check for convergence
387 if np.alltrue((previousReservedRawRepeatability -
388 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) <
389 self.config.convergenceTolerance):
390 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber))
391 converged = True
392 else:
393 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
394 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
395 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False
396 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
397 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
398 self.log.info("Setting exposure gray photometricity cuts to:")
399 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
400 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
401 continue
402 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
403 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut))
405 cycleNumber += 1
407 # Log warning if not converged
408 if not converged:
409 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
411 # Do final clean-up iteration
412 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
413 fgcmFitCycle.fgcmConfig.resetParameters = False
414 fgcmFitCycle.fgcmConfig.maxIter = 0
415 fgcmFitCycle.fgcmConfig.outputZeropoints = True
416 fgcmFitCycle.fgcmConfig.outputStandards = True
417 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
418 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
419 fgcmFitCycle.initialCycle = False
421 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
422 fgcmExpInfo,
423 previousParInfo,
424 previousParams,
425 previousSuperStar)
426 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
427 fgcmStarObservationCat['instMagErr'][obsIndex])
428 fgcmFitCycle.setPars(fgcmPars)
429 fgcmFitCycle.finishSetup()
431 self.log.info("Running final clean-up fit cycle...")
432 fgcmFitCycle.run()
434 self.log.info("Raw repeatability after clean-up cycle is:")
435 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
436 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
437 continue
438 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
439 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
441 # Do the outputs. Need to keep track of tract.
443 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
444 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
446 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
447 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
449 atmSchema = makeAtmSchema()
450 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
452 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
453 stdSchema = makeStdSchema(len(goodBands))
454 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
456 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract,
457 visitCat,
458 zptCat, atmCat, stdCat,
459 self.config.fgcmBuildStars,
460 self.config.fgcmFitCycle)
461 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
463 return outStruct