Coverage for python/lsst/fgcmcal/fgcmCalibrateTractBase.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Base class for running fgcmcal on a single tract using src tables
22or sourceTable_visit tables.
23"""
25import sys
26import traceback
28import numpy as np
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
33from .fgcmBuildStars import FgcmBuildStarsTask, FgcmBuildStarsConfig
34from .fgcmFitCycle import FgcmFitCycleConfig
35from .fgcmOutputProducts import FgcmOutputProductsTask
36from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
37from .utilities import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
38from .utilities import makeZptSchema, makeZptCat
39from .utilities import makeAtmSchema, makeAtmCat
40from .utilities import makeStdSchema, makeStdCat
42import fgcm
44__all__ = ['FgcmCalibrateTractConfigBase', 'FgcmCalibrateTractBaseTask', 'FgcmCalibrateTractRunner']
47class FgcmCalibrateTractConfigBase(pexConfig.Config):
48 """Config for FgcmCalibrateTract"""
50 fgcmBuildStars = pexConfig.ConfigurableField(
51 target=FgcmBuildStarsTask,
52 doc="Task to load and match stars for fgcm",
53 )
54 fgcmFitCycle = pexConfig.ConfigField(
55 dtype=FgcmFitCycleConfig,
56 doc="Config to run a single fgcm fit cycle",
57 )
58 fgcmOutputProducts = pexConfig.ConfigurableField(
59 target=FgcmOutputProductsTask,
60 doc="Task to output fgcm products",
61 )
62 convergenceTolerance = pexConfig.Field(
63 doc="Tolerance on repeatability convergence (per band)",
64 dtype=float,
65 default=0.005,
66 )
67 maxFitCycles = pexConfig.Field(
68 doc="Maximum number of fit cycles",
69 dtype=int,
70 default=5,
71 )
72 doDebuggingPlots = pexConfig.Field(
73 doc="Make plots for debugging purposes?",
74 dtype=bool,
75 default=False,
76 )
78 def setDefaults(self):
79 pexConfig.Config.setDefaults(self)
81 self.fgcmFitCycle.quietMode = True
82 self.fgcmOutputProducts.doReferenceCalibration = False
83 self.fgcmOutputProducts.doRefcatOutput = False
84 self.fgcmOutputProducts.cycleNumber = 0
85 self.fgcmOutputProducts.photoCal.applyColorTerms = False
87 def validate(self):
88 super().validate()
90 for band in self.fgcmFitCycle.bands:
91 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
92 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
93 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
94 self, msg)
97class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner):
98 """Subclass of TaskRunner for FgcmCalibrateTractTask
100 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
101 the butler (for persistence and mapper data), and a list of dataRefs
102 extracted from the command line. This task runs on a constrained set
103 of dataRefs, typically a single tract.
104 This class transforms the process arguments generated by the ArgumentParser
105 into the arguments expected by FgcmCalibrateTractTask.run().
106 This runner does not use any parallelization.
107 """
109 @staticmethod
110 def getTargetList(parsedCmd):
111 """
112 Return a list with one element: a tuple with the butler and
113 list of dataRefs.
114 """
115 return [(parsedCmd.butler, parsedCmd.id.refList)]
117 def __call__(self, args):
118 """
119 Parameters
120 ----------
121 args: `tuple` with (butler, dataRefList)
123 Returns
124 -------
125 exitStatus: `list` with `lsst.pipe.base.Struct`
126 exitStatus (0: success; 1: failure)
127 May also contain results if `self.doReturnResults` is `True`.
128 """
129 butler, dataRefList = args
131 task = self.TaskClass(config=self.config, log=self.log)
133 exitStatus = 0
134 if self.doRaise:
135 results = task.runDataRef(butler, dataRefList)
136 else:
137 try:
138 results = task.runDataRef(butler, dataRefList)
139 except Exception as e:
140 exitStatus = 1
141 task.log.fatal("Failed: %s" % e)
142 if not isinstance(e, pipeBase.TaskError):
143 traceback.print_exc(file=sys.stderr)
145 task.writeMetadata(butler)
147 if self.doReturnResults:
148 return [pipeBase.Struct(exitStatus=exitStatus,
149 results=results)]
150 else:
151 return [pipeBase.Struct(exitStatus=exitStatus)]
153 def run(self, parsedCmd):
154 """
155 Run the task, with no multiprocessing
157 Parameters
158 ----------
159 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
160 """
162 resultList = []
164 if self.precall(parsedCmd):
165 targetList = self.getTargetList(parsedCmd)
166 resultList = self(targetList[0])
168 return resultList
171class FgcmCalibrateTractBaseTask(pipeBase.CmdLineTask):
172 """
173 Base class to calibrate a single tract using fgcmcal
174 """
175 def __init__(self, butler=None, **kwargs):
176 """
177 Instantiate an `FgcmCalibrateTractTask`.
179 Parameters
180 ----------
181 butler : `lsst.daf.persistence.Butler`
182 """
184 pipeBase.CmdLineTask.__init__(self, **kwargs)
186 # no saving of metadata for now
187 def _getMetadataName(self):
188 return None
190 @pipeBase.timeMethod
191 def runDataRef(self, butler, dataRefs):
192 """
193 Run full FGCM calibration on a single tract, including building star list,
194 fitting multiple cycles, and making outputs.
196 Parameters
197 ----------
198 butler: `lsst.daf.persistence.Butler`
199 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
200 Data references for the input visits.
201 These may be either per-ccd "src" or per-visit"sourceTable_visit"
202 references.
204 Raises
205 ------
206 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
207 not True, or if fgcmLookUpTable is not available, or if
208 doSubtractLocalBackground is True and aperture radius cannot be
209 determined.
210 """
211 datasetType = dataRefs[0].butlerSubset.datasetType
212 self.log.info("Running with %d %s dataRefs" % (len(dataRefs), datasetType))
214 if not butler.datasetExists('fgcmLookUpTable'):
215 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable")
217 if not self.config.fgcmBuildStars.doReferenceMatches:
218 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
219 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
220 if self.config.fgcmBuildStars.checkAllCcds:
221 raise RuntimeError("Cannot run FgcmCalibrateTract with "
222 "fgcmBuildStars.checkAllCcds set to True")
224 self.makeSubtask("fgcmBuildStars", butler=butler)
225 self.makeSubtask("fgcmOutputProducts", butler=butler)
227 # Compute the aperture radius if necessary. This is useful to do now before
228 # any heavy lifting has happened (fail early).
229 calibFluxApertureRadius = None
230 if self.config.fgcmBuildStars.doSubtractLocalBackground:
231 try:
232 field = self.config.fgcmBuildStars.instFluxField
233 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
234 field)
235 except RuntimeError:
236 raise RuntimeError("Could not determine aperture radius from %s. "
237 "Cannot use doSubtractLocalBackground." %
238 (field))
240 # Run the build stars tasks
241 tract = int(dataRefs[0].dataId['tract'])
242 self.log.info("Running on tract %d" % (tract))
244 # Note that we will need visitCat at the end of the procedure for the outputs
245 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs)
246 camera = butler.get('camera')
247 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs)
248 rad = calibFluxApertureRadius
249 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
250 visitCat,
251 calibFluxApertureRadius=rad)
253 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
254 self.fgcmBuildStars.fgcmMatchStars(butler,
255 visitCat,
256 fgcmStarObservationCat)
258 # Load the LUT
259 lutCat = butler.get('fgcmLookUpTable')
260 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
261 dict(self.config.fgcmFitCycle.filterMap))
262 del lutCat
264 # Translate the visit catalog into fgcm format
265 fgcmExpInfo = translateVisitCatalog(visitCat)
267 camera = butler.get('camera')
268 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera,
269 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
270 True, False, tract=tract)
271 # Turn off plotting in tract mode
272 configDict['doPlots'] = False
274 # Use the first orientation.
275 # TODO: DM-21215 will generalize to arbitrary camera orientations
276 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
277 del camera
279 # Set up the fit cycle task
281 noFitsDict = {'lutIndex': lutIndexVals,
282 'lutStd': lutStd,
283 'expInfo': fgcmExpInfo,
284 'ccdOffsets': ccdOffsets}
286 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
287 noFitsDict=noFitsDict, noOutput=True)
289 # We determine the conversion from the native units (typically radians) to
290 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
291 # numpy arrays rather than Angles, which would we approximately 600x slower.
292 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra'])
294 # To load the stars, we need an initial parameter object
295 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
296 fgcmLut,
297 fgcmExpInfo)
299 # Match star observations to visits
300 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
301 # actually be transferred into fgcm using the indexing below.
303 obsIndex = fgcmStarIndicesCat['obsIndex']
304 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'],
305 fgcmStarObservationCat['visit'][obsIndex])
307 refMag, refMagErr = extractReferenceMags(fgcmRefCat,
308 self.config.fgcmFitCycle.bands,
309 self.config.fgcmFitCycle.filterMap)
310 refId = fgcmRefCat['fgcm_id'][:]
312 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
313 fgcmStars.loadStars(fgcmPars,
314 fgcmStarObservationCat['visit'][obsIndex],
315 fgcmStarObservationCat['ccd'][obsIndex],
316 fgcmStarObservationCat['ra'][obsIndex] * conv,
317 fgcmStarObservationCat['dec'][obsIndex] * conv,
318 fgcmStarObservationCat['instMag'][obsIndex],
319 fgcmStarObservationCat['instMagErr'][obsIndex],
320 fgcmExpInfo['FILTERNAME'][visitIndex],
321 fgcmStarIdCat['fgcm_id'][:],
322 fgcmStarIdCat['ra'][:],
323 fgcmStarIdCat['dec'][:],
324 fgcmStarIdCat['obsArrIndex'][:],
325 fgcmStarIdCat['nObs'][:],
326 obsX=fgcmStarObservationCat['x'][obsIndex],
327 obsY=fgcmStarObservationCat['y'][obsIndex],
328 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex],
329 refID=refId,
330 refMag=refMag,
331 refMagErr=refMagErr,
332 flagID=None,
333 flagFlag=None,
334 computeNobs=True)
336 # Clear out some memory
337 del fgcmStarIdCat
338 del fgcmStarIndicesCat
339 del fgcmRefCat
341 fgcmFitCycle.setLUT(fgcmLut)
342 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
344 converged = False
345 cycleNumber = 0
347 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
348 previousParInfo = None
349 previousParams = None
350 previousSuperStar = None
352 while (not converged and cycleNumber < self.config.maxFitCycles):
354 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
356 if cycleNumber > 0:
357 # Use parameters from previous cycle
358 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
359 fgcmExpInfo,
360 previousParInfo,
361 previousParams,
362 previousSuperStar)
363 # We need to reset the star magnitudes and errors for the next
364 # cycle
365 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
366 fgcmStarObservationCat['instMagErr'][obsIndex])
367 fgcmFitCycle.initialCycle = False
369 fgcmFitCycle.setPars(fgcmPars)
370 fgcmFitCycle.finishSetup()
372 fgcmFitCycle.run()
374 # Grab the parameters for the next cycle
375 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
376 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
378 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber))
379 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
380 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
381 continue
382 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
383 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
385 # Check for convergence
386 if np.alltrue((previousReservedRawRepeatability -
387 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) <
388 self.config.convergenceTolerance):
389 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber))
390 converged = True
391 else:
392 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
393 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
394 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False
395 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
396 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
397 self.log.info("Setting exposure gray photometricity cuts to:")
398 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
399 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
400 continue
401 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
402 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut))
404 cycleNumber += 1
406 # Log warning if not converged
407 if not converged:
408 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
410 # Do final clean-up iteration
411 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
412 fgcmFitCycle.fgcmConfig.resetParameters = False
413 fgcmFitCycle.fgcmConfig.maxIter = 0
414 fgcmFitCycle.fgcmConfig.outputZeropoints = True
415 fgcmFitCycle.fgcmConfig.outputStandards = True
416 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
417 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
418 fgcmFitCycle.initialCycle = False
420 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
421 fgcmExpInfo,
422 previousParInfo,
423 previousParams,
424 previousSuperStar)
425 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
426 fgcmStarObservationCat['instMagErr'][obsIndex])
427 fgcmFitCycle.setPars(fgcmPars)
428 fgcmFitCycle.finishSetup()
430 self.log.info("Running final clean-up fit cycle...")
431 fgcmFitCycle.run()
433 self.log.info("Raw repeatability after clean-up cycle is:")
434 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
435 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
436 continue
437 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
438 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
440 # Do the outputs. Need to keep track of tract.
442 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
443 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
445 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
446 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
448 atmSchema = makeAtmSchema()
449 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
451 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
452 stdSchema = makeStdSchema(len(goodBands))
453 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
455 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract,
456 visitCat,
457 zptCat, atmCat, stdCat,
458 self.config.fgcmBuildStars,
459 self.config.fgcmFitCycle)
460 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
462 return outStruct