Coverage for python/lsst/fgcmcal/fgcmCalibrateTract.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Run fgcmcal on a single tract.
25"""
27import sys
28import traceback
30import numpy as np
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34from lsst.jointcal.dataIds import PerTractCcdDataIdContainer
36from .fgcmBuildStars import FgcmBuildStarsTask
37from .fgcmFitCycle import FgcmFitCycleConfig
38from .fgcmOutputProducts import FgcmOutputProductsTask
39from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
40from .utilities import computeCcdOffsets, computeApertureRadius, extractReferenceMags
41from .utilities import makeZptSchema, makeZptCat
42from .utilities import makeAtmSchema, makeAtmCat
43from .utilities import makeStdSchema, makeStdCat
45import fgcm
47__all__ = ['FgcmCalibrateTractConfig', 'FgcmCalibrateTractTask', 'FgcmCalibrateTractRunner']
50class FgcmCalibrateTractConfig(pexConfig.Config):
51 """Config for FgcmCalibrateTract"""
53 fgcmBuildStars = pexConfig.ConfigurableField(
54 target=FgcmBuildStarsTask,
55 doc="Task to load and match stars for fgcm",
56 )
57 fgcmFitCycle = pexConfig.ConfigField(
58 dtype=FgcmFitCycleConfig,
59 doc="Config to run a single fgcm fit cycle",
60 )
61 fgcmOutputProducts = pexConfig.ConfigurableField(
62 target=FgcmOutputProductsTask,
63 doc="Task to output fgcm products",
64 )
65 convergenceTolerance = pexConfig.Field(
66 doc="Tolerance on repeatability convergence (per band)",
67 dtype=float,
68 default=0.005,
69 )
70 maxFitCycles = pexConfig.Field(
71 doc="Maximum number of fit cycles",
72 dtype=int,
73 default=5,
74 )
75 doDebuggingPlots = pexConfig.Field(
76 doc="Make plots for debugging purposes?",
77 dtype=bool,
78 default=False,
79 )
81 def setDefaults(self):
82 pexConfig.Config.setDefaults(self)
84 self.fgcmBuildStars.checkAllCcds = False
85 self.fgcmBuildStars.densityCutMaxPerPixel = 10000
86 self.fgcmFitCycle.quietMode = True
87 self.fgcmOutputProducts.doReferenceCalibration = False
88 self.fgcmOutputProducts.doRefcatOutput = False
89 self.fgcmOutputProducts.cycleNumber = 0
90 self.fgcmOutputProducts.photoCal.applyColorTerms = False
92 def validate(self):
93 super().validate()
95 for band in self.fgcmFitCycle.bands:
96 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
97 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
98 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
99 self, msg)
102class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner):
103 """Subclass of TaskRunner for FgcmCalibrateTractTask
105 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
106 the butler (for persistence and mapper data), and a list of dataRefs
107 extracted from the command line. This task runs on a constrained set
108 of dataRefs, typically a single tract.
109 This class transforms the process arguments generated by the ArgumentParser
110 into the arguments expected by FgcmCalibrateTractTask.run().
111 This runner does not use any parallelization.
112 """
114 @staticmethod
115 def getTargetList(parsedCmd):
116 """
117 Return a list with one element: a tuple with the butler and
118 list of dataRefs
119 """
120 # we want to combine the butler with any (or no!) dataRefs
121 return [(parsedCmd.butler, parsedCmd.id.refList)]
123 def __call__(self, args):
124 """
125 Parameters
126 ----------
127 args: `tuple` with (butler, dataRefList)
129 Returns
130 -------
131 exitStatus: `list` with `lsst.pipe.base.Struct`
132 exitStatus (0: success; 1: failure)
133 """
134 butler, dataRefList = args
136 task = self.TaskClass(config=self.config, log=self.log)
138 exitStatus = 0
139 if self.doRaise:
140 results = task.runDataRef(butler, dataRefList)
141 else:
142 try:
143 results = task.runDataRef(butler, dataRefList)
144 except Exception as e:
145 exitStatus = 1
146 task.log.fatal("Failed: %s" % e)
147 if not isinstance(e, pipeBase.TaskError):
148 traceback.print_exc(file=sys.stderr)
150 task.writeMetadata(butler)
152 if self.doReturnResults:
153 return [pipeBase.Struct(exitStatus=exitStatus,
154 results=results)]
155 else:
156 return [pipeBase.Struct(exitStatus=exitStatus)]
158 def run(self, parsedCmd):
159 """
160 Run the task, with no multiprocessing
162 Parameters
163 ----------
164 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
165 """
167 resultList = []
169 if self.precall(parsedCmd):
170 targetList = self.getTargetList(parsedCmd)
171 resultList = self(targetList[0])
173 return resultList
176class FgcmCalibrateTractTask(pipeBase.CmdLineTask):
177 """
178 Calibrate a single tract using fgcmcal
179 """
181 ConfigClass = FgcmCalibrateTractConfig
182 RunnerClass = FgcmCalibrateTractRunner
183 _DefaultName = "fgcmCalibrateTract"
185 def __init__(self, butler=None, **kwargs):
186 """
187 Instantiate an `FgcmCalibrateTractTask`.
189 Parameters
190 ----------
191 butler : `lsst.daf.persistence.Butler`
192 """
194 pipeBase.CmdLineTask.__init__(self, **kwargs)
196 @classmethod
197 def _makeArgumentParser(cls):
198 """Create an argument parser"""
200 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
201 parser.add_id_argument("--id", "calexp", help="Data ID, e.g. --id visit=6789",
202 ContainerClass=PerTractCcdDataIdContainer)
204 return parser
206 # no saving of metadata for now
207 def _getMetadataName(self):
208 return None
210 @pipeBase.timeMethod
211 def runDataRef(self, butler, dataRefs):
212 """
213 Run full FGCM calibration on a single tract, including building star list,
214 fitting multiple cycles, and making outputs.
216 Parameters
217 ----------
218 butler: `lsst.daf.persistence.Butler`
219 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
220 Data references for the input visits.
221 If this is an empty list, all visits with src catalogs in
222 the repository are used.
223 Only one individual dataRef from a visit need be specified
224 and the code will find the other source catalogs from
225 each visit.
227 Raises
228 ------
229 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
230 not True, or if fgcmLookUpTable is not available, or if
231 doSubtractLocalBackground is True and aperture radius cannot be
232 determined.
233 """
235 if not butler.datasetExists('fgcmLookUpTable'):
236 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable")
238 if not self.config.fgcmBuildStars.doReferenceMatches:
239 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
240 if self.config.fgcmBuildStars.checkAllCcds:
241 raise RuntimeError("Cannot run FgcmCalibrateTract with fgcmBuildStars.checkAllCcds set to True")
243 self.makeSubtask("fgcmBuildStars", butler=butler)
244 self.makeSubtask("fgcmOutputProducts", butler=butler)
246 # Compute the aperture radius if necessary. This is useful to do now before
247 # any heavy lifting has happened (fail early).
248 calibFluxApertureRadius = None
249 if self.config.fgcmBuildStars.doSubtractLocalBackground:
250 sourceSchema = butler.get('src_schema').schema
251 try:
252 calibFluxApertureRadius = computeApertureRadius(sourceSchema,
253 self.config.fgcmBuildStars.instFluxField)
254 except (RuntimeError, LookupError):
255 raise RuntimeError("Could not determine aperture radius from %s. "
256 "Cannot use doSubtractLocalBackground." %
257 (self.config.instFluxField))
259 # Run the build stars tasks
260 tract = dataRefs[0].dataId['tract']
261 self.log.info("Running on tract %d" % (tract))
263 # Note that we will need visitCat at the end of the procedure for the outputs
264 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs)
265 camera = butler.get('camera')
266 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs)
267 rad = calibFluxApertureRadius
268 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
269 visitCat,
270 calibFluxApertureRadius=rad)
272 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
273 self.fgcmBuildStars.fgcmMatchStars(butler,
274 visitCat,
275 fgcmStarObservationCat)
277 # Load the LUT
278 lutCat = butler.get('fgcmLookUpTable')
279 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
280 dict(self.config.fgcmFitCycle.filterMap))
281 del lutCat
283 # Translate the visit catalog into fgcm format
284 fgcmExpInfo = translateVisitCatalog(visitCat)
286 camera = butler.get('camera')
287 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera,
288 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
289 True, False, tract=tract)
290 # Turn off plotting in tract mode
291 configDict['doPlots'] = False
293 # Use the first orientation.
294 # TODO: DM-21215 will generalize to arbitrary camera orientations
295 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
296 del camera
298 # Set up the fit cycle task
300 noFitsDict = {'lutIndex': lutIndexVals,
301 'lutStd': lutStd,
302 'expInfo': fgcmExpInfo,
303 'ccdOffsets': ccdOffsets}
305 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
306 noFitsDict=noFitsDict, noOutput=True)
308 # We determine the conversion from the native units (typically radians) to
309 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
310 # numpy arrays rather than Angles, which would we approximately 600x slower.
311 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra'])
313 # To load the stars, we need an initial parameter object
314 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
315 fgcmLut,
316 fgcmExpInfo)
318 # Match star observations to visits
319 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
320 # actually be transferred into fgcm using the indexing below.
322 obsIndex = fgcmStarIndicesCat['obsIndex']
323 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'],
324 fgcmStarObservationCat['visit'][obsIndex])
326 refMag, refMagErr = extractReferenceMags(fgcmRefCat,
327 self.config.fgcmFitCycle.bands,
328 self.config.fgcmFitCycle.filterMap)
329 refId = fgcmRefCat['fgcm_id'][:]
331 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
332 fgcmStars.loadStars(fgcmPars,
333 fgcmStarObservationCat['visit'][obsIndex],
334 fgcmStarObservationCat['ccd'][obsIndex],
335 fgcmStarObservationCat['ra'][obsIndex] * conv,
336 fgcmStarObservationCat['dec'][obsIndex] * conv,
337 fgcmStarObservationCat['instMag'][obsIndex],
338 fgcmStarObservationCat['instMagErr'][obsIndex],
339 fgcmExpInfo['FILTERNAME'][visitIndex],
340 fgcmStarIdCat['fgcm_id'][:],
341 fgcmStarIdCat['ra'][:],
342 fgcmStarIdCat['dec'][:],
343 fgcmStarIdCat['obsArrIndex'][:],
344 fgcmStarIdCat['nObs'][:],
345 obsX=fgcmStarObservationCat['x'][obsIndex],
346 obsY=fgcmStarObservationCat['y'][obsIndex],
347 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex],
348 refID=refId,
349 refMag=refMag,
350 refMagErr=refMagErr,
351 flagID=None,
352 flagFlag=None,
353 computeNobs=True)
355 fgcmFitCycle.setLUT(fgcmLut)
356 fgcmFitCycle.setStars(fgcmStars)
358 # Clear out some memory
359 # del fgcmStarObservationCat
360 del fgcmStarIdCat
361 del fgcmStarIndicesCat
362 del fgcmRefCat
364 converged = False
365 cycleNumber = 0
367 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
368 previousParInfo = None
369 previousParams = None
370 previousSuperStar = None
372 while (not converged and cycleNumber < self.config.maxFitCycles):
374 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
376 if cycleNumber > 0:
377 # Use parameters from previous cycle
378 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
379 fgcmExpInfo,
380 previousParInfo,
381 previousParams,
382 previousSuperStar)
383 # We need to reset the star magnitudes and errors for the next
384 # cycle
385 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
386 fgcmStarObservationCat['instMagErr'][obsIndex])
387 fgcmFitCycle.initialCycle = False
389 fgcmFitCycle.setPars(fgcmPars)
390 fgcmFitCycle.finishSetup()
392 fgcmFitCycle.run()
394 # Grab the parameters for the next cycle
395 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
396 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
398 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber))
399 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
400 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
401 continue
402 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
403 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
405 # Check for convergence
406 if np.alltrue((previousReservedRawRepeatability -
407 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) <
408 self.config.convergenceTolerance):
409 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber))
410 converged = True
411 else:
412 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
413 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
414 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False
415 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
416 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
417 self.log.info("Setting exposure gray photometricity cuts to:")
418 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
419 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
420 continue
421 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
422 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut))
424 cycleNumber += 1
426 # Log warning if not converged
427 if not converged:
428 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
430 # Do final clean-up iteration
431 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
432 fgcmFitCycle.fgcmConfig.resetParameters = False
433 fgcmFitCycle.fgcmConfig.maxIter = 0
434 fgcmFitCycle.fgcmConfig.outputZeropoints = True
435 fgcmFitCycle.fgcmConfig.outputStandards = True
436 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
437 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
438 fgcmFitCycle.initialCycle = False
440 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
441 fgcmExpInfo,
442 previousParInfo,
443 previousParams,
444 previousSuperStar)
445 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
446 fgcmStarObservationCat['instMagErr'][obsIndex])
447 fgcmFitCycle.setPars(fgcmPars)
448 fgcmFitCycle.finishSetup()
450 self.log.info("Running final clean-up fit cycle...")
451 fgcmFitCycle.run()
453 self.log.info("Raw repeatability after clean-up cycle is:")
454 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
455 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
456 continue
457 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
458 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
460 # Do the outputs. Need to keep track of tract.
462 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
463 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
465 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
466 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
468 atmSchema = makeAtmSchema()
469 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
471 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
472 stdSchema = makeStdSchema(len(goodBands))
473 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
475 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract,
476 visitCat,
477 zptCat, atmCat, stdCat,
478 self.config.fgcmBuildStars,
479 self.config.fgcmFitCycle)
480 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
482 return outStruct