Coverage for python/lsst/fgcmcal/fgcmCalibrateTract.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Run fgcmcal on a single tract.
25"""
27import sys
28import traceback
30import numpy as np
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34from lsst.jointcal.dataIds import PerTractCcdDataIdContainer
36from .fgcmBuildStars import FgcmBuildStarsTask
37from .fgcmFitCycle import FgcmFitCycleConfig
38from .fgcmOutputProducts import FgcmOutputProductsTask
39from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
40from .utilities import computeCcdOffsets, computeApertureRadius, extractReferenceMags
41from .utilities import makeZptSchema, makeZptCat
42from .utilities import makeAtmSchema, makeAtmCat
43from .utilities import makeStdSchema, makeStdCat
45import fgcm
47__all__ = ['FgcmCalibrateTractConfig', 'FgcmCalibrateTractTask', 'FgcmCalibrateTractRunner']
50class FgcmCalibrateTractConfig(pexConfig.Config):
51 """Config for FgcmCalibrateTract"""
53 fgcmBuildStars = pexConfig.ConfigurableField(
54 target=FgcmBuildStarsTask,
55 doc="Task to load and match stars for fgcm",
56 )
57 fgcmFitCycle = pexConfig.ConfigField(
58 dtype=FgcmFitCycleConfig,
59 doc="Config to run a single fgcm fit cycle",
60 )
61 fgcmOutputProducts = pexConfig.ConfigurableField(
62 target=FgcmOutputProductsTask,
63 doc="Task to output fgcm products",
64 )
65 convergenceTolerance = pexConfig.Field(
66 doc="Tolerance on repeatability convergence (per band)",
67 dtype=float,
68 default=0.005,
69 )
70 maxFitCycles = pexConfig.Field(
71 doc="Maximum number of fit cycles",
72 dtype=int,
73 default=5,
74 )
75 doDebuggingPlots = pexConfig.Field(
76 doc="Make plots for debugging purposes?",
77 dtype=bool,
78 default=False,
79 )
81 def setDefaults(self):
82 pexConfig.Config.setDefaults(self)
84 self.fgcmBuildStars.checkAllCcds = False
85 self.fgcmBuildStars.densityCutMaxPerPixel = 10000
86 self.fgcmFitCycle.quietMode = True
87 self.fgcmOutputProducts.doReferenceCalibration = False
88 self.fgcmOutputProducts.doRefcatOutput = False
89 self.fgcmOutputProducts.cycleNumber = 0
90 self.fgcmOutputProducts.photoCal.applyColorTerms = False
92 def validate(self):
93 super().validate()
95 for band in self.fgcmFitCycle.bands:
96 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
97 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
98 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
99 self, msg)
102class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner):
103 """Subclass of TaskRunner for FgcmCalibrateTractTask
105 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
106 the butler (for persistence and mapper data), and a list of dataRefs
107 extracted from the command line. This task runs on a constrained set
108 of dataRefs, typically a single tract.
109 This class transforms the process arguments generated by the ArgumentParser
110 into the arguments expected by FgcmCalibrateTractTask.run().
111 This runner does not use any parallelization.
112 """
114 @staticmethod
115 def getTargetList(parsedCmd):
116 """
117 Return a list with one element: a tuple with the butler and
118 list of dataRefs
119 """
120 # we want to combine the butler with any (or no!) dataRefs
121 return [(parsedCmd.butler, parsedCmd.id.refList)]
123 def __call__(self, args):
124 """
125 Parameters
126 ----------
127 args: `tuple` with (butler, dataRefList)
129 Returns
130 -------
131 exitStatus: `list` with `lsst.pipe.base.Struct`
132 exitStatus (0: success; 1: failure)
133 """
134 butler, dataRefList = args
136 task = self.TaskClass(config=self.config, log=self.log)
138 exitStatus = 0
139 if self.doRaise:
140 results = task.runDataRef(butler, dataRefList)
141 else:
142 try:
143 results = task.runDataRef(butler, dataRefList)
144 except Exception as e:
145 exitStatus = 1
146 task.log.fatal("Failed: %s" % e)
147 if not isinstance(e, pipeBase.TaskError):
148 traceback.print_exc(file=sys.stderr)
150 task.writeMetadata(butler)
152 if self.doReturnResults:
153 return [pipeBase.Struct(exitStatus=exitStatus,
154 results=results)]
155 else:
156 return [pipeBase.Struct(exitStatus=exitStatus)]
158 def run(self, parsedCmd):
159 """
160 Run the task, with no multiprocessing
162 Parameters
163 ----------
164 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
165 """
167 resultList = []
169 if self.precall(parsedCmd):
170 targetList = self.getTargetList(parsedCmd)
171 resultList = self(targetList[0])
173 return resultList
176class FgcmCalibrateTractTask(pipeBase.CmdLineTask):
177 """
178 Calibrate a single tract using fgcmcal
179 """
181 ConfigClass = FgcmCalibrateTractConfig
182 RunnerClass = FgcmCalibrateTractRunner
183 _DefaultName = "fgcmCalibrateTract"
185 def __init__(self, butler=None, **kwargs):
186 """
187 Instantiate an `FgcmCalibrateTractTask`.
189 Parameters
190 ----------
191 butler : `lsst.daf.persistence.Butler`
192 """
194 pipeBase.CmdLineTask.__init__(self, **kwargs)
196 @classmethod
197 def _makeArgumentParser(cls):
198 """Create an argument parser"""
200 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
201 parser.add_id_argument("--id", "calexp", help="Data ID, e.g. --id visit=6789",
202 ContainerClass=PerTractCcdDataIdContainer)
204 return parser
206 # no saving of metadata for now
207 def _getMetadataName(self):
208 return None
210 @pipeBase.timeMethod
211 def runDataRef(self, butler, dataRefs):
212 """
213 Run full FGCM calibration on a single tract, including building star list,
214 fitting multiple cycles, and making outputs.
216 Parameters
217 ----------
218 butler: `lsst.daf.persistence.Butler`
219 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
220 Data references for the input visits.
221 If this is an empty list, all visits with src catalogs in
222 the repository are used.
223 Only one individual dataRef from a visit need be specified
224 and the code will find the other source catalogs from
225 each visit.
227 Raises
228 ------
229 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
230 not True, or if fgcmLookUpTable is not available, or if
231 doSubtractLocalBackground is True and aperture radius cannot be
232 determined.
233 """
235 if not butler.datasetExists('fgcmLookUpTable'):
236 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable")
238 if not self.config.fgcmBuildStars.doReferenceMatches:
239 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
240 if self.config.fgcmBuildStars.checkAllCcds:
241 raise RuntimeError("Cannot run FgcmCalibrateTract with fgcmBuildStars.checkAllCcds set to True")
243 self.makeSubtask("fgcmBuildStars", butler=butler)
244 self.makeSubtask("fgcmOutputProducts", butler=butler)
246 # Compute the aperture radius if necessary. This is useful to do now before
247 # any heavy lifting has happened (fail early).
248 calibFluxApertureRadius = None
249 if self.config.fgcmBuildStars.doSubtractLocalBackground:
250 sourceSchema = butler.get('src_schema').schema
251 try:
252 calibFluxApertureRadius = computeApertureRadius(sourceSchema,
253 self.config.fgcmBuildStars.instFluxField)
254 except (RuntimeError, LookupError):
255 raise RuntimeError("Could not determine aperture radius from %s. "
256 "Cannot use doSubtractLocalBackground." %
257 (self.config.instFluxField))
259 # Run the build stars tasks
260 tract = dataRefs[0].dataId['tract']
261 self.log.info("Running on tract %d" % (tract))
263 # Note that we will need visitCat at the end of the procedure for the outputs
264 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs)
265 camera = butler.get('camera')
266 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs)
267 rad = calibFluxApertureRadius
268 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
269 visitCat,
270 calibFluxApertureRadius=rad)
272 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
273 self.fgcmBuildStars.fgcmMatchStars(butler,
274 visitCat,
275 fgcmStarObservationCat)
277 # Load the LUT
278 lutCat = butler.get('fgcmLookUpTable')
279 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
280 dict(self.config.fgcmFitCycle.filterMap))
281 del lutCat
283 # Translate the visit catalog into fgcm format
284 fgcmExpInfo = translateVisitCatalog(visitCat)
286 camera = butler.get('camera')
287 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera,
288 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
289 True, False, tract=tract)
290 # Turn off plotting in tract mode
291 configDict['doPlots'] = False
293 # Use the first orientation.
294 # TODO: DM-21215 will generalize to arbitrary camera orientations
295 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
296 del camera
298 # Set up the fit cycle task
300 noFitsDict = {'lutIndex': lutIndexVals,
301 'lutStd': lutStd,
302 'expInfo': fgcmExpInfo,
303 'ccdOffsets': ccdOffsets}
305 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
306 noFitsDict=noFitsDict, noOutput=True)
308 # We determine the conversion from the native units (typically radians) to
309 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
310 # numpy arrays rather than Angles, which would we approximately 600x slower.
311 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra'])
313 # To load the stars, we need an initial parameter object
314 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
315 fgcmLut,
316 fgcmExpInfo)
318 # Match star observations to visits
319 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
320 # actually be transferred into fgcm using the indexing below.
322 obsIndex = fgcmStarIndicesCat['obsIndex']
323 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'],
324 fgcmStarObservationCat['visit'][obsIndex])
326 refMag, refMagErr = extractReferenceMags(fgcmRefCat,
327 self.config.fgcmFitCycle.bands,
328 self.config.fgcmFitCycle.filterMap)
329 refId = fgcmRefCat['fgcm_id'][:]
331 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
332 fgcmStars.loadStars(fgcmPars,
333 fgcmStarObservationCat['visit'][obsIndex],
334 fgcmStarObservationCat['ccd'][obsIndex],
335 fgcmStarObservationCat['ra'][obsIndex] * conv,
336 fgcmStarObservationCat['dec'][obsIndex] * conv,
337 fgcmStarObservationCat['instMag'][obsIndex],
338 fgcmStarObservationCat['instMagErr'][obsIndex],
339 fgcmExpInfo['FILTERNAME'][visitIndex],
340 fgcmStarIdCat['fgcm_id'][:],
341 fgcmStarIdCat['ra'][:],
342 fgcmStarIdCat['dec'][:],
343 fgcmStarIdCat['obsArrIndex'][:],
344 fgcmStarIdCat['nObs'][:],
345 obsX=fgcmStarObservationCat['x'][obsIndex],
346 obsY=fgcmStarObservationCat['y'][obsIndex],
347 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex],
348 refID=refId,
349 refMag=refMag,
350 refMagErr=refMagErr,
351 flagID=None,
352 flagFlag=None,
353 computeNobs=True)
355 # Clear out some memory
356 del fgcmStarIdCat
357 del fgcmStarIndicesCat
358 del fgcmRefCat
360 fgcmFitCycle.setLUT(fgcmLut)
361 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
363 converged = False
364 cycleNumber = 0
366 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
367 previousParInfo = None
368 previousParams = None
369 previousSuperStar = None
371 while (not converged and cycleNumber < self.config.maxFitCycles):
373 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
375 if cycleNumber > 0:
376 # Use parameters from previous cycle
377 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
378 fgcmExpInfo,
379 previousParInfo,
380 previousParams,
381 previousSuperStar)
382 # We need to reset the star magnitudes and errors for the next
383 # cycle
384 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
385 fgcmStarObservationCat['instMagErr'][obsIndex])
386 fgcmFitCycle.initialCycle = False
388 fgcmFitCycle.setPars(fgcmPars)
389 fgcmFitCycle.finishSetup()
391 fgcmFitCycle.run()
393 # Grab the parameters for the next cycle
394 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
395 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
397 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber))
398 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
399 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
400 continue
401 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
402 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
404 # Check for convergence
405 if np.alltrue((previousReservedRawRepeatability -
406 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) <
407 self.config.convergenceTolerance):
408 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber))
409 converged = True
410 else:
411 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
412 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
413 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False
414 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
415 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
416 self.log.info("Setting exposure gray photometricity cuts to:")
417 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
418 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
419 continue
420 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
421 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut))
423 cycleNumber += 1
425 # Log warning if not converged
426 if not converged:
427 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
429 # Do final clean-up iteration
430 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
431 fgcmFitCycle.fgcmConfig.resetParameters = False
432 fgcmFitCycle.fgcmConfig.maxIter = 0
433 fgcmFitCycle.fgcmConfig.outputZeropoints = True
434 fgcmFitCycle.fgcmConfig.outputStandards = True
435 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
436 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
437 fgcmFitCycle.initialCycle = False
439 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
440 fgcmExpInfo,
441 previousParInfo,
442 previousParams,
443 previousSuperStar)
444 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
445 fgcmStarObservationCat['instMagErr'][obsIndex])
446 fgcmFitCycle.setPars(fgcmPars)
447 fgcmFitCycle.finishSetup()
449 self.log.info("Running final clean-up fit cycle...")
450 fgcmFitCycle.run()
452 self.log.info("Raw repeatability after clean-up cycle is:")
453 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
454 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
455 continue
456 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
457 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
459 # Do the outputs. Need to keep track of tract.
461 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
462 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
464 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
465 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
467 atmSchema = makeAtmSchema()
468 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
470 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
471 stdSchema = makeStdSchema(len(goodBands))
472 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
474 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract,
475 visitCat,
476 zptCat, atmCat, stdCat,
477 self.config.fgcmBuildStars,
478 self.config.fgcmFitCycle)
479 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
481 return outStruct