23 """Run fgcmcal on a single tract.
32 import lsst.pex.config
as pexConfig
33 import lsst.pipe.base
as pipeBase
34 from lsst.jointcal.dataIds
import PerTractCcdDataIdContainer
36 from .fgcmBuildStars
import FgcmBuildStarsTask
37 from .fgcmFitCycle
import FgcmFitCycleConfig
38 from .fgcmOutputProducts
import FgcmOutputProductsTask
39 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
40 from .utilities
import computeCcdOffsets, computeApertureRadius, extractReferenceMags
41 from .utilities
import makeZptSchema, makeZptCat
42 from .utilities
import makeAtmSchema, makeAtmCat
43 from .utilities
import makeStdSchema, makeStdCat
47 __all__ = [
'FgcmCalibrateTractConfig',
'FgcmCalibrateTractTask',
'FgcmCalibrateTractRunner']
51 """Config for FgcmCalibrateTract"""
53 fgcmBuildStars = pexConfig.ConfigurableField(
54 target=FgcmBuildStarsTask,
55 doc=
"Task to load and match stars for fgcm",
57 fgcmFitCycle = pexConfig.ConfigField(
58 dtype=FgcmFitCycleConfig,
59 doc=
"Config to run a single fgcm fit cycle",
61 fgcmOutputProducts = pexConfig.ConfigurableField(
62 target=FgcmOutputProductsTask,
63 doc=
"Task to output fgcm products",
65 convergenceTolerance = pexConfig.Field(
66 doc=
"Tolerance on repeatability convergence (per band)",
70 maxFitCycles = pexConfig.Field(
71 doc=
"Maximum number of fit cycles",
75 doDebuggingPlots = pexConfig.Field(
76 doc=
"Make plots for debugging purposes?",
82 pexConfig.Config.setDefaults(self)
96 if not self.
fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
97 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
98 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
103 """Subclass of TaskRunner for FgcmCalibrateTractTask
105 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
106 the butler (for persistence and mapper data), and a list of dataRefs
107 extracted from the command line. This task runs on a constrained set
108 of dataRefs, typically a single tract.
109 This class transforms the process arguments generated by the ArgumentParser
110 into the arguments expected by FgcmCalibrateTractTask.run().
111 This runner does not use any parallelization.
117 Return a list with one element: a tuple with the butler and
121 return [(parsedCmd.butler, parsedCmd.id.refList)]
127 args: `tuple` with (butler, dataRefList)
131 exitStatus: `list` with `lsst.pipe.base.Struct`
132 exitStatus (0: success; 1: failure)
134 butler, dataRefList = args
136 task = self.TaskClass(config=self.config, log=self.log)
140 results = task.runDataRef(butler, dataRefList)
143 results = task.runDataRef(butler, dataRefList)
144 except Exception
as e:
146 task.log.fatal(
"Failed: %s" % e)
147 if not isinstance(e, pipeBase.TaskError):
148 traceback.print_exc(file=sys.stderr)
150 task.writeMetadata(butler)
152 if self.doReturnResults:
153 return [pipeBase.Struct(exitStatus=exitStatus,
156 return [pipeBase.Struct(exitStatus=exitStatus)]
160 Run the task, with no multiprocessing
164 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
169 if self.precall(parsedCmd):
171 resultList = self(targetList[0])
178 Calibrate a single tract using fgcmcal
181 ConfigClass = FgcmCalibrateTractConfig
182 RunnerClass = FgcmCalibrateTractRunner
183 _DefaultName =
"fgcmCalibrateTract"
187 Instantiate an `FgcmCalibrateTractTask`.
191 butler : `lsst.daf.persistence.Butler`
194 pipeBase.CmdLineTask.__init__(self, **kwargs)
197 def _makeArgumentParser(cls):
198 """Create an argument parser"""
201 parser.add_id_argument(
"--id",
"calexp", help=
"Data ID, e.g. --id visit=6789",
202 ContainerClass=PerTractCcdDataIdContainer)
207 def _getMetadataName(self):
213 Run full FGCM calibration on a single tract, including building star list,
214 fitting multiple cycles, and making outputs.
218 butler: `lsst.daf.persistence.Butler`
219 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
220 Data references for the input visits.
221 If this is an empty list, all visits with src catalogs in
222 the repository are used.
223 Only one individual dataRef from a visit need be specified
224 and the code will find the other source catalogs from
229 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
230 not True, or if fgcmLookUpTable is not available, or if
231 doSubtractLocalBackground is True and aperture radius cannot be
235 if not butler.datasetExists(
'fgcmLookUpTable'):
236 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
238 if not self.config.fgcmBuildStars.doReferenceMatches:
239 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
240 if self.config.fgcmBuildStars.checkAllCcds:
241 raise RuntimeError(
"Cannot run FgcmCalibrateTract with fgcmBuildStars.checkAllCcds set to True")
243 self.makeSubtask(
"fgcmBuildStars", butler=butler)
244 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
248 calibFluxApertureRadius =
None
249 if self.config.fgcmBuildStars.doSubtractLocalBackground:
250 sourceSchema = butler.get(
'src_schema').schema
253 self.config.fgcmBuildStars.instFluxField)
254 except (RuntimeError, LookupError):
255 raise RuntimeError(
"Could not determine aperture radius from %s. "
256 "Cannot use doSubtractLocalBackground." %
257 (self.config.instFluxField))
260 tract = dataRefs[0].dataId[
'tract']
261 self.log.info(
"Running on tract %d" % (tract))
264 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs)
265 camera = butler.get(
'camera')
266 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs)
267 rad = calibFluxApertureRadius
268 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
270 calibFluxApertureRadius=rad)
272 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
273 self.fgcmBuildStars.fgcmMatchStars(butler,
275 fgcmStarObservationCat)
278 lutCat = butler.get(
'fgcmLookUpTable')
280 dict(self.config.fgcmFitCycle.filterMap))
286 camera = butler.get(
'camera')
287 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, camera,
288 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
289 True,
False, tract=tract)
291 configDict[
'doPlots'] =
False
300 noFitsDict = {
'lutIndex': lutIndexVals,
302 'expInfo': fgcmExpInfo,
303 'ccdOffsets': ccdOffsets}
305 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
306 noFitsDict=noFitsDict, noOutput=
True)
311 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
314 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
322 obsIndex = fgcmStarIndicesCat[
'obsIndex']
323 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
324 fgcmStarObservationCat[
'visit'][obsIndex])
327 self.config.fgcmFitCycle.bands,
328 self.config.fgcmFitCycle.filterMap)
329 refId = fgcmRefCat[
'fgcm_id'][:]
331 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
332 fgcmStars.loadStars(fgcmPars,
333 fgcmStarObservationCat[
'visit'][obsIndex],
334 fgcmStarObservationCat[
'ccd'][obsIndex],
335 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
336 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
337 fgcmStarObservationCat[
'instMag'][obsIndex],
338 fgcmStarObservationCat[
'instMagErr'][obsIndex],
339 fgcmExpInfo[
'FILTERNAME'][visitIndex],
340 fgcmStarIdCat[
'fgcm_id'][:],
341 fgcmStarIdCat[
'ra'][:],
342 fgcmStarIdCat[
'dec'][:],
343 fgcmStarIdCat[
'obsArrIndex'][:],
344 fgcmStarIdCat[
'nObs'][:],
345 obsX=fgcmStarObservationCat[
'x'][obsIndex],
346 obsY=fgcmStarObservationCat[
'y'][obsIndex],
347 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
357 del fgcmStarIndicesCat
360 fgcmFitCycle.setLUT(fgcmLut)
361 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
366 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
367 previousParInfo =
None
368 previousParams =
None
369 previousSuperStar =
None
371 while (
not converged
and cycleNumber < self.config.maxFitCycles):
373 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
377 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
384 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
385 fgcmStarObservationCat[
'instMagErr'][obsIndex])
386 fgcmFitCycle.initialCycle =
False
388 fgcmFitCycle.setPars(fgcmPars)
389 fgcmFitCycle.finishSetup()
394 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
395 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
397 self.log.info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
398 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
399 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
401 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
402 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
405 if np.alltrue((previousReservedRawRepeatability -
406 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) <
407 self.config.convergenceTolerance):
408 self.log.info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
411 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
412 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
413 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
414 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
415 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
416 self.log.info(
"Setting exposure gray photometricity cuts to:")
417 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
418 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
420 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
421 self.log.info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
427 self.log.warn(
"Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
430 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
431 fgcmFitCycle.fgcmConfig.resetParameters =
False
432 fgcmFitCycle.fgcmConfig.maxIter = 0
433 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
434 fgcmFitCycle.fgcmConfig.outputStandards =
True
435 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
436 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
437 fgcmFitCycle.initialCycle =
False
439 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
444 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
445 fgcmStarObservationCat[
'instMagErr'][obsIndex])
446 fgcmFitCycle.setPars(fgcmPars)
447 fgcmFitCycle.finishSetup()
449 self.log.info(
"Running final clean-up fit cycle...")
452 self.log.info(
"Raw repeatability after clean-up cycle is:")
453 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
454 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
456 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
457 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
461 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
462 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
465 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
468 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
470 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
472 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
474 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract,
476 zptCat, atmCat, stdCat,
477 self.config.fgcmBuildStars,
478 self.config.fgcmFitCycle)
479 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability