21 """Base class for running fgcmcal on a single tract using src tables
22 or sourceTable_visit tables.
30 import lsst.pex.config
as pexConfig
31 import lsst.pipe.base
as pipeBase
33 from .fgcmBuildStars
import FgcmBuildStarsTask, FgcmBuildStarsConfig
34 from .fgcmFitCycle
import FgcmFitCycleConfig
35 from .fgcmOutputProducts
import FgcmOutputProductsTask
36 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
37 from .utilities
import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
38 from .utilities
import makeZptSchema, makeZptCat
39 from .utilities
import makeAtmSchema, makeAtmCat
40 from .utilities
import makeStdSchema, makeStdCat
44 __all__ = [
'FgcmCalibrateTractConfigBase',
'FgcmCalibrateTractBaseTask',
'FgcmCalibrateTractRunner']
48 """Config for FgcmCalibrateTract"""
50 fgcmBuildStars = pexConfig.ConfigurableField(
51 target=FgcmBuildStarsTask,
52 doc=
"Task to load and match stars for fgcm",
54 fgcmFitCycle = pexConfig.ConfigField(
55 dtype=FgcmFitCycleConfig,
56 doc=
"Config to run a single fgcm fit cycle",
58 fgcmOutputProducts = pexConfig.ConfigurableField(
59 target=FgcmOutputProductsTask,
60 doc=
"Task to output fgcm products",
62 convergenceTolerance = pexConfig.Field(
63 doc=
"Tolerance on repeatability convergence (per band)",
67 maxFitCycles = pexConfig.Field(
68 doc=
"Maximum number of fit cycles",
72 doDebuggingPlots = pexConfig.Field(
73 doc=
"Make plots for debugging purposes?",
79 pexConfig.Config.setDefaults(self)
91 if not self.
fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
92 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
93 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
98 """Subclass of TaskRunner for FgcmCalibrateTractTask
100 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
101 the butler (for persistence and mapper data), and a list of dataRefs
102 extracted from the command line. This task runs on a constrained set
103 of dataRefs, typically a single tract.
104 This class transforms the process arguments generated by the ArgumentParser
105 into the arguments expected by FgcmCalibrateTractTask.run().
106 This runner does not use any parallelization.
112 Return a list with one element: a tuple with the butler and
115 return [(parsedCmd.butler, parsedCmd.id.refList)]
121 args: `tuple` with (butler, dataRefList)
125 exitStatus: `list` with `lsst.pipe.base.Struct`
126 exitStatus (0: success; 1: failure)
127 May also contain results if `self.doReturnResults` is `True`.
129 butler, dataRefList = args
131 task = self.TaskClass(config=self.config, log=self.log)
135 results = task.runDataRef(butler, dataRefList)
138 results = task.runDataRef(butler, dataRefList)
139 except Exception
as e:
141 task.log.fatal(
"Failed: %s" % e)
142 if not isinstance(e, pipeBase.TaskError):
143 traceback.print_exc(file=sys.stderr)
145 task.writeMetadata(butler)
147 if self.doReturnResults:
148 return [pipeBase.Struct(exitStatus=exitStatus,
151 return [pipeBase.Struct(exitStatus=exitStatus)]
155 Run the task, with no multiprocessing
159 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
164 if self.precall(parsedCmd):
166 resultList = self(targetList[0])
173 Base class to calibrate a single tract using fgcmcal
177 Instantiate an `FgcmCalibrateTractTask`.
181 butler : `lsst.daf.persistence.Butler`
184 pipeBase.CmdLineTask.__init__(self, **kwargs)
187 def _getMetadataName(self):
193 Run full FGCM calibration on a single tract, including building star list,
194 fitting multiple cycles, and making outputs.
198 butler: `lsst.daf.persistence.Butler`
199 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
200 Data references for the input visits.
201 These may be either per-ccd "src" or per-visit"sourceTable_visit"
206 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
207 not True, or if fgcmLookUpTable is not available, or if
208 doSubtractLocalBackground is True and aperture radius cannot be
211 datasetType = dataRefs[0].butlerSubset.datasetType
212 self.log.info(
"Running with %d %s dataRefs" % (len(dataRefs), datasetType))
214 if not butler.datasetExists(
'fgcmLookUpTable'):
215 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
217 if not self.config.fgcmBuildStars.doReferenceMatches:
218 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
219 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
220 if self.config.fgcmBuildStars.checkAllCcds:
221 raise RuntimeError(
"Cannot run FgcmCalibrateTract with "
222 "fgcmBuildStars.checkAllCcds set to True")
224 self.makeSubtask(
"fgcmBuildStars", butler=butler)
225 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
229 calibFluxApertureRadius =
None
230 if self.config.fgcmBuildStars.doSubtractLocalBackground:
232 field = self.config.fgcmBuildStars.instFluxField
236 raise RuntimeError(
"Could not determine aperture radius from %s. "
237 "Cannot use doSubtractLocalBackground." %
241 tract = int(dataRefs[0].dataId[
'tract'])
242 self.log.info(
"Running on tract %d" % (tract))
245 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs)
246 camera = butler.get(
'camera')
247 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs)
248 rad = calibFluxApertureRadius
249 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
251 calibFluxApertureRadius=rad)
253 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
254 self.fgcmBuildStars.fgcmMatchStars(butler,
256 fgcmStarObservationCat)
259 lutCat = butler.get(
'fgcmLookUpTable')
261 dict(self.config.fgcmFitCycle.filterMap))
267 camera = butler.get(
'camera')
268 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, camera,
269 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
270 True,
False, tract=tract)
272 configDict[
'doPlots'] =
False
281 noFitsDict = {
'lutIndex': lutIndexVals,
283 'expInfo': fgcmExpInfo,
284 'ccdOffsets': ccdOffsets}
286 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
287 noFitsDict=noFitsDict, noOutput=
True)
292 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
295 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
303 obsIndex = fgcmStarIndicesCat[
'obsIndex']
304 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
305 fgcmStarObservationCat[
'visit'][obsIndex])
308 self.config.fgcmFitCycle.bands,
309 self.config.fgcmFitCycle.filterMap)
310 refId = fgcmRefCat[
'fgcm_id'][:]
312 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
313 fgcmStars.loadStars(fgcmPars,
314 fgcmStarObservationCat[
'visit'][obsIndex],
315 fgcmStarObservationCat[
'ccd'][obsIndex],
316 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
317 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
318 fgcmStarObservationCat[
'instMag'][obsIndex],
319 fgcmStarObservationCat[
'instMagErr'][obsIndex],
320 fgcmExpInfo[
'FILTERNAME'][visitIndex],
321 fgcmStarIdCat[
'fgcm_id'][:],
322 fgcmStarIdCat[
'ra'][:],
323 fgcmStarIdCat[
'dec'][:],
324 fgcmStarIdCat[
'obsArrIndex'][:],
325 fgcmStarIdCat[
'nObs'][:],
326 obsX=fgcmStarObservationCat[
'x'][obsIndex],
327 obsY=fgcmStarObservationCat[
'y'][obsIndex],
328 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
338 del fgcmStarIndicesCat
341 fgcmFitCycle.setLUT(fgcmLut)
342 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
347 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
348 previousParInfo =
None
349 previousParams =
None
350 previousSuperStar =
None
352 while (
not converged
and cycleNumber < self.config.maxFitCycles):
354 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
358 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
365 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
366 fgcmStarObservationCat[
'instMagErr'][obsIndex])
367 fgcmFitCycle.initialCycle =
False
369 fgcmFitCycle.setPars(fgcmPars)
370 fgcmFitCycle.finishSetup()
375 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
376 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
378 self.log.info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
379 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
380 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
382 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
383 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
386 if np.alltrue((previousReservedRawRepeatability -
387 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) <
388 self.config.convergenceTolerance):
389 self.log.info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
392 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
393 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
394 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
395 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
396 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
397 self.log.info(
"Setting exposure gray photometricity cuts to:")
398 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
399 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
401 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
402 self.log.info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
408 self.log.warn(
"Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
411 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
412 fgcmFitCycle.fgcmConfig.resetParameters =
False
413 fgcmFitCycle.fgcmConfig.maxIter = 0
414 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
415 fgcmFitCycle.fgcmConfig.outputStandards =
True
416 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
417 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
418 fgcmFitCycle.initialCycle =
False
420 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
425 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
426 fgcmStarObservationCat[
'instMagErr'][obsIndex])
427 fgcmFitCycle.setPars(fgcmPars)
428 fgcmFitCycle.finishSetup()
430 self.log.info(
"Running final clean-up fit cycle...")
433 self.log.info(
"Raw repeatability after clean-up cycle is:")
434 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
435 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
437 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
438 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
442 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
443 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
446 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
449 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
451 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
453 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
455 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract,
457 zptCat, atmCat, stdCat,
458 self.config.fgcmBuildStars,
459 self.config.fgcmFitCycle)
460 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability