21"""Base class for running fgcmcal on a single tract using src tables
22or sourceTable_visit tables.
31import lsst.daf.persistence
as dafPersist
32import lsst.pex.config
as pexConfig
33import lsst.pipe.base
as pipeBase
34from lsst.utils.timer
import timeMethod
36from .fgcmBuildStars
import FgcmBuildStarsTask, FgcmBuildStarsConfig
37from .fgcmFitCycle
import FgcmFitCycleConfig
38from .fgcmOutputProducts
import FgcmOutputProductsTask
39from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
40from .utilities
import computeApertureRadiusFromDataRef, extractReferenceMags
41from .utilities
import makeZptSchema, makeZptCat
42from .utilities
import makeAtmSchema, makeAtmCat
43from .utilities
import makeStdSchema, makeStdCat
44from .focalPlaneProjector
import FocalPlaneProjector
48__all__ = [
'FgcmCalibrateTractConfigBase',
'FgcmCalibrateTractBaseTask',
'FgcmCalibrateTractRunner']
52 """Config for FgcmCalibrateTract"""
54 fgcmBuildStars = pexConfig.ConfigurableField(
55 target=FgcmBuildStarsTask,
56 doc=
"Task to load and match stars for fgcm",
58 fgcmFitCycle = pexConfig.ConfigField(
59 dtype=FgcmFitCycleConfig,
60 doc=
"Config to run a single fgcm fit cycle",
62 fgcmOutputProducts = pexConfig.ConfigurableField(
63 target=FgcmOutputProductsTask,
64 doc=
"Task to output fgcm products",
66 convergenceTolerance = pexConfig.Field(
67 doc=
"Tolerance on repeatability convergence (per band)",
71 maxFitCycles = pexConfig.Field(
72 doc=
"Maximum number of fit cycles",
76 doDebuggingPlots = pexConfig.Field(
77 doc=
"Make plots for debugging purposes?",
83 pexConfig.Config.setDefaults(self)
96 if not self.
fgcmFitCyclefgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
97 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
98 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
103 """Subclass of TaskRunner for FgcmCalibrateTractTask
105 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
106 the butler (
for persistence
and mapper data),
and a list of dataRefs
107 extracted
from the command line. This task runs on a constrained set
108 of dataRefs, typically a single tract.
109 This
class transforms the process arguments generated by the ArgumentParser
110 into the arguments expected by FgcmCalibrateTractTask.run().
111 This runner does
not use any parallelization.
117 Return a list with one element: a tuple
with the butler
and
120 return [(parsedCmd.butler, parsedCmd.id.refList)]
126 args: `tuple` with (butler, dataRefList)
130 exitStatus: `list`
with `lsst.pipe.base.Struct`
131 exitStatus (0: success; 1: failure)
132 May also contain results
if `self.doReturnResults`
is `
True`.
134 butler, dataRefList = args
136 task = self.TaskClass(config=self.config, log=self.log)
140 results = task.runDataRef(butler, dataRefList)
143 results = task.runDataRef(butler, dataRefList)
144 except Exception
as e:
146 task.log.fatal(
"Failed: %s" % e)
147 if not isinstance(e, pipeBase.TaskError):
148 traceback.print_exc(file=sys.stderr)
150 task.writeMetadata(butler)
152 if self.doReturnResults:
153 return [pipeBase.Struct(exitStatus=exitStatus,
156 return [pipeBase.Struct(exitStatus=exitStatus)]
160 Run the task, with no multiprocessing
164 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
169 if self.precall(parsedCmd):
171 resultList = self(targetList[0])
177 """Base class to calibrate a single tract using fgcmcal
179 def __init__(self, initInputs=None, butler=None, **kwargs):
181 Instantiate an `FgcmCalibrateTractTask`.
185 butler : `lsst.daf.persistence.Butler`, optional
188 self.makeSubtask("fgcmBuildStars", initInputs=initInputs, butler=butler)
189 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
192 def _getMetadataName(self):
198 Run full FGCM calibration on a single tract, including building star list,
199 fitting multiple cycles, and making outputs.
203 butler: `lsst.daf.persistence.Butler`
204 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
205 Data references
for the input visits.
206 These may be either per-ccd
"src" or per-visit
"sourceTable_visit"
211 RuntimeError: Raised
if `config.fgcmBuildStars.doReferenceMatches`
is
212 not True,
or if fgcmLookUpTable
is not available,
or if
213 doSubtractLocalBackground
is True and aperture radius cannot be
216 datasetType = dataRefs[0].butlerSubset.datasetType
217 self.log.info("Running with %d %s dataRefs" % (len(dataRefs), datasetType))
219 if not butler.datasetExists(
'fgcmLookUpTable'):
220 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
222 if not self.config.fgcmBuildStars.doReferenceMatches:
223 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
224 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
225 if self.config.fgcmBuildStars.checkAllCcds:
226 raise RuntimeError(
"Cannot run FgcmCalibrateTract with "
227 "fgcmBuildStars.checkAllCcds set to True")
229 tract = int(dataRefs[0].dataId[
'tract'])
230 camera = butler.get(
'camera')
233 dataRefDict[
'camera'] = camera
234 dataRefDict[
'source_catalogs'] = dataRefs
235 dataRefDict[
'sourceSchema'] = butler.get(
'src_schema', immediate=
True).schema
236 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
238 struct = self.
runrun(dataRefDict, tract, butler=butler, returnCatalogs=
False)
240 visitDataRefName = self.config.fgcmBuildStars.visitDataRefName
241 ccdDataRefName = self.config.fgcmBuildStars.ccdDataRefName
243 if struct.photoCalibs
is not None:
244 self.log.info(
"Outputting photoCalib files.")
246 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
247 butler.put(photoCalib,
'fgcm_tract_photoCalib',
248 dataId={visitDataRefName: visit,
249 ccdDataRefName: detector,
250 'filter': physicalFilter,
253 self.log.info(
"Done outputting photoCalib files.")
255 if struct.atmospheres
is not None:
256 self.log.info(
"Outputting atmosphere files.")
257 for visit, atm
in struct.atmospheres:
258 butler.put(atm,
"transmission_atmosphere_fgcm_tract",
259 dataId={visitDataRefName: visit,
261 self.log.info(
"Done outputting atmosphere transmissions.")
263 return pipeBase.Struct(repeatability=struct.repeatability)
265 def run(self, dataRefDict, tract,
266 buildStarsRefObjLoader=None, returnCatalogs=True, butler=None):
267 """Run the calibrations for a single tract with fgcm.
272 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
273 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
274 dataRef dictionary
with the following keys. Note that all
275 keys need
not be set based on config parameters.
278 Camera object (`lsst.afw.cameraGeom.Camera`)
279 ``
"source_catalogs"``
280 `list` of dataRefs
for input source catalogs.
282 Schema
for the source catalogs.
283 ``
"fgcmLookUpTable"``
284 dataRef
for the FGCM look-up table.
286 `list` of dataRefs
for the input calexps (Gen3 only)
287 ``
"fgcmPhotoCalibs"``
288 `dict` of output photoCalib dataRefs. Key
is
289 (tract, visit, detector). (Gen3 only)
290 Present
if doZeropointOutput
is True.
291 ``
"fgcmTransmissionAtmospheres"``
292 `dict` of output atmosphere transmission dataRefs.
293 Key
is (tract, visit). (Gen3 only)
294 Present
if doAtmosphereOutput
is True.
297 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
298 Reference object loader object
for fgcmBuildStars.
299 returnCatalogs : `bool`, optional
300 Return photoCalibs
as per-visit exposure catalogs.
301 butler : `lsst.daf.persistence.Butler`, optional
302 Gen2 butler used
for reference star outputs
306 outstruct : `lsst.pipe.base.Struct`
307 Output structure
with keys:
309 offsets : `np.ndarray`
310 Final reference offsets, per band.
311 repeatability : `np.ndarray`
312 Raw fgcm repeatability
for bright stars, per band.
313 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
314 Generator that returns (visit, transmissionCurve) tuples.
315 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
316 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
317 (returned
if returnCatalogs
is False).
318 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
319 Generator that returns (visit, exposureCatalog) tuples.
320 (returned
if returnCatalogs
is True).
322 self.log.info("Running on tract %d", (tract))
326 calibFluxApertureRadius =
None
327 if self.config.fgcmBuildStars.doSubtractLocalBackground:
329 field = self.config.fgcmBuildStars.instFluxField
333 raise RuntimeError(
"Could not determine aperture radius from %s. "
334 "Cannot use doSubtractLocalBackground." %
340 if isinstance(butler, dafPersist.Butler):
342 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefsGen2(butler, dataRefDict[
'camera'],
343 dataRefDict[
'source_catalogs'])
346 groupedDataRefs = self.fgcmBuildStars._groupDataRefs(dataRefDict[
'sourceTableDataRefDict'],
347 dataRefDict[
'visitSummaryDataRefDict'])
348 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict[
'camera'], groupedDataRefs)
349 rad = calibFluxApertureRadius
350 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
352 dataRefDict[
'sourceSchema'],
353 dataRefDict[
'camera'],
354 calibFluxApertureRadius=rad)
356 if self.fgcmBuildStars.config.doReferenceMatches:
357 lutDataRef = dataRefDict[
'fgcmLookUpTable']
358 if buildStarsRefObjLoader
is not None:
359 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog",
360 refObjLoader=buildStarsRefObjLoader)
362 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog", butler=butler)
366 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
367 self.fgcmBuildStars.fgcmMatchStars(visitCat,
368 fgcmStarObservationCat,
369 lutDataRef=lutDataRef)
372 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
374 dict(self.config.fgcmFitCycle.physicalFilterMap))
380 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict[
'camera'],
381 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
382 True,
False, lutIndexVals[0][
'FILTERNAMES'],
386 self.config.fgcmFitCycle.defaultCameraOrientation)
390 noFitsDict = {
'lutIndex': lutIndexVals,
392 'expInfo': fgcmExpInfo,
393 'focalPlaneProjector': focalPlaneProjector}
395 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
396 noFitsDict=noFitsDict, noOutput=
True)
401 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
404 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
412 obsIndex = fgcmStarIndicesCat[
'obsIndex']
413 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
414 fgcmStarObservationCat[
'visit'][obsIndex])
417 self.config.fgcmFitCycle.bands,
418 self.config.fgcmFitCycle.physicalFilterMap)
419 refId = fgcmRefCat[
'fgcm_id'][:]
421 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
422 fgcmStars.loadStars(fgcmPars,
423 fgcmStarObservationCat[
'visit'][obsIndex],
424 fgcmStarObservationCat[
'ccd'][obsIndex],
425 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
426 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
427 fgcmStarObservationCat[
'instMag'][obsIndex],
428 fgcmStarObservationCat[
'instMagErr'][obsIndex],
429 fgcmExpInfo[
'FILTERNAME'][visitIndex],
430 fgcmStarIdCat[
'fgcm_id'][:],
431 fgcmStarIdCat[
'ra'][:],
432 fgcmStarIdCat[
'dec'][:],
433 fgcmStarIdCat[
'obsArrIndex'][:],
434 fgcmStarIdCat[
'nObs'][:],
435 obsX=fgcmStarObservationCat[
'x'][obsIndex],
436 obsY=fgcmStarObservationCat[
'y'][obsIndex],
437 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
438 obsDeltaAper=fgcmStarObservationCat[
'deltaMagAper'][obsIndex],
439 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
449 del fgcmStarIndicesCat
452 fgcmFitCycle.setLUT(fgcmLut)
453 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
458 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
459 previousParInfo =
None
460 previousParams =
None
461 previousSuperStar =
None
463 while (
not converged
and cycleNumber < self.config.maxFitCycles):
465 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
469 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
476 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
477 fgcmStarObservationCat[
'instMagErr'][obsIndex])
478 fgcmFitCycle.initialCycle =
False
480 fgcmFitCycle.setPars(fgcmPars)
481 fgcmFitCycle.finishSetup()
486 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
487 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
489 self.log.info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
490 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
491 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
493 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
494 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
497 if np.alltrue((previousReservedRawRepeatability
498 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
499 < self.config.convergenceTolerance):
500 self.log.info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
503 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
504 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
505 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
506 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
507 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
508 self.log.info(
"Setting exposure gray photometricity cuts to:")
509 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
510 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
512 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
513 self.log.info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
519 self.log.warning(
"Maximum number of fit cycles exceeded (%d) without convergence.", cycleNumber)
522 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
523 fgcmFitCycle.fgcmConfig.resetParameters =
False
524 fgcmFitCycle.fgcmConfig.maxIter = 0
525 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
526 fgcmFitCycle.fgcmConfig.outputStandards =
True
527 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
528 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
529 fgcmFitCycle.initialCycle =
False
531 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
536 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
537 fgcmStarObservationCat[
'instMagErr'][obsIndex])
538 fgcmFitCycle.setPars(fgcmPars)
539 fgcmFitCycle.finishSetup()
541 self.log.info(
"Running final clean-up fit cycle...")
544 self.log.info(
"Raw repeatability after clean-up cycle is:")
545 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
546 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
548 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
549 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
553 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
554 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
557 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
560 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
562 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
564 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
566 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict,
569 zptCat, atmCat, stdCat,
570 self.config.fgcmBuildStars,
571 returnCatalogs=returnCatalogs,
574 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
576 fgcmFitCycle.freeSharedMemory()
def __init__(self, initInputs=None, butler=None, **kwargs)
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)
def runDataRef(self, butler, dataRefs)
def getTargetList(parsedCmd)
def extractReferenceMags(refStars, bands, filterMap)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
def makeStdSchema(nBands)
def makeAtmCat(atmSchema, atmStruct)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def translateVisitCatalog(visitCat)