21 """Base class for running fgcmcal on a single tract using src tables
22 or sourceTable_visit tables.
31 import lsst.daf.persistence
as dafPersist
32 import lsst.pex.config
as pexConfig
33 import lsst.pipe.base
as pipeBase
34 from lsst.utils.timer
import timeMethod
36 from .fgcmBuildStars
import FgcmBuildStarsTask, FgcmBuildStarsConfig
37 from .fgcmFitCycle
import FgcmFitCycleConfig
38 from .fgcmOutputProducts
import FgcmOutputProductsTask
39 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
40 from .utilities
import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
41 from .utilities
import makeZptSchema, makeZptCat
42 from .utilities
import makeAtmSchema, makeAtmCat
43 from .utilities
import makeStdSchema, makeStdCat
47 __all__ = [
'FgcmCalibrateTractConfigBase',
'FgcmCalibrateTractBaseTask',
'FgcmCalibrateTractRunner']
51 """Config for FgcmCalibrateTract"""
53 fgcmBuildStars = pexConfig.ConfigurableField(
54 target=FgcmBuildStarsTask,
55 doc=
"Task to load and match stars for fgcm",
57 fgcmFitCycle = pexConfig.ConfigField(
58 dtype=FgcmFitCycleConfig,
59 doc=
"Config to run a single fgcm fit cycle",
61 fgcmOutputProducts = pexConfig.ConfigurableField(
62 target=FgcmOutputProductsTask,
63 doc=
"Task to output fgcm products",
65 convergenceTolerance = pexConfig.Field(
66 doc=
"Tolerance on repeatability convergence (per band)",
70 maxFitCycles = pexConfig.Field(
71 doc=
"Maximum number of fit cycles",
75 doDebuggingPlots = pexConfig.Field(
76 doc=
"Make plots for debugging purposes?",
82 pexConfig.Config.setDefaults(self)
95 if not self.
fgcmFitCyclefgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
96 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
97 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
102 """Subclass of TaskRunner for FgcmCalibrateTractTask
104 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
105 the butler (for persistence and mapper data), and a list of dataRefs
106 extracted from the command line. This task runs on a constrained set
107 of dataRefs, typically a single tract.
108 This class transforms the process arguments generated by the ArgumentParser
109 into the arguments expected by FgcmCalibrateTractTask.run().
110 This runner does not use any parallelization.
116 Return a list with one element: a tuple with the butler and
119 return [(parsedCmd.butler, parsedCmd.id.refList)]
125 args: `tuple` with (butler, dataRefList)
129 exitStatus: `list` with `lsst.pipe.base.Struct`
130 exitStatus (0: success; 1: failure)
131 May also contain results if `self.doReturnResults` is `True`.
133 butler, dataRefList = args
135 task = self.TaskClass(config=self.config, log=self.log)
139 results = task.runDataRef(butler, dataRefList)
142 results = task.runDataRef(butler, dataRefList)
143 except Exception
as e:
145 task.log.fatal(
"Failed: %s" % e)
146 if not isinstance(e, pipeBase.TaskError):
147 traceback.print_exc(file=sys.stderr)
149 task.writeMetadata(butler)
151 if self.doReturnResults:
152 return [pipeBase.Struct(exitStatus=exitStatus,
155 return [pipeBase.Struct(exitStatus=exitStatus)]
159 Run the task, with no multiprocessing
163 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
168 if self.precall(parsedCmd):
170 resultList = self(targetList[0])
176 """Base class to calibrate a single tract using fgcmcal
178 def __init__(self, initInputs=None, butler=None, **kwargs):
180 Instantiate an `FgcmCalibrateTractTask`.
184 butler : `lsst.daf.persistence.Butler`, optional
187 self.makeSubtask(
"fgcmBuildStars", initInputs=initInputs, butler=butler)
188 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
191 def _getMetadataName(self):
197 Run full FGCM calibration on a single tract, including building star list,
198 fitting multiple cycles, and making outputs.
202 butler: `lsst.daf.persistence.Butler`
203 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
204 Data references for the input visits.
205 These may be either per-ccd "src" or per-visit"sourceTable_visit"
210 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
211 not True, or if fgcmLookUpTable is not available, or if
212 doSubtractLocalBackground is True and aperture radius cannot be
215 datasetType = dataRefs[0].butlerSubset.datasetType
216 self.log.info(
"Running with %d %s dataRefs" % (len(dataRefs), datasetType))
218 if not butler.datasetExists(
'fgcmLookUpTable'):
219 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
221 if not self.config.fgcmBuildStars.doReferenceMatches:
222 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
223 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
224 if self.config.fgcmBuildStars.checkAllCcds:
225 raise RuntimeError(
"Cannot run FgcmCalibrateTract with "
226 "fgcmBuildStars.checkAllCcds set to True")
228 tract = int(dataRefs[0].dataId[
'tract'])
229 camera = butler.get(
'camera')
232 dataRefDict[
'camera'] = camera
233 dataRefDict[
'source_catalogs'] = dataRefs
234 dataRefDict[
'sourceSchema'] = butler.get(
'src_schema', immediate=
True).schema
235 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
237 struct = self.
runrun(dataRefDict, tract, butler=butler, returnCatalogs=
False)
239 visitDataRefName = self.config.fgcmBuildStars.visitDataRefName
240 ccdDataRefName = self.config.fgcmBuildStars.ccdDataRefName
242 if struct.photoCalibs
is not None:
243 self.log.info(
"Outputting photoCalib files.")
245 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
246 butler.put(photoCalib,
'fgcm_tract_photoCalib',
247 dataId={visitDataRefName: visit,
248 ccdDataRefName: detector,
249 'filter': physicalFilter,
252 self.log.info(
"Done outputting photoCalib files.")
254 if struct.atmospheres
is not None:
255 self.log.info(
"Outputting atmosphere files.")
256 for visit, atm
in struct.atmospheres:
257 butler.put(atm,
"transmission_atmosphere_fgcm_tract",
258 dataId={visitDataRefName: visit,
260 self.log.info(
"Done outputting atmosphere transmissions.")
262 return pipeBase.Struct(repeatability=struct.repeatability)
264 def run(self, dataRefDict, tract,
265 buildStarsRefObjLoader=None, returnCatalogs=True, butler=None):
266 """Run the calibrations for a single tract with fgcm.
271 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
272 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
273 dataRef dictionary with the following keys. Note that all
274 keys need not be set based on config parameters.
277 Camera object (`lsst.afw.cameraGeom.Camera`)
278 ``"source_catalogs"``
279 `list` of dataRefs for input source catalogs.
281 Schema for the source catalogs.
282 ``"fgcmLookUpTable"``
283 dataRef for the FGCM look-up table.
285 `list` of dataRefs for the input calexps (Gen3 only)
286 ``"fgcmPhotoCalibs"``
287 `dict` of output photoCalib dataRefs. Key is
288 (tract, visit, detector). (Gen3 only)
289 Present if doZeropointOutput is True.
290 ``"fgcmTransmissionAtmospheres"``
291 `dict` of output atmosphere transmission dataRefs.
292 Key is (tract, visit). (Gen3 only)
293 Present if doAtmosphereOutput is True.
296 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
297 Reference object loader object for fgcmBuildStars.
298 returnCatalogs : `bool`, optional
299 Return photoCalibs as per-visit exposure catalogs.
300 butler : `lsst.daf.persistence.Butler`, optional
301 Gen2 butler used for reference star outputs
305 outstruct : `lsst.pipe.base.Struct`
306 Output structure with keys:
308 offsets : `np.ndarray`
309 Final reference offsets, per band.
310 repeatability : `np.ndarray`
311 Raw fgcm repeatability for bright stars, per band.
312 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
313 Generator that returns (visit, transmissionCurve) tuples.
314 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
315 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
316 (returned if returnCatalogs is False).
317 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
318 Generator that returns (visit, exposureCatalog) tuples.
319 (returned if returnCatalogs is True).
321 self.log.info(
"Running on tract %d", (tract))
325 calibFluxApertureRadius =
None
326 if self.config.fgcmBuildStars.doSubtractLocalBackground:
328 field = self.config.fgcmBuildStars.instFluxField
332 raise RuntimeError(
"Could not determine aperture radius from %s. "
333 "Cannot use doSubtractLocalBackground." %
339 if isinstance(butler, dafPersist.Butler):
341 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefsGen2(butler, dataRefDict[
'camera'],
342 dataRefDict[
'source_catalogs'])
345 groupedDataRefs = self.fgcmBuildStars._groupDataRefs(dataRefDict[
'sourceTableDataRefDict'],
346 dataRefDict[
'visitSummaryDataRefDict'])
347 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict[
'camera'], groupedDataRefs)
348 rad = calibFluxApertureRadius
349 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
351 dataRefDict[
'sourceSchema'],
352 dataRefDict[
'camera'],
353 calibFluxApertureRadius=rad)
355 if self.fgcmBuildStars.config.doReferenceMatches:
356 lutDataRef = dataRefDict[
'fgcmLookUpTable']
357 if buildStarsRefObjLoader
is not None:
358 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog",
359 refObjLoader=buildStarsRefObjLoader)
361 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog", butler=butler)
365 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
366 self.fgcmBuildStars.fgcmMatchStars(visitCat,
367 fgcmStarObservationCat,
368 lutDataRef=lutDataRef)
371 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
373 dict(self.config.fgcmFitCycle.physicalFilterMap))
379 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict[
'camera'],
380 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
381 True,
False, lutIndexVals[0][
'FILTERNAMES'],
390 noFitsDict = {
'lutIndex': lutIndexVals,
392 'expInfo': fgcmExpInfo,
393 'ccdOffsets': ccdOffsets}
395 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
396 noFitsDict=noFitsDict, noOutput=
True)
401 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
404 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
412 obsIndex = fgcmStarIndicesCat[
'obsIndex']
413 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
414 fgcmStarObservationCat[
'visit'][obsIndex])
417 self.config.fgcmFitCycle.bands,
418 self.config.fgcmFitCycle.physicalFilterMap)
419 refId = fgcmRefCat[
'fgcm_id'][:]
421 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
422 fgcmStars.loadStars(fgcmPars,
423 fgcmStarObservationCat[
'visit'][obsIndex],
424 fgcmStarObservationCat[
'ccd'][obsIndex],
425 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
426 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
427 fgcmStarObservationCat[
'instMag'][obsIndex],
428 fgcmStarObservationCat[
'instMagErr'][obsIndex],
429 fgcmExpInfo[
'FILTERNAME'][visitIndex],
430 fgcmStarIdCat[
'fgcm_id'][:],
431 fgcmStarIdCat[
'ra'][:],
432 fgcmStarIdCat[
'dec'][:],
433 fgcmStarIdCat[
'obsArrIndex'][:],
434 fgcmStarIdCat[
'nObs'][:],
435 obsX=fgcmStarObservationCat[
'x'][obsIndex],
436 obsY=fgcmStarObservationCat[
'y'][obsIndex],
437 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
438 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
448 del fgcmStarIndicesCat
451 fgcmFitCycle.setLUT(fgcmLut)
452 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
457 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
458 previousParInfo =
None
459 previousParams =
None
460 previousSuperStar =
None
462 while (
not converged
and cycleNumber < self.config.maxFitCycles):
464 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
468 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
475 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
476 fgcmStarObservationCat[
'instMagErr'][obsIndex])
477 fgcmFitCycle.initialCycle =
False
479 fgcmFitCycle.setPars(fgcmPars)
480 fgcmFitCycle.finishSetup()
485 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
486 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
488 self.log.info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
489 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
490 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
492 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
493 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
496 if np.alltrue((previousReservedRawRepeatability
497 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
498 < self.config.convergenceTolerance):
499 self.log.info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
502 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
503 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
504 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
505 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
506 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
507 self.log.info(
"Setting exposure gray photometricity cuts to:")
508 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
509 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
511 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
512 self.log.info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
518 self.log.warning(
"Maximum number of fit cycles exceeded (%d) without convergence.", cycleNumber)
521 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
522 fgcmFitCycle.fgcmConfig.resetParameters =
False
523 fgcmFitCycle.fgcmConfig.maxIter = 0
524 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
525 fgcmFitCycle.fgcmConfig.outputStandards =
True
526 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
527 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
528 fgcmFitCycle.initialCycle =
False
530 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
535 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
536 fgcmStarObservationCat[
'instMagErr'][obsIndex])
537 fgcmFitCycle.setPars(fgcmPars)
538 fgcmFitCycle.finishSetup()
540 self.log.info(
"Running final clean-up fit cycle...")
543 self.log.info(
"Raw repeatability after clean-up cycle is:")
544 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
545 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
547 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
548 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
552 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
553 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
556 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
559 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
561 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
563 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
565 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict,
568 zptCat, atmCat, stdCat,
569 self.config.fgcmBuildStars,
570 returnCatalogs=returnCatalogs,
573 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
575 fgcmFitCycle.freeSharedMemory()
def __init__(self, initInputs=None, butler=None, **kwargs)
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)
def runDataRef(self, butler, dataRefs)
def getTargetList(parsedCmd)
def extractReferenceMags(refStars, bands, filterMap)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
def makeStdSchema(nBands)
def makeAtmCat(atmSchema, atmStruct)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def computeCcdOffsets(camera, defaultOrientation)
def translateVisitCatalog(visitCat)