21 """Base class for running fgcmcal on a single tract using src tables
22 or sourceTable_visit tables.
31 import lsst.daf.persistence
as dafPersist
32 import lsst.pex.config
as pexConfig
33 import lsst.pipe.base
as pipeBase
35 from .fgcmBuildStars
import FgcmBuildStarsTask, FgcmBuildStarsConfig
36 from .fgcmFitCycle
import FgcmFitCycleConfig
37 from .fgcmOutputProducts
import FgcmOutputProductsTask
38 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
39 from .utilities
import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
40 from .utilities
import makeZptSchema, makeZptCat
41 from .utilities
import makeAtmSchema, makeAtmCat
42 from .utilities
import makeStdSchema, makeStdCat
46 __all__ = [
'FgcmCalibrateTractConfigBase',
'FgcmCalibrateTractBaseTask',
'FgcmCalibrateTractRunner']
50 """Config for FgcmCalibrateTract"""
52 fgcmBuildStars = pexConfig.ConfigurableField(
53 target=FgcmBuildStarsTask,
54 doc=
"Task to load and match stars for fgcm",
56 fgcmFitCycle = pexConfig.ConfigField(
57 dtype=FgcmFitCycleConfig,
58 doc=
"Config to run a single fgcm fit cycle",
60 fgcmOutputProducts = pexConfig.ConfigurableField(
61 target=FgcmOutputProductsTask,
62 doc=
"Task to output fgcm products",
64 convergenceTolerance = pexConfig.Field(
65 doc=
"Tolerance on repeatability convergence (per band)",
69 maxFitCycles = pexConfig.Field(
70 doc=
"Maximum number of fit cycles",
74 doDebuggingPlots = pexConfig.Field(
75 doc=
"Make plots for debugging purposes?",
81 pexConfig.Config.setDefaults(self)
93 if not self.
fgcmFitCyclefgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
94 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
95 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
100 """Subclass of TaskRunner for FgcmCalibrateTractTask
102 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
103 the butler (for persistence and mapper data), and a list of dataRefs
104 extracted from the command line. This task runs on a constrained set
105 of dataRefs, typically a single tract.
106 This class transforms the process arguments generated by the ArgumentParser
107 into the arguments expected by FgcmCalibrateTractTask.run().
108 This runner does not use any parallelization.
114 Return a list with one element: a tuple with the butler and
117 return [(parsedCmd.butler, parsedCmd.id.refList)]
123 args: `tuple` with (butler, dataRefList)
127 exitStatus: `list` with `lsst.pipe.base.Struct`
128 exitStatus (0: success; 1: failure)
129 May also contain results if `self.doReturnResults` is `True`.
131 butler, dataRefList = args
133 task = self.TaskClass(config=self.config, log=self.log)
137 results = task.runDataRef(butler, dataRefList)
140 results = task.runDataRef(butler, dataRefList)
141 except Exception
as e:
143 task.log.fatal(
"Failed: %s" % e)
144 if not isinstance(e, pipeBase.TaskError):
145 traceback.print_exc(file=sys.stderr)
147 task.writeMetadata(butler)
149 if self.doReturnResults:
150 return [pipeBase.Struct(exitStatus=exitStatus,
153 return [pipeBase.Struct(exitStatus=exitStatus)]
157 Run the task, with no multiprocessing
161 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
166 if self.precall(parsedCmd):
168 resultList = self(targetList[0])
174 """Base class to calibrate a single tract using fgcmcal
178 Instantiate an `FgcmCalibrateTractTask`.
182 butler : `lsst.daf.persistence.Butler`, optional
185 self.makeSubtask(
"fgcmBuildStars", butler=butler)
186 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
189 def _getMetadataName(self):
195 Run full FGCM calibration on a single tract, including building star list,
196 fitting multiple cycles, and making outputs.
200 butler: `lsst.daf.persistence.Butler`
201 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
202 Data references for the input visits.
203 These may be either per-ccd "src" or per-visit"sourceTable_visit"
208 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
209 not True, or if fgcmLookUpTable is not available, or if
210 doSubtractLocalBackground is True and aperture radius cannot be
213 datasetType = dataRefs[0].butlerSubset.datasetType
214 self.log.info(
"Running with %d %s dataRefs" % (len(dataRefs), datasetType))
216 if not butler.datasetExists(
'fgcmLookUpTable'):
217 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
219 if not self.config.fgcmBuildStars.doReferenceMatches:
220 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
221 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
222 if self.config.fgcmBuildStars.checkAllCcds:
223 raise RuntimeError(
"Cannot run FgcmCalibrateTract with "
224 "fgcmBuildStars.checkAllCcds set to True")
226 tract = int(dataRefs[0].dataId[
'tract'])
227 camera = butler.get(
'camera')
230 dataRefDict[
'camera'] = camera
231 dataRefDict[
'source_catalogs'] = dataRefs
232 dataRefDict[
'sourceSchema'] = butler.dataRef(
'src_schema')
233 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
235 struct = self.
runrun(dataRefDict, tract, butler=butler, returnCatalogs=
False)
237 visitDataRefName = self.config.fgcmBuildStars.visitDataRefName
238 ccdDataRefName = self.config.fgcmBuildStars.ccdDataRefName
240 if struct.photoCalibs
is not None:
241 self.log.info(
"Outputting photoCalib files.")
243 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
244 butler.put(photoCalib,
'fgcm_tract_photoCalib',
245 dataId={visitDataRefName: visit,
246 ccdDataRefName: detector,
247 'filter': physicalFilter,
250 self.log.info(
"Done outputting photoCalib files.")
252 if struct.atmospheres
is not None:
253 self.log.info(
"Outputting atmosphere files.")
254 for visit, atm
in struct.atmospheres:
255 butler.put(atm,
"transmission_atmosphere_fgcm_tract",
256 dataId={visitDataRefName: visit,
258 self.log.info(
"Done outputting atmosphere transmissions.")
260 return pipeBase.Struct(repeatability=struct.repeatability)
262 def run(self, dataRefDict, tract,
263 buildStarsRefObjLoader=None, returnCatalogs=True, butler=None):
264 """Run the calibrations for a single tract with fgcm.
269 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
270 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
271 dataRef dictionary with the following keys. Note that all
272 keys need not be set based on config parameters.
275 Camera object (`lsst.afw.cameraGeom.Camera`)
276 ``"source_catalogs"``
277 `list` of dataRefs for input source catalogs.
279 Schema for the source catalogs.
280 ``"fgcmLookUpTable"``
281 dataRef for the FGCM look-up table.
283 `list` of dataRefs for the input calexps (Gen3 only)
284 ``"fgcmPhotoCalibs"``
285 `dict` of output photoCalib dataRefs. Key is
286 (tract, visit, detector). (Gen3 only)
287 Present if doZeropointOutput is True.
288 ``"fgcmTransmissionAtmospheres"``
289 `dict` of output atmosphere transmission dataRefs.
290 Key is (tract, visit). (Gen3 only)
291 Present if doAtmosphereOutput is True.
294 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
295 Reference object loader object for fgcmBuildStars.
296 returnCatalogs : `bool`, optional
297 Return photoCalibs as per-visit exposure catalogs.
298 butler : `lsst.daf.persistence.Butler`, optional
299 Gen2 butler used for reference star outputs
303 outstruct : `lsst.pipe.base.Struct`
304 Output structure with keys:
306 offsets : `np.ndarray`
307 Final reference offsets, per band.
308 repeatability : `np.ndarray`
309 Raw fgcm repeatability for bright stars, per band.
310 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
311 Generator that returns (visit, transmissionCurve) tuples.
312 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
313 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
314 (returned if returnCatalogs is False).
315 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
316 Generator that returns (visit, exposureCatalog) tuples.
317 (returned if returnCatalogs is True).
319 self.log.info(
"Running on tract %d", (tract))
323 calibFluxApertureRadius =
None
324 if self.config.fgcmBuildStars.doSubtractLocalBackground:
326 field = self.config.fgcmBuildStars.instFluxField
330 raise RuntimeError(
"Could not determine aperture radius from %s. "
331 "Cannot use doSubtractLocalBackground." %
337 if isinstance(butler, dafPersist.Butler):
339 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefs(dataRefDict[
'camera'],
340 dataRefDict[
'source_catalogs'],
344 cdrd = dataRefDict[
'calexps']
345 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefs(dataRefDict[
'camera'],
346 dataRefDict[
'source_catalogs'],
347 calexpDataRefDict=cdrd)
348 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict[
'camera'], groupedDataRefs)
349 rad = calibFluxApertureRadius
350 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
352 dataRefDict[
'sourceSchema'],
353 dataRefDict[
'camera'],
354 calibFluxApertureRadius=rad)
356 if self.fgcmBuildStars.config.doReferenceMatches:
357 lutDataRef = dataRefDict[
'fgcmLookUpTable']
358 if buildStarsRefObjLoader
is not None:
359 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog",
360 refObjLoader=buildStarsRefObjLoader)
362 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog", butler=butler)
366 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
367 self.fgcmBuildStars.fgcmMatchStars(visitCat,
368 fgcmStarObservationCat,
369 lutDataRef=lutDataRef)
372 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
374 dict(self.config.fgcmFitCycle.physicalFilterMap))
380 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict[
'camera'],
381 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
382 True,
False, tract=tract)
384 configDict[
'doPlots'] =
False
392 noFitsDict = {
'lutIndex': lutIndexVals,
394 'expInfo': fgcmExpInfo,
395 'ccdOffsets': ccdOffsets}
397 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
398 noFitsDict=noFitsDict, noOutput=
True)
403 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
406 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
414 obsIndex = fgcmStarIndicesCat[
'obsIndex']
415 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
416 fgcmStarObservationCat[
'visit'][obsIndex])
419 self.config.fgcmFitCycle.bands,
420 self.config.fgcmFitCycle.physicalFilterMap)
421 refId = fgcmRefCat[
'fgcm_id'][:]
423 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
424 fgcmStars.loadStars(fgcmPars,
425 fgcmStarObservationCat[
'visit'][obsIndex],
426 fgcmStarObservationCat[
'ccd'][obsIndex],
427 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
428 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
429 fgcmStarObservationCat[
'instMag'][obsIndex],
430 fgcmStarObservationCat[
'instMagErr'][obsIndex],
431 fgcmExpInfo[
'FILTERNAME'][visitIndex],
432 fgcmStarIdCat[
'fgcm_id'][:],
433 fgcmStarIdCat[
'ra'][:],
434 fgcmStarIdCat[
'dec'][:],
435 fgcmStarIdCat[
'obsArrIndex'][:],
436 fgcmStarIdCat[
'nObs'][:],
437 obsX=fgcmStarObservationCat[
'x'][obsIndex],
438 obsY=fgcmStarObservationCat[
'y'][obsIndex],
439 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
440 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
450 del fgcmStarIndicesCat
453 fgcmFitCycle.setLUT(fgcmLut)
454 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
459 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
460 previousParInfo =
None
461 previousParams =
None
462 previousSuperStar =
None
464 while (
not converged
and cycleNumber < self.config.maxFitCycles):
466 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
470 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
477 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
478 fgcmStarObservationCat[
'instMagErr'][obsIndex])
479 fgcmFitCycle.initialCycle =
False
481 fgcmFitCycle.setPars(fgcmPars)
482 fgcmFitCycle.finishSetup()
487 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
488 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
490 self.log.info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
491 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
492 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
494 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
495 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
498 if np.alltrue((previousReservedRawRepeatability
499 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
500 < self.config.convergenceTolerance):
501 self.log.info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
504 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
505 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
506 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
507 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
508 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
509 self.log.info(
"Setting exposure gray photometricity cuts to:")
510 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
511 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
513 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
514 self.log.info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
520 self.log.warn(
"Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
523 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
524 fgcmFitCycle.fgcmConfig.resetParameters =
False
525 fgcmFitCycle.fgcmConfig.maxIter = 0
526 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
527 fgcmFitCycle.fgcmConfig.outputStandards =
True
528 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
529 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
530 fgcmFitCycle.initialCycle =
False
532 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
537 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
538 fgcmStarObservationCat[
'instMagErr'][obsIndex])
539 fgcmFitCycle.setPars(fgcmPars)
540 fgcmFitCycle.finishSetup()
542 self.log.info(
"Running final clean-up fit cycle...")
545 self.log.info(
"Raw repeatability after clean-up cycle is:")
546 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
547 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
549 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
550 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
554 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
555 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
558 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
561 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
563 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
565 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
567 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict,
570 zptCat, atmCat, stdCat,
571 self.config.fgcmBuildStars,
572 returnCatalogs=returnCatalogs,
575 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)
def runDataRef(self, butler, dataRefs)
def __init__(self, butler=None, **kwargs)
def getTargetList(parsedCmd)
def extractReferenceMags(refStars, bands, filterMap)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
def makeStdSchema(nBands)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
def makeAtmCat(atmSchema, atmStruct)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def computeCcdOffsets(camera, defaultOrientation)
def translateVisitCatalog(visitCat)