21 """Base class for running fgcmcal on a single tract using src tables
22 or sourceTable_visit tables.
31 import lsst.daf.persistence
as dafPersist
32 import lsst.pex.config
as pexConfig
33 import lsst.pipe.base
as pipeBase
35 from .fgcmBuildStars
import FgcmBuildStarsTask, FgcmBuildStarsConfig
36 from .fgcmFitCycle
import FgcmFitCycleConfig
37 from .fgcmOutputProducts
import FgcmOutputProductsTask
38 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
39 from .utilities
import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
40 from .utilities
import makeZptSchema, makeZptCat
41 from .utilities
import makeAtmSchema, makeAtmCat
42 from .utilities
import makeStdSchema, makeStdCat
46 __all__ = [
'FgcmCalibrateTractConfigBase',
'FgcmCalibrateTractBaseTask',
'FgcmCalibrateTractRunner']
50 """Config for FgcmCalibrateTract"""
52 fgcmBuildStars = pexConfig.ConfigurableField(
53 target=FgcmBuildStarsTask,
54 doc=
"Task to load and match stars for fgcm",
56 fgcmFitCycle = pexConfig.ConfigField(
57 dtype=FgcmFitCycleConfig,
58 doc=
"Config to run a single fgcm fit cycle",
60 fgcmOutputProducts = pexConfig.ConfigurableField(
61 target=FgcmOutputProductsTask,
62 doc=
"Task to output fgcm products",
64 convergenceTolerance = pexConfig.Field(
65 doc=
"Tolerance on repeatability convergence (per band)",
69 maxFitCycles = pexConfig.Field(
70 doc=
"Maximum number of fit cycles",
74 doDebuggingPlots = pexConfig.Field(
75 doc=
"Make plots for debugging purposes?",
81 pexConfig.Config.setDefaults(self)
93 if not self.
fgcmFitCyclefgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
94 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
95 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
100 """Subclass of TaskRunner for FgcmCalibrateTractTask
102 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
103 the butler (for persistence and mapper data), and a list of dataRefs
104 extracted from the command line. This task runs on a constrained set
105 of dataRefs, typically a single tract.
106 This class transforms the process arguments generated by the ArgumentParser
107 into the arguments expected by FgcmCalibrateTractTask.run().
108 This runner does not use any parallelization.
114 Return a list with one element: a tuple with the butler and
117 return [(parsedCmd.butler, parsedCmd.id.refList)]
123 args: `tuple` with (butler, dataRefList)
127 exitStatus: `list` with `lsst.pipe.base.Struct`
128 exitStatus (0: success; 1: failure)
129 May also contain results if `self.doReturnResults` is `True`.
131 butler, dataRefList = args
133 task = self.TaskClass(config=self.config, log=self.log)
137 results = task.runDataRef(butler, dataRefList)
140 results = task.runDataRef(butler, dataRefList)
141 except Exception
as e:
143 task.log.fatal(
"Failed: %s" % e)
144 if not isinstance(e, pipeBase.TaskError):
145 traceback.print_exc(file=sys.stderr)
147 task.writeMetadata(butler)
149 if self.doReturnResults:
150 return [pipeBase.Struct(exitStatus=exitStatus,
153 return [pipeBase.Struct(exitStatus=exitStatus)]
157 Run the task, with no multiprocessing
161 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
166 if self.precall(parsedCmd):
168 resultList = self(targetList[0])
174 """Base class to calibrate a single tract using fgcmcal
178 Instantiate an `FgcmCalibrateTractTask`.
182 butler : `lsst.daf.persistence.Butler`, optional
185 self.makeSubtask(
"fgcmBuildStars", butler=butler)
186 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
189 def _getMetadataName(self):
195 Run full FGCM calibration on a single tract, including building star list,
196 fitting multiple cycles, and making outputs.
200 butler: `lsst.daf.persistence.Butler`
201 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
202 Data references for the input visits.
203 These may be either per-ccd "src" or per-visit"sourceTable_visit"
208 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
209 not True, or if fgcmLookUpTable is not available, or if
210 doSubtractLocalBackground is True and aperture radius cannot be
213 datasetType = dataRefs[0].butlerSubset.datasetType
214 self.log.info(
"Running with %d %s dataRefs" % (len(dataRefs), datasetType))
216 if not butler.datasetExists(
'fgcmLookUpTable'):
217 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
219 if not self.config.fgcmBuildStars.doReferenceMatches:
220 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
221 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
222 if self.config.fgcmBuildStars.checkAllCcds:
223 raise RuntimeError(
"Cannot run FgcmCalibrateTract with "
224 "fgcmBuildStars.checkAllCcds set to True")
226 tract = int(dataRefs[0].dataId[
'tract'])
227 camera = butler.get(
'camera')
230 dataRefDict[
'camera'] = camera
231 dataRefDict[
'source_catalogs'] = dataRefs
232 dataRefDict[
'sourceSchema'] = butler.dataRef(
'src_schema')
233 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
235 struct = self.
runrun(dataRefDict, tract, butler=butler, returnCatalogs=
False)
237 visitDataRefName = self.config.fgcmBuildStars.visitDataRefName
238 ccdDataRefName = self.config.fgcmBuildStars.ccdDataRefName
240 if struct.photoCalibs
is not None:
241 self.log.info(
"Outputting photoCalib files.")
243 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
244 butler.put(photoCalib,
'fgcm_tract_photoCalib',
245 dataId={visitDataRefName: visit,
246 ccdDataRefName: detector,
247 'filter': physicalFilter,
250 self.log.info(
"Done outputting photoCalib files.")
252 if struct.atmospheres
is not None:
253 self.log.info(
"Outputting atmosphere files.")
254 for visit, atm
in struct.atmospheres:
255 butler.put(atm,
"transmission_atmosphere_fgcm_tract",
256 dataId={visitDataRefName: visit,
258 self.log.info(
"Done outputting atmosphere transmissions.")
260 return pipeBase.Struct(repeatability=struct.repeatability)
262 def run(self, dataRefDict, tract,
263 buildStarsRefObjLoader=None, returnCatalogs=True, butler=None):
264 """Run the calibrations for a single tract with fgcm.
269 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
270 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
271 dataRef dictionary with the following keys. Note that all
272 keys need not be set based on config parameters.
275 Camera object (`lsst.afw.cameraGeom.Camera`)
276 ``"source_catalogs"``
277 `list` of dataRefs for input source catalogs.
279 Schema for the source catalogs.
280 ``"fgcmLookUpTable"``
281 dataRef for the FGCM look-up table.
283 `list` of dataRefs for the input calexps (Gen3 only)
284 ``"fgcmPhotoCalibs"``
285 `dict` of output photoCalib dataRefs. Key is
286 (tract, visit, detector). (Gen3 only)
287 Present if doZeropointOutput is True.
288 ``"fgcmTransmissionAtmospheres"``
289 `dict` of output atmosphere transmission dataRefs.
290 Key is (tract, visit). (Gen3 only)
291 Present if doAtmosphereOutput is True.
294 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
295 Reference object loader object for fgcmBuildStars.
296 returnCatalogs : `bool`, optional
297 Return photoCalibs as per-visit exposure catalogs.
298 butler : `lsst.daf.persistence.Butler`, optional
299 Gen2 butler used for reference star outputs
303 outstruct : `lsst.pipe.base.Struct`
304 Output structure with keys:
306 offsets : `np.ndarray`
307 Final reference offsets, per band.
308 repeatability : `np.ndarray`
309 Raw fgcm repeatability for bright stars, per band.
310 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
311 Generator that returns (visit, transmissionCurve) tuples.
312 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
313 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
314 (returned if returnCatalogs is False).
315 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
316 Generator that returns (visit, exposureCatalog) tuples.
317 (returned if returnCatalogs is True).
319 self.log.info(
"Running on tract %d", (tract))
323 calibFluxApertureRadius =
None
324 if self.config.fgcmBuildStars.doSubtractLocalBackground:
326 field = self.config.fgcmBuildStars.instFluxField
330 raise RuntimeError(
"Could not determine aperture radius from %s. "
331 "Cannot use doSubtractLocalBackground." %
337 if isinstance(butler, dafPersist.Butler):
339 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefsGen2(butler, dataRefDict[
'camera'],
340 dataRefDict[
'source_catalogs'])
343 groupedDataRefs = self.fgcmBuildStars._groupDataRefs(dataRefDict[
'sourceTableDataRefDict'],
344 dataRefDict[
'visitSummaryDataRefDict'])
345 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict[
'camera'], groupedDataRefs)
346 rad = calibFluxApertureRadius
347 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
349 dataRefDict[
'sourceSchema'],
350 dataRefDict[
'camera'],
351 calibFluxApertureRadius=rad)
353 if self.fgcmBuildStars.config.doReferenceMatches:
354 lutDataRef = dataRefDict[
'fgcmLookUpTable']
355 if buildStarsRefObjLoader
is not None:
356 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog",
357 refObjLoader=buildStarsRefObjLoader)
359 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog", butler=butler)
363 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
364 self.fgcmBuildStars.fgcmMatchStars(visitCat,
365 fgcmStarObservationCat,
366 lutDataRef=lutDataRef)
369 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
371 dict(self.config.fgcmFitCycle.physicalFilterMap))
377 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict[
'camera'],
378 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
379 True,
False, tract=tract)
381 configDict[
'doPlots'] =
False
389 noFitsDict = {
'lutIndex': lutIndexVals,
391 'expInfo': fgcmExpInfo,
392 'ccdOffsets': ccdOffsets}
394 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
395 noFitsDict=noFitsDict, noOutput=
True)
400 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
403 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
411 obsIndex = fgcmStarIndicesCat[
'obsIndex']
412 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
413 fgcmStarObservationCat[
'visit'][obsIndex])
416 self.config.fgcmFitCycle.bands,
417 self.config.fgcmFitCycle.physicalFilterMap)
418 refId = fgcmRefCat[
'fgcm_id'][:]
420 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
421 fgcmStars.loadStars(fgcmPars,
422 fgcmStarObservationCat[
'visit'][obsIndex],
423 fgcmStarObservationCat[
'ccd'][obsIndex],
424 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
425 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
426 fgcmStarObservationCat[
'instMag'][obsIndex],
427 fgcmStarObservationCat[
'instMagErr'][obsIndex],
428 fgcmExpInfo[
'FILTERNAME'][visitIndex],
429 fgcmStarIdCat[
'fgcm_id'][:],
430 fgcmStarIdCat[
'ra'][:],
431 fgcmStarIdCat[
'dec'][:],
432 fgcmStarIdCat[
'obsArrIndex'][:],
433 fgcmStarIdCat[
'nObs'][:],
434 obsX=fgcmStarObservationCat[
'x'][obsIndex],
435 obsY=fgcmStarObservationCat[
'y'][obsIndex],
436 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
437 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
447 del fgcmStarIndicesCat
450 fgcmFitCycle.setLUT(fgcmLut)
451 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
456 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
457 previousParInfo =
None
458 previousParams =
None
459 previousSuperStar =
None
461 while (
not converged
and cycleNumber < self.config.maxFitCycles):
463 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
467 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
474 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
475 fgcmStarObservationCat[
'instMagErr'][obsIndex])
476 fgcmFitCycle.initialCycle =
False
478 fgcmFitCycle.setPars(fgcmPars)
479 fgcmFitCycle.finishSetup()
484 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
485 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
487 self.log.info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
488 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
489 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
491 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
492 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
495 if np.alltrue((previousReservedRawRepeatability
496 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
497 < self.config.convergenceTolerance):
498 self.log.info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
501 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
502 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
503 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
504 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
505 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
506 self.log.info(
"Setting exposure gray photometricity cuts to:")
507 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
508 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
510 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
511 self.log.info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
517 self.log.warn(
"Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
520 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
521 fgcmFitCycle.fgcmConfig.resetParameters =
False
522 fgcmFitCycle.fgcmConfig.maxIter = 0
523 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
524 fgcmFitCycle.fgcmConfig.outputStandards =
True
525 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
526 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
527 fgcmFitCycle.initialCycle =
False
529 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
534 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
535 fgcmStarObservationCat[
'instMagErr'][obsIndex])
536 fgcmFitCycle.setPars(fgcmPars)
537 fgcmFitCycle.finishSetup()
539 self.log.info(
"Running final clean-up fit cycle...")
542 self.log.info(
"Raw repeatability after clean-up cycle is:")
543 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
544 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
546 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
547 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
551 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
552 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
555 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
558 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
560 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
562 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
564 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict,
567 zptCat, atmCat, stdCat,
568 self.config.fgcmBuildStars,
569 returnCatalogs=returnCatalogs,
572 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)
def runDataRef(self, butler, dataRefs)
def __init__(self, butler=None, **kwargs)
def getTargetList(parsedCmd)
def extractReferenceMags(refStars, bands, filterMap)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
def makeStdSchema(nBands)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
def makeAtmCat(atmSchema, atmStruct)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def computeCcdOffsets(camera, defaultOrientation)
def translateVisitCatalog(visitCat)