21 """Base class for running fgcmcal on a single tract using src tables
22 or sourceTable_visit tables.
31 import lsst.daf.persistence
as dafPersist
32 import lsst.pex.config
as pexConfig
33 import lsst.pipe.base
as pipeBase
35 from .fgcmBuildStars
import FgcmBuildStarsTask, FgcmBuildStarsConfig
36 from .fgcmFitCycle
import FgcmFitCycleConfig
37 from .fgcmOutputProducts
import FgcmOutputProductsTask
38 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
39 from .utilities
import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
40 from .utilities
import makeZptSchema, makeZptCat
41 from .utilities
import makeAtmSchema, makeAtmCat
42 from .utilities
import makeStdSchema, makeStdCat
46 __all__ = [
'FgcmCalibrateTractConfigBase',
'FgcmCalibrateTractBaseTask',
'FgcmCalibrateTractRunner']
50 """Config for FgcmCalibrateTract"""
52 fgcmBuildStars = pexConfig.ConfigurableField(
53 target=FgcmBuildStarsTask,
54 doc=
"Task to load and match stars for fgcm",
56 fgcmFitCycle = pexConfig.ConfigField(
57 dtype=FgcmFitCycleConfig,
58 doc=
"Config to run a single fgcm fit cycle",
60 fgcmOutputProducts = pexConfig.ConfigurableField(
61 target=FgcmOutputProductsTask,
62 doc=
"Task to output fgcm products",
64 convergenceTolerance = pexConfig.Field(
65 doc=
"Tolerance on repeatability convergence (per band)",
69 maxFitCycles = pexConfig.Field(
70 doc=
"Maximum number of fit cycles",
74 doDebuggingPlots = pexConfig.Field(
75 doc=
"Make plots for debugging purposes?",
81 pexConfig.Config.setDefaults(self)
93 if not self.
fgcmFitCyclefgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
94 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
95 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
100 """Subclass of TaskRunner for FgcmCalibrateTractTask
102 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is
103 the butler (for persistence and mapper data), and a list of dataRefs
104 extracted from the command line. This task runs on a constrained set
105 of dataRefs, typically a single tract.
106 This class transforms the process arguments generated by the ArgumentParser
107 into the arguments expected by FgcmCalibrateTractTask.run().
108 This runner does not use any parallelization.
114 Return a list with one element: a tuple with the butler and
117 return [(parsedCmd.butler, parsedCmd.id.refList)]
123 args: `tuple` with (butler, dataRefList)
127 exitStatus: `list` with `lsst.pipe.base.Struct`
128 exitStatus (0: success; 1: failure)
129 May also contain results if `self.doReturnResults` is `True`.
131 butler, dataRefList = args
133 task = self.TaskClass(config=self.config, log=self.log)
137 results = task.runDataRef(butler, dataRefList)
140 results = task.runDataRef(butler, dataRefList)
141 except Exception
as e:
143 task.log.fatal(
"Failed: %s" % e)
144 if not isinstance(e, pipeBase.TaskError):
145 traceback.print_exc(file=sys.stderr)
147 task.writeMetadata(butler)
149 if self.doReturnResults:
150 return [pipeBase.Struct(exitStatus=exitStatus,
153 return [pipeBase.Struct(exitStatus=exitStatus)]
157 Run the task, with no multiprocessing
161 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
166 if self.precall(parsedCmd):
168 resultList = self(targetList[0])
174 """Base class to calibrate a single tract using fgcmcal
178 Instantiate an `FgcmCalibrateTractTask`.
182 butler : `lsst.daf.persistence.Butler`, optional
185 self.makeSubtask(
"fgcmBuildStars", butler=butler)
186 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
189 def _getMetadataName(self):
195 Run full FGCM calibration on a single tract, including building star list,
196 fitting multiple cycles, and making outputs.
200 butler: `lsst.daf.persistence.Butler`
201 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
202 Data references for the input visits.
203 These may be either per-ccd "src" or per-visit"sourceTable_visit"
208 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is
209 not True, or if fgcmLookUpTable is not available, or if
210 doSubtractLocalBackground is True and aperture radius cannot be
213 datasetType = dataRefs[0].butlerSubset.datasetType
214 self.log.info(
"Running with %d %s dataRefs" % (len(dataRefs), datasetType))
216 if not butler.datasetExists(
'fgcmLookUpTable'):
217 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
219 if not self.config.fgcmBuildStars.doReferenceMatches:
220 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
221 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
222 if self.config.fgcmBuildStars.checkAllCcds:
223 raise RuntimeError(
"Cannot run FgcmCalibrateTract with "
224 "fgcmBuildStars.checkAllCcds set to True")
226 tract = int(dataRefs[0].dataId[
'tract'])
227 camera = butler.get(
'camera')
230 dataRefDict[
'camera'] = camera
231 dataRefDict[
'source_catalogs'] = dataRefs
232 dataRefDict[
'sourceSchema'] = butler.dataRef(
'src_schema')
233 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
235 struct = self.
runrun(dataRefDict, tract, butler=butler, returnCatalogs=
False)
237 visitDataRefName = self.config.fgcmBuildStars.visitDataRefName
238 ccdDataRefName = self.config.fgcmBuildStars.ccdDataRefName
240 if struct.photoCalibs
is not None:
241 self.log.info(
"Outputting photoCalib files.")
244 for visit, detector, filtername, photoCalib
in struct.photoCalibs:
245 if filtername
not in filterMapping:
248 dataId = {visitDataRefName: visit,
249 ccdDataRefName: detector}
250 dataRef = butler.dataRef(
'raw', dataId=dataId)
251 filterMapping[filtername] = dataRef.dataId[
'filter']
253 butler.put(photoCalib,
'fgcm_tract_photoCalib',
254 dataId={visitDataRefName: visit,
255 ccdDataRefName: detector,
256 'filter': filterMapping[filtername],
259 self.log.info(
"Done outputting photoCalib files.")
261 if struct.atmospheres
is not None:
262 self.log.info(
"Outputting atmosphere files.")
263 for visit, atm
in struct.atmospheres:
264 butler.put(atm,
"transmission_atmosphere_fgcm_tract",
265 dataId={visitDataRefName: visit,
267 self.log.info(
"Done outputting atmosphere transmissions.")
269 return pipeBase.Struct(repeatability=struct.repeatability)
271 def run(self, dataRefDict, tract,
272 buildStarsRefObjLoader=None, returnCatalogs=True, butler=None):
273 """Run the calibrations for a single tract with fgcm.
278 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
279 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
280 dataRef dictionary with the following keys. Note that all
281 keys need not be set based on config parameters.
284 Camera object (`lsst.afw.cameraGeom.Camera`)
285 ``"source_catalogs"``
286 `list` of dataRefs for input source catalogs.
288 Schema for the source catalogs.
289 ``"fgcmLookUpTable"``
290 dataRef for the FGCM look-up table.
292 `list` of dataRefs for the input calexps (Gen3 only)
293 ``"fgcmPhotoCalibs"``
294 `dict` of output photoCalib dataRefs. Key is
295 (tract, visit, detector). (Gen3 only)
296 Present if doZeropointOutput is True.
297 ``"fgcmTransmissionAtmospheres"``
298 `dict` of output atmosphere transmission dataRefs.
299 Key is (tract, visit). (Gen3 only)
300 Present if doAtmosphereOutput is True.
303 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
304 Reference object loader object for fgcmBuildStars.
305 returnCatalogs : `bool`, optional
306 Return photoCalibs as per-visit exposure catalogs.
307 butler : `lsst.daf.persistence.Butler`, optional
308 Gen2 butler used for reference star outputs
312 outstruct : `lsst.pipe.base.Struct`
313 Output structure with keys:
315 offsets : `np.ndarray`
316 Final reference offsets, per band.
317 repeatability : `np.ndarray`
318 Raw fgcm repeatability for bright stars, per band.
319 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
320 Generator that returns (visit, transmissionCurve) tuples.
321 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
322 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
323 (returned if returnCatalogs is False).
324 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
325 Generator that returns (visit, exposureCatalog) tuples.
326 (returned if returnCatalogs is True).
328 self.log.info(
"Running on tract %d", (tract))
332 calibFluxApertureRadius =
None
333 if self.config.fgcmBuildStars.doSubtractLocalBackground:
335 field = self.config.fgcmBuildStars.instFluxField
339 raise RuntimeError(
"Could not determine aperture radius from %s. "
340 "Cannot use doSubtractLocalBackground." %
346 if isinstance(butler, dafPersist.Butler):
348 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefs(dataRefDict[
'camera'],
349 dataRefDict[
'source_catalogs'],
353 cdrd = dataRefDict[
'calexps']
354 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefs(dataRefDict[
'camera'],
355 dataRefDict[
'source_catalogs'],
356 calexpDataRefDict=cdrd)
357 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict[
'camera'], groupedDataRefs)
358 rad = calibFluxApertureRadius
359 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
361 dataRefDict[
'sourceSchema'],
362 dataRefDict[
'camera'],
363 calibFluxApertureRadius=rad)
365 if self.fgcmBuildStars.config.doReferenceMatches:
366 lutDataRef = dataRefDict[
'fgcmLookUpTable']
367 if buildStarsRefObjLoader
is not None:
368 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog",
369 refObjLoader=buildStarsRefObjLoader)
371 self.fgcmBuildStars.makeSubtask(
"fgcmLoadReferenceCatalog", butler=butler)
375 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
376 self.fgcmBuildStars.fgcmMatchStars(visitCat,
377 fgcmStarObservationCat,
378 lutDataRef=lutDataRef)
381 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
383 dict(self.config.fgcmFitCycle.filterMap))
389 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict[
'camera'],
390 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
391 True,
False, tract=tract)
393 configDict[
'doPlots'] =
False
401 noFitsDict = {
'lutIndex': lutIndexVals,
403 'expInfo': fgcmExpInfo,
404 'ccdOffsets': ccdOffsets}
406 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
407 noFitsDict=noFitsDict, noOutput=
True)
412 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
415 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
423 obsIndex = fgcmStarIndicesCat[
'obsIndex']
424 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
425 fgcmStarObservationCat[
'visit'][obsIndex])
428 self.config.fgcmFitCycle.bands,
429 self.config.fgcmFitCycle.filterMap)
430 refId = fgcmRefCat[
'fgcm_id'][:]
432 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
433 fgcmStars.loadStars(fgcmPars,
434 fgcmStarObservationCat[
'visit'][obsIndex],
435 fgcmStarObservationCat[
'ccd'][obsIndex],
436 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
437 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
438 fgcmStarObservationCat[
'instMag'][obsIndex],
439 fgcmStarObservationCat[
'instMagErr'][obsIndex],
440 fgcmExpInfo[
'FILTERNAME'][visitIndex],
441 fgcmStarIdCat[
'fgcm_id'][:],
442 fgcmStarIdCat[
'ra'][:],
443 fgcmStarIdCat[
'dec'][:],
444 fgcmStarIdCat[
'obsArrIndex'][:],
445 fgcmStarIdCat[
'nObs'][:],
446 obsX=fgcmStarObservationCat[
'x'][obsIndex],
447 obsY=fgcmStarObservationCat[
'y'][obsIndex],
448 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
449 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
459 del fgcmStarIndicesCat
462 fgcmFitCycle.setLUT(fgcmLut)
463 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
468 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
469 previousParInfo =
None
470 previousParams =
None
471 previousSuperStar =
None
473 while (
not converged
and cycleNumber < self.config.maxFitCycles):
475 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
479 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
486 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
487 fgcmStarObservationCat[
'instMagErr'][obsIndex])
488 fgcmFitCycle.initialCycle =
False
490 fgcmFitCycle.setPars(fgcmPars)
491 fgcmFitCycle.finishSetup()
496 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
497 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
499 self.log.info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
500 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
501 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
503 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
504 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
507 if np.alltrue((previousReservedRawRepeatability
508 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
509 < self.config.convergenceTolerance):
510 self.log.info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
513 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
514 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
515 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False
516 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
517 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
518 self.log.info(
"Setting exposure gray photometricity cuts to:")
519 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
520 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
522 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
523 self.log.info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
529 self.log.warn(
"Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
532 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False
533 fgcmFitCycle.fgcmConfig.resetParameters =
False
534 fgcmFitCycle.fgcmConfig.maxIter = 0
535 fgcmFitCycle.fgcmConfig.outputZeropoints =
True
536 fgcmFitCycle.fgcmConfig.outputStandards =
True
537 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
538 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
539 fgcmFitCycle.initialCycle =
False
541 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
546 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
547 fgcmStarObservationCat[
'instMagErr'][obsIndex])
548 fgcmFitCycle.setPars(fgcmPars)
549 fgcmFitCycle.finishSetup()
551 self.log.info(
"Running final clean-up fit cycle...")
554 self.log.info(
"Raw repeatability after clean-up cycle is:")
555 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
556 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
558 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
559 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
563 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
564 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
567 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
570 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
572 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
574 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
576 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict,
579 zptCat, atmCat, stdCat,
580 self.config.fgcmBuildStars,
581 returnCatalogs=returnCatalogs,
584 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)
def runDataRef(self, butler, dataRefs)
def __init__(self, butler=None, **kwargs)
def getTargetList(parsedCmd)
def extractReferenceMags(refStars, bands, filterMap)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
def makeStdSchema(nBands)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
def translateFgcmLut(lutCat, filterMap)
def makeAtmCat(atmSchema, atmStruct)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def computeCcdOffsets(camera, defaultOrientation)
def translateVisitCatalog(visitCat)