23 """Make the final fgcmcal output products.
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
41 from astropy
import units
43 import lsst.daf.base
as dafBase
44 import lsst.pex.config
as pexConfig
45 import lsst.pipe.base
as pipeBase
46 from lsst.pipe.base
import connectionTypes
47 from lsst.afw.image
import TransmissionCurve
48 from lsst.meas.algorithms
import LoadIndexedReferenceObjectsTask
49 from lsst.meas.algorithms
import ReferenceObjectLoader
50 from lsst.pipe.tasks.photoCal
import PhotoCalTask
52 import lsst.afw.image
as afwImage
53 import lsst.afw.math
as afwMath
54 import lsst.afw.table
as afwTable
55 from lsst.meas.algorithms
import IndexerRegistry
56 from lsst.meas.algorithms
import DatasetConfig
57 from lsst.meas.algorithms.ingestIndexReferenceTask
import addRefCatMetadata
59 from .utilities
import computeApproxPixelAreaFields
60 from .utilities
import lookupStaticCalibrations
64 __all__ = [
'FgcmOutputProductsConfig',
'FgcmOutputProductsTask',
'FgcmOutputProductsRunner']
68 dimensions=(
"instrument",),
69 defaultTemplates={
"cycleNumber":
"0"}):
70 camera = connectionTypes.PrerequisiteInput(
71 doc=
"Camera instrument",
73 storageClass=
"Camera",
74 dimensions=(
"instrument",),
75 lookupFunction=lookupStaticCalibrations,
79 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
80 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
81 "chromatic corrections."),
82 name=
"fgcmLookUpTable",
83 storageClass=
"Catalog",
84 dimensions=(
"instrument",),
88 fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
89 doc=
"Catalog of visit information for fgcm",
90 name=
"fgcmVisitCatalog",
91 storageClass=
"Catalog",
92 dimensions=(
"instrument",),
96 fgcmStandardStars = connectionTypes.PrerequisiteInput(
97 doc=
"Catalog of standard star data from fgcm fit",
98 name=
"fgcmStandardStars{cycleNumber}",
99 storageClass=
"SimpleCatalog",
100 dimensions=(
"instrument",),
104 fgcmZeropoints = connectionTypes.PrerequisiteInput(
105 doc=
"Catalog of zeropoints from fgcm fit",
106 name=
"fgcmZeropoints{cycleNumber}",
107 storageClass=
"Catalog",
108 dimensions=(
"instrument",),
112 fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
113 doc=
"Catalog of atmosphere parameters from fgcm fit",
114 name=
"fgcmAtmosphereParameters{cycleNumber}",
115 storageClass=
"Catalog",
116 dimensions=(
"instrument",),
120 refCat = connectionTypes.PrerequisiteInput(
121 doc=
"Reference catalog to use for photometric calibration",
123 storageClass=
"SimpleCatalog",
124 dimensions=(
"skypix",),
129 fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
130 doc=
"Config used to build FGCM input stars",
131 name=
"fgcmBuildStarsTable_config",
132 storageClass=
"Config",
135 fgcmPhotoCalib = connectionTypes.Output(
136 doc=(
"Per-visit photometric calibrations derived from fgcm calibration. "
137 "These catalogs use detector id for the id and are sorted for "
138 "fast lookups of a detector."),
139 name=
"fgcmPhotoCalibCatalog",
140 storageClass=
"ExposureCatalog",
141 dimensions=(
"instrument",
"visit",),
145 fgcmTransmissionAtmosphere = connectionTypes.Output(
146 doc=
"Per-visit atmosphere transmission files produced from fgcm calibration",
147 name=
"transmission_atmosphere_fgcm",
148 storageClass=
"TransmissionCurve",
149 dimensions=(
"instrument",
154 fgcmOffsets = connectionTypes.Output(
155 doc=
"Per-band offsets computed from doReferenceCalibration",
156 name=
"fgcmReferenceCalibrationOffsets",
157 storageClass=
"Catalog",
158 dimensions=(
"instrument",),
162 def __init__(self, *, config=None):
163 super().__init__(config=config)
165 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
166 raise ValueError(
"cycleNumber must be of integer format")
167 if config.connections.refCat != config.refObjLoader.ref_dataset_name:
168 raise ValueError(
"connections.refCat must be the same as refObjLoader.ref_dataset_name")
170 if config.doRefcatOutput:
171 raise ValueError(
"FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
173 if not config.doReferenceCalibration:
174 self.prerequisiteInputs.remove(
"refCat")
175 if not config.doAtmosphereOutput:
176 self.prerequisiteInputs.remove(
"fgcmAtmosphereParameters")
177 if not config.doZeropointOutput:
178 self.prerequisiteInputs.remove(
"fgcmZeropoints")
179 if not config.doReferenceCalibration:
180 self.outputs.remove(
"fgcmOffsets")
183 class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
184 pipelineConnections=FgcmOutputProductsConnections):
185 """Config for FgcmOutputProductsTask"""
187 cycleNumber = pexConfig.Field(
188 doc=
"Final fit cycle from FGCM fit",
195 doReferenceCalibration = pexConfig.Field(
196 doc=(
"Transfer 'absolute' calibration from reference catalog? "
197 "This afterburner step is unnecessary if reference stars "
198 "were used in the full fit in FgcmFitCycleTask."),
202 doRefcatOutput = pexConfig.Field(
203 doc=
"Output standard stars in reference catalog format",
207 doAtmosphereOutput = pexConfig.Field(
208 doc=
"Output atmospheres in transmission_atmosphere_fgcm format",
212 doZeropointOutput = pexConfig.Field(
213 doc=
"Output zeropoints in fgcm_photoCalib format",
217 doComposeWcsJacobian = pexConfig.Field(
218 doc=
"Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
222 doApplyMeanChromaticCorrection = pexConfig.Field(
223 doc=
"Apply the mean chromatic correction to the zeropoints?",
227 refObjLoader = pexConfig.ConfigurableField(
228 target=LoadIndexedReferenceObjectsTask,
229 doc=
"reference object loader for 'absolute' photometric calibration",
231 photoCal = pexConfig.ConfigurableField(
233 doc=
"task to perform 'absolute' calibration",
235 referencePixelizationNside = pexConfig.Field(
236 doc=
"Healpix nside to pixelize catalog to compare to reference catalog",
240 referencePixelizationMinStars = pexConfig.Field(
241 doc=(
"Minimum number of stars per healpix pixel to select for comparison"
242 "to the specified reference catalog"),
246 referenceMinMatch = pexConfig.Field(
247 doc=
"Minimum number of stars matched to reference catalog to be used in statistics",
251 referencePixelizationNPixels = pexConfig.Field(
252 doc=(
"Number of healpix pixels to sample to do comparison. "
253 "Doing too many will take a long time and not yield any more "
254 "precise results because the final number is the median offset "
255 "(per band) from the set of pixels."),
259 datasetConfig = pexConfig.ConfigField(
261 doc=
"Configuration for writing/reading ingested catalog",
264 def setDefaults(self):
265 pexConfig.Config.setDefaults(self)
275 self.photoCal.applyColorTerms =
False
276 self.photoCal.fluxField =
'instFlux'
277 self.photoCal.magErrFloor = 0.003
278 self.photoCal.match.referenceSelection.doSignalToNoise =
True
279 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
280 self.photoCal.match.sourceSelection.doSignalToNoise =
True
281 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
282 self.photoCal.match.sourceSelection.signalToNoise.fluxField =
'instFlux'
283 self.photoCal.match.sourceSelection.signalToNoise.errField =
'instFluxErr'
284 self.photoCal.match.sourceSelection.doFlags =
True
285 self.photoCal.match.sourceSelection.flags.good = []
286 self.photoCal.match.sourceSelection.flags.bad = [
'flag_badStar']
287 self.photoCal.match.sourceSelection.doUnresolved =
False
288 self.datasetConfig.ref_dataset_name =
'fgcm_stars'
289 self.datasetConfig.format_version = 1
295 self.connections.cycleNumber = str(self.cycleNumber)
298 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
299 """Subclass of TaskRunner for fgcmOutputProductsTask
301 fgcmOutputProductsTask.run() takes one argument, the butler, and
302 does not run on any data in the repository.
303 This runner does not use any parallelization.
307 def getTargetList(parsedCmd):
309 Return a list with one element, the butler.
311 return [parsedCmd.butler]
313 def __call__(self, butler):
317 butler: `lsst.daf.persistence.Butler`
321 exitStatus: `list` with `pipeBase.Struct`
322 exitStatus (0: success; 1: failure)
323 if self.doReturnResults also
324 results (`np.array` with absolute zeropoint offsets)
326 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
330 results = task.runDataRef(butler)
333 results = task.runDataRef(butler)
334 except Exception
as e:
336 task.log.fatal(
"Failed: %s" % e)
337 if not isinstance(e, pipeBase.TaskError):
338 traceback.print_exc(file=sys.stderr)
340 task.writeMetadata(butler)
342 if self.doReturnResults:
344 return [pipeBase.Struct(exitStatus=exitStatus,
347 return [pipeBase.Struct(exitStatus=exitStatus)]
349 def run(self, parsedCmd):
351 Run the task, with no multiprocessing
355 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
360 if self.precall(parsedCmd):
361 targetList = self.getTargetList(parsedCmd)
363 resultList = self(targetList[0])
368 class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
370 Output products from FGCM global calibration.
373 ConfigClass = FgcmOutputProductsConfig
374 RunnerClass = FgcmOutputProductsRunner
375 _DefaultName =
"fgcmOutputProducts"
377 def __init__(self, butler=None, **kwargs):
378 super().__init__(**kwargs)
381 def _getMetadataName(self):
384 def runQuantum(self, butlerQC, inputRefs, outputRefs):
386 dataRefDict[
'camera'] = butlerQC.get(inputRefs.camera)
387 dataRefDict[
'fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
388 dataRefDict[
'fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
389 dataRefDict[
'fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
391 if self.config.doZeropointOutput:
392 dataRefDict[
'fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
393 photoCalibRefDict = {photoCalibRef.dataId.byName()[
'visit']:
394 photoCalibRef
for photoCalibRef
in outputRefs.fgcmPhotoCalib}
396 if self.config.doAtmosphereOutput:
397 dataRefDict[
'fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
398 atmRefDict = {atmRef.dataId.byName()[
'visit']: atmRef
for
399 atmRef
in outputRefs.fgcmTransmissionAtmosphere}
401 if self.config.doReferenceCalibration:
402 refConfig = self.config.refObjLoader
403 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
404 for ref
in inputRefs.refCat],
405 refCats=butlerQC.get(inputRefs.refCat),
409 self.refObjLoader =
None
411 dataRefDict[
'fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
413 fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
414 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
416 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
417 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
418 "in fgcmBuildStarsTask.")
419 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
420 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
422 struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=
True)
425 if struct.photoCalibCatalogs
is not None:
426 self.log.info(
"Outputting photoCalib catalogs.")
427 for visit, expCatalog
in struct.photoCalibCatalogs:
428 butlerQC.put(expCatalog, photoCalibRefDict[visit])
429 self.log.info(
"Done outputting photoCalib catalogs.")
432 if struct.atmospheres
is not None:
433 self.log.info(
"Outputting atmosphere transmission files.")
434 for visit, atm
in struct.atmospheres:
435 butlerQC.put(atm, atmRefDict[visit])
436 self.log.info(
"Done outputting atmosphere files.")
438 if self.config.doReferenceCalibration:
440 schema = afwTable.Schema()
441 schema.addField(
'offset', type=np.float64,
442 doc=
"Post-process calibration offset (mag)")
443 offsetCat = afwTable.BaseCatalog(schema)
444 offsetCat.resize(len(struct.offsets))
445 offsetCat[
'offset'][:] = struct.offsets
447 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
452 def runDataRef(self, butler):
454 Make FGCM output products for use in the stack
458 butler: `lsst.daf.persistence.Butler`
460 Final fit cycle number, override config.
464 offsets: `lsst.pipe.base.Struct`
465 A structure with array of zeropoint offsets
470 Raised if any one of the following is true:
472 - butler cannot find "fgcmBuildStars_config" or
473 "fgcmBuildStarsTable_config".
474 - butler cannot find "fgcmFitCycle_config".
475 - "fgcmFitCycle_config" does not refer to
476 `self.config.cycleNumber`.
477 - butler cannot find "fgcmAtmosphereParameters" and
478 `self.config.doAtmosphereOutput` is `True`.
479 - butler cannot find "fgcmStandardStars" and
480 `self.config.doReferenceCalibration` is `True` or
481 `self.config.doRefcatOutput` is `True`.
482 - butler cannot find "fgcmZeropoints" and
483 `self.config.doZeropointOutput` is `True`.
485 if self.config.doReferenceCalibration:
487 self.makeSubtask(
"refObjLoader", butler=butler)
491 if not butler.datasetExists(
'fgcmBuildStarsTable_config')
and \
492 not butler.datasetExists(
'fgcmBuildStars_config'):
493 raise RuntimeError(
"Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
494 "which is prereq for fgcmOutputProducts")
496 if butler.datasetExists(
'fgcmBuildStarsTable_config'):
497 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStarsTable_config')
499 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStars_config')
500 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
501 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
502 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
504 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
505 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
506 "in fgcmBuildStarsTask.")
508 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
509 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
512 if (self.config.doAtmosphereOutput
513 and not butler.datasetExists(
'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
514 raise RuntimeError(f
"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
516 if not butler.datasetExists(
'fgcmStandardStars',
517 fgcmcycle=self.config.cycleNumber):
518 raise RuntimeError(
"Standard stars are missing for cycle %d." %
519 (self.config.cycleNumber))
521 if (self.config.doZeropointOutput
522 and (
not butler.datasetExists(
'fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
523 raise RuntimeError(
"Zeropoints are missing for cycle %d." %
524 (self.config.cycleNumber))
528 dataRefDict[
'camera'] = butler.get(
'camera')
529 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
530 dataRefDict[
'fgcmVisitCatalog'] = butler.dataRef(
'fgcmVisitCatalog')
531 dataRefDict[
'fgcmStandardStars'] = butler.dataRef(
'fgcmStandardStars',
532 fgcmcycle=self.config.cycleNumber)
534 if self.config.doZeropointOutput:
535 dataRefDict[
'fgcmZeropoints'] = butler.dataRef(
'fgcmZeropoints',
536 fgcmcycle=self.config.cycleNumber)
537 if self.config.doAtmosphereOutput:
538 dataRefDict[
'fgcmAtmosphereParameters'] = butler.dataRef(
'fgcmAtmosphereParameters',
539 fgcmcycle=self.config.cycleNumber)
541 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=
False)
543 if struct.photoCalibs
is not None:
544 self.log.info(
"Outputting photoCalib files.")
546 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
547 butler.put(photoCalib,
'fgcm_photoCalib',
548 dataId={visitDataRefName: visit,
549 ccdDataRefName: detector,
550 'filter': physicalFilter})
552 self.log.info(
"Done outputting photoCalib files.")
554 if struct.atmospheres
is not None:
555 self.log.info(
"Outputting atmosphere transmission files.")
556 for visit, atm
in struct.atmospheres:
557 butler.put(atm,
"transmission_atmosphere_fgcm",
558 dataId={visitDataRefName: visit})
559 self.log.info(
"Done outputting atmosphere transmissions.")
561 return pipeBase.Struct(offsets=struct.offsets)
563 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
564 """Run the output products task.
569 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
570 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
571 dataRef dictionary with keys:
574 Camera object (`lsst.afw.cameraGeom.Camera`)
575 ``"fgcmLookUpTable"``
576 dataRef for the FGCM look-up table.
577 ``"fgcmVisitCatalog"``
578 dataRef for visit summary catalog.
579 ``"fgcmStandardStars"``
580 dataRef for the output standard star catalog.
582 dataRef for the zeropoint data catalog.
583 ``"fgcmAtmosphereParameters"``
584 dataRef for the atmosphere parameter catalog.
585 ``"fgcmBuildStarsTableConfig"``
586 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
587 physicalFilterMap : `dict`
588 Dictionary of mappings from physical filter to FGCM band.
589 returnCatalogs : `bool`, optional
590 Return photoCalibs as per-visit exposure catalogs.
591 butler : `lsst.daf.persistence.Butler`, optional
592 Gen2 butler used for reference star outputs
596 retStruct : `lsst.pipe.base.Struct`
597 Output structure with keys:
599 offsets : `np.ndarray`
600 Final reference offsets, per band.
601 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
602 Generator that returns (visit, transmissionCurve) tuples.
603 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
604 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
605 (returned if returnCatalogs is False).
606 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
607 Generator that returns (visit, exposureCatalog) tuples.
608 (returned if returnCatalogs is True).
610 stdCat = dataRefDict[
'fgcmStandardStars'].get()
611 md = stdCat.getMetadata()
612 bands = md.getArray(
'BANDS')
614 if self.config.doReferenceCalibration:
615 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
616 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands)
618 offsets = np.zeros(len(bands))
621 if self.config.doRefcatOutput
and butler
is not None:
622 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
626 if self.config.doZeropointOutput:
627 zptCat = dataRefDict[
'fgcmZeropoints'].get()
628 visitCat = dataRefDict[
'fgcmVisitCatalog'].get()
630 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
631 physicalFilterMap, returnCatalogs=returnCatalogs)
635 if self.config.doAtmosphereOutput:
636 atmCat = dataRefDict[
'fgcmAtmosphereParameters'].get()
637 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
641 retStruct = pipeBase.Struct(offsets=offsets,
644 retStruct.photoCalibCatalogs = pcgen
646 retStruct.photoCalibs = pcgen
650 def generateTractOutputProducts(self, dataRefDict, tract,
651 visitCat, zptCat, atmCat, stdCat,
652 fgcmBuildStarsConfig,
656 Generate the output products for a given tract, as specified in the config.
658 This method is here to have an alternate entry-point for
664 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
665 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
666 dataRef dictionary with keys:
669 Camera object (`lsst.afw.cameraGeom.Camera`)
670 ``"fgcmLookUpTable"``
671 dataRef for the FGCM look-up table.
674 visitCat : `lsst.afw.table.BaseCatalog`
675 FGCM visitCat from `FgcmBuildStarsTask`
676 zptCat : `lsst.afw.table.BaseCatalog`
677 FGCM zeropoint catalog from `FgcmFitCycleTask`
678 atmCat : `lsst.afw.table.BaseCatalog`
679 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
680 stdCat : `lsst.afw.table.SimpleCatalog`
681 FGCM standard star catalog from `FgcmFitCycleTask`
682 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
683 Configuration object from `FgcmBuildStarsTask`
684 returnCatalogs : `bool`, optional
685 Return photoCalibs as per-visit exposure catalogs.
686 butler: `lsst.daf.persistence.Butler`, optional
687 Gen2 butler used for reference star outputs
691 retStruct : `lsst.pipe.base.Struct`
692 Output structure with keys:
694 offsets : `np.ndarray`
695 Final reference offsets, per band.
696 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
697 Generator that returns (visit, transmissionCurve) tuples.
698 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
699 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
700 (returned if returnCatalogs is False).
701 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
702 Generator that returns (visit, exposureCatalog) tuples.
703 (returned if returnCatalogs is True).
705 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
707 md = stdCat.getMetadata()
708 bands = md.getArray(
'BANDS')
710 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
711 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
712 "in fgcmBuildStarsTask.")
714 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
715 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
717 if self.config.doReferenceCalibration:
718 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
719 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap)
721 offsets = np.zeros(len(bands))
723 if self.config.doRefcatOutput
and butler
is not None:
725 datasetConfig = copy.copy(self.config.datasetConfig)
726 datasetConfig.ref_dataset_name =
'%s_%d' % (self.config.datasetConfig.ref_dataset_name,
728 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
730 if self.config.doZeropointOutput:
731 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
732 physicalFilterMap, returnCatalogs=returnCatalogs)
736 if self.config.doAtmosphereOutput:
737 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
741 retStruct = pipeBase.Struct(offsets=offsets,
744 retStruct.photoCalibCatalogs = pcgen
746 retStruct.photoCalibs = pcgen
750 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
752 Compute offsets relative to a reference catalog.
754 This method splits the star catalog into healpix pixels
755 and computes the calibration transfer for a sample of
756 these pixels to approximate the 'absolute' calibration
757 values (on for each band) to apply to transfer the
762 stdCat : `lsst.afw.table.SimpleCatalog`
764 lutCat : `lsst.afw.table.SimpleCatalog`
766 physicalFilterMap : `dict`
767 Dictionary of mappings from physical filter to FGCM band.
768 bands : `list` [`str`]
769 List of band names from FGCM output
772 offsets : `numpy.array` of floats
773 Per band zeropoint offsets
779 minObs = stdCat[
'ngood'].min(axis=1)
781 goodStars = (minObs >= 1)
782 stdCat = stdCat[goodStars]
784 self.log.info(
"Found %d stars with at least 1 good observation in each band" %
791 lutPhysicalFilters = lutCat[0][
'physicalFilters'].split(
',')
792 lutStdPhysicalFilters = lutCat[0][
'stdPhysicalFilters'].split(
',')
793 physicalFilterMapBands = list(physicalFilterMap.values())
794 physicalFilterMapFilters = list(physicalFilterMap.keys())
798 physicalFilterMapIndex = physicalFilterMapBands.index(band)
799 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
801 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
802 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
803 filterLabels.append(afwImage.FilterLabel(band=band,
804 physical=stdPhysicalFilter))
813 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
814 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
815 sourceMapper.editOutputSchema().addField(
'instFlux', type=np.float64,
816 doc=
"instrumental flux (counts)")
817 sourceMapper.editOutputSchema().addField(
'instFluxErr', type=np.float64,
818 doc=
"instrumental flux error (counts)")
819 badStarKey = sourceMapper.editOutputSchema().addField(
'flag_badStar',
827 theta = np.pi/2. - stdCat[
'coord_dec']
828 phi = stdCat[
'coord_ra']
830 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
831 h, rev = esutil.stat.histogram(ipring, rev=
True)
833 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
835 self.log.info(
"Found %d pixels (nside=%d) with at least %d good stars" %
837 self.config.referencePixelizationNside,
838 self.config.referencePixelizationMinStars))
840 if gdpix.size < self.config.referencePixelizationNPixels:
841 self.log.warn(
"Found fewer good pixels (%d) than preferred in configuration (%d)" %
842 (gdpix.size, self.config.referencePixelizationNPixels))
845 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=
False)
847 results = np.zeros(gdpix.size, dtype=[(
'hpix',
'i4'),
848 (
'nstar',
'i4', len(bands)),
849 (
'nmatch',
'i4', len(bands)),
850 (
'zp',
'f4', len(bands)),
851 (
'zpErr',
'f4', len(bands))])
852 results[
'hpix'] = ipring[rev[rev[gdpix]]]
855 selected = np.zeros(len(stdCat), dtype=bool)
857 refFluxFields = [
None]*len(bands)
859 for p_index, pix
in enumerate(gdpix):
860 i1a = rev[rev[pix]: rev[pix + 1]]
868 for b_index, filterLabel
in enumerate(filterLabels):
869 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index,
871 selected, refFluxFields)
872 results[
'nstar'][p_index, b_index] = len(i1a)
873 results[
'nmatch'][p_index, b_index] = len(struct.arrays.refMag)
874 results[
'zp'][p_index, b_index] = struct.zp
875 results[
'zpErr'][p_index, b_index] = struct.sigma
878 offsets = np.zeros(len(bands))
880 for b_index, band
in enumerate(bands):
882 ok, = np.where(results[
'nmatch'][:, b_index] >= self.config.referenceMinMatch)
883 offsets[b_index] = np.median(results[
'zp'][ok, b_index])
886 madSigma = 1.4826*np.median(np.abs(results[
'zp'][ok, b_index] - offsets[b_index]))
887 self.log.info(
"Reference catalog offset for %s band: %.12f +/- %.12f",
888 band, offsets[b_index], madSigma)
892 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
893 b_index, filterLabel, stdCat, selected, refFluxFields):
895 Compute the zeropoint offset between the fgcm stdCat and the reference
896 stars for one pixel in one band
900 sourceMapper : `lsst.afw.table.SchemaMapper`
901 Mapper to go from stdCat to calibratable catalog
902 badStarKey : `lsst.afw.table.Key`
903 Key for the field with bad stars
905 Index of the band in the star catalog
906 filterLabel : `lsst.afw.image.FilterLabel`
907 filterLabel with band and physical filter
908 stdCat : `lsst.afw.table.SimpleCatalog`
910 selected : `numpy.array(dtype=bool)`
911 Boolean array of which stars are in the pixel
912 refFluxFields : `list`
913 List of names of flux fields for reference catalog
916 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
917 sourceCat.reserve(selected.sum())
918 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
919 sourceCat[
'instFlux'] = 10.**(stdCat[
'mag_std_noabs'][selected, b_index]/(-2.5))
920 sourceCat[
'instFluxErr'] = (np.log(10.)/2.5)*(stdCat[
'magErr_std'][selected, b_index]
921 * sourceCat[
'instFlux'])
925 badStar = (stdCat[
'mag_std_noabs'][selected, b_index] > 90.0)
926 for rec
in sourceCat[badStar]:
927 rec.set(badStarKey,
True)
929 exposure = afwImage.ExposureF()
930 exposure.setFilterLabel(filterLabel)
932 if refFluxFields[b_index]
is None:
935 ctr = stdCat[0].getCoord()
936 rad = 0.05*lsst.geom.degrees
937 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel)
938 refFluxFields[b_index] = refDataTest.fluxField
941 calConfig = copy.copy(self.config.photoCal.value)
942 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index]
943 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] +
'Err'
944 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
946 schema=sourceCat.getSchema())
948 struct = calTask.run(exposure, sourceCat)
952 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
954 Output standard stars in indexed reference catalog format.
955 This is not currently supported in Gen3.
959 butler : `lsst.daf.persistence.Butler`
960 stdCat : `lsst.afw.table.SimpleCatalog`
961 FGCM standard star catalog from fgcmFitCycleTask
962 offsets : `numpy.array` of floats
963 Per band zeropoint offsets
964 bands : `list` [`str`]
965 List of band names from FGCM output
966 datasetConfig : `lsst.meas.algorithms.DatasetConfig`
967 Config for reference dataset
970 self.log.info(
"Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
972 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
973 self.config.datasetConfig.indexer.active)
980 conv = stdCat[0][
'coord_ra'].asDegrees()/float(stdCat[0][
'coord_ra'])
981 indices = np.array(indexer.indexPoints(stdCat[
'coord_ra']*conv,
982 stdCat[
'coord_dec']*conv))
984 formattedCat = self._formatCatalog(stdCat, offsets, bands)
987 dataId = indexer.makeDataId(
'master_schema',
988 datasetConfig.ref_dataset_name)
989 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
990 addRefCatMetadata(masterCat)
991 butler.put(masterCat,
'ref_cat', dataId=dataId)
994 h, rev = esutil.stat.histogram(indices, rev=
True)
995 gd, = np.where(h > 0)
996 selected = np.zeros(len(formattedCat), dtype=bool)
998 i1a = rev[rev[i]: rev[i + 1]]
1004 selected[i1a] =
True
1007 dataId = indexer.makeDataId(indices[i1a[0]],
1008 datasetConfig.ref_dataset_name)
1009 butler.put(formattedCat[selected],
'ref_cat', dataId=dataId)
1012 dataId = indexer.makeDataId(
None, datasetConfig.ref_dataset_name)
1013 butler.put(datasetConfig,
'ref_cat_config', dataId=dataId)
1015 self.log.info(
"Done outputting standard stars.")
1017 def _formatCatalog(self, fgcmStarCat, offsets, bands):
1019 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
1023 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
1024 SimpleCatalog as output by fgcmcal
1025 offsets : `list` with len(self.bands) entries
1026 Zeropoint offsets to apply
1027 bands : `list` [`str`]
1028 List of band names from FGCM output
1032 formattedCat: `lsst.afw.table.SimpleCatalog`
1033 SimpleCatalog suitable for using as a reference catalog
1036 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1037 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1041 sourceMapper.addMinimalSchema(minSchema)
1043 sourceMapper.editOutputSchema().addField(
'%s_nGood' % (band), type=np.int32)
1044 sourceMapper.editOutputSchema().addField(
'%s_nTotal' % (band), type=np.int32)
1045 sourceMapper.editOutputSchema().addField(
'%s_nPsfCandidate' % (band), type=np.int32)
1047 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1048 formattedCat.reserve(len(fgcmStarCat))
1049 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1053 for b, band
in enumerate(bands):
1054 mag = fgcmStarCat[
'mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1057 flux = (mag*units.ABmag).to_value(units.nJy)
1058 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat[
'magErr_std'][:, b].astype(np.float64)
1060 formattedCat[
'%s_flux' % (band)][:] = flux
1061 formattedCat[
'%s_fluxErr' % (band)][:] = fluxErr
1062 formattedCat[
'%s_nGood' % (band)][:] = fgcmStarCat[
'ngood'][:, b]
1063 formattedCat[
'%s_nTotal' % (band)][:] = fgcmStarCat[
'ntotal'][:, b]
1064 formattedCat[
'%s_nPsfCandidate' % (band)][:] = fgcmStarCat[
'npsfcand'][:, b]
1066 addRefCatMetadata(formattedCat)
1070 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1071 physicalFilterMap, returnCatalogs=True,
1073 """Output the zeropoints in fgcm_photoCalib format.
1077 camera : `lsst.afw.cameraGeom.Camera`
1078 Camera from the butler.
1079 zptCat : `lsst.afw.table.BaseCatalog`
1080 FGCM zeropoint catalog from `FgcmFitCycleTask`.
1081 visitCat : `lsst.afw.table.BaseCatalog`
1082 FGCM visitCat from `FgcmBuildStarsTask`.
1083 offsets : `numpy.array`
1084 Float array of absolute calibration offsets, one for each filter.
1085 bands : `list` [`str`]
1086 List of band names from FGCM output.
1087 physicalFilterMap : `dict`
1088 Dictionary of mappings from physical filter to FGCM band.
1089 returnCatalogs : `bool`, optional
1090 Return photoCalibs as per-visit exposure catalogs.
1091 tract: `int`, optional
1092 Tract number to output. Default is None (global calibration)
1096 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1097 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1098 (returned if returnCatalogs is False).
1099 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1100 Generator that returns (visit, exposureCatalog) tuples.
1101 (returned if returnCatalogs is True).
1106 cannot_compute = fgcm.fgcmUtilities.zpFlagDict[
'CANNOT_COMPUTE_ZEROPOINT']
1107 selected = (((zptCat[
'fgcmFlag'] & cannot_compute) == 0)
1108 & (zptCat[
'fgcmZptVar'] > 0.0))
1111 badVisits = np.unique(zptCat[
'visit'][~selected])
1112 goodVisits = np.unique(zptCat[
'visit'][selected])
1113 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1114 for allBadVisit
in allBadVisits:
1115 self.log.warn(f
'No suitable photoCalib for visit {allBadVisit}')
1119 for f
in physicalFilterMap:
1121 if physicalFilterMap[f]
in bands:
1122 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1126 for ccdIndex, detector
in enumerate(camera):
1127 ccdMapping[detector.getId()] = ccdIndex
1131 for rec
in visitCat:
1132 scalingMapping[rec[
'visit']] = rec[
'scaling']
1134 if self.config.doComposeWcsJacobian:
1139 zptVisitCatalog =
None
1141 metadata = dafBase.PropertyList()
1142 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
1143 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
1145 for rec
in zptCat[selected]:
1147 scaling = scalingMapping[rec[
'visit']][ccdMapping[rec[
'detector']]]
1154 postCalibrationOffset = offsetMapping[rec[
'filtername']]
1155 if self.config.doApplyMeanChromaticCorrection:
1156 postCalibrationOffset += rec[
'fgcmDeltaChrom']
1158 fgcmSuperStarField = self._getChebyshevBoundedField(rec[
'fgcmfZptSstarCheb'],
1159 rec[
'fgcmfZptChebXyMax'])
1161 fgcmZptField = self._getChebyshevBoundedField((rec[
'fgcmfZptCheb']*units.AB).to_value(units.nJy),
1162 rec[
'fgcmfZptChebXyMax'],
1163 offset=postCalibrationOffset,
1166 if self.config.doComposeWcsJacobian:
1168 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec[
'detector']],
1174 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1177 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1178 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec[
'fgcmZptVar'])
1179 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1180 calibrationErr=calibErr,
1181 calibration=fgcmField,
1184 if not returnCatalogs:
1186 yield (int(rec[
'visit']), int(rec[
'detector']), rec[
'filtername'], photoCalib)
1189 if rec[
'visit'] != lastVisit:
1194 zptVisitCatalog.sort()
1195 yield (int(lastVisit), zptVisitCatalog)
1198 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1199 zptExpCatSchema.addField(
'visit', type=
'I', doc=
'Visit number')
1202 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1203 zptVisitCatalog.setMetadata(metadata)
1205 lastVisit = int(rec[
'visit'])
1207 catRecord = zptVisitCatalog.addNew()
1208 catRecord[
'id'] = int(rec[
'detector'])
1209 catRecord[
'visit'] = rec[
'visit']
1210 catRecord.setPhotoCalib(photoCalib)
1215 zptVisitCatalog.sort()
1216 yield (int(lastVisit), zptVisitCatalog)
1218 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1220 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1225 coefficients: `numpy.array`
1226 Flattened array of chebyshev coefficients
1227 xyMax: `list` of length 2
1228 Maximum x and y of the chebyshev bounding box
1229 offset: `float`, optional
1230 Absolute calibration offset. Default is 0.0
1231 scaling: `float`, optional
1232 Flat scaling value from fgcmBuildStars. Default is 1.0
1236 boundedField: `lsst.afw.math.ChebyshevBoundedField`
1239 orderPlus1 = int(np.sqrt(coefficients.size))
1240 pars = np.zeros((orderPlus1, orderPlus1))
1242 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1243 lsst.geom.Point2I(*xyMax))
1245 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1246 * (10.**(offset/-2.5))*scaling)
1248 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1252 def _outputAtmospheres(self, dataRefDict, atmCat):
1254 Output the atmospheres.
1258 dataRefDict : `dict`
1259 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1260 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1261 dataRef dictionary with keys:
1263 ``"fgcmLookUpTable"``
1264 dataRef for the FGCM look-up table.
1265 atmCat : `lsst.afw.table.BaseCatalog`
1266 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1270 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1271 Generator that returns (visit, transmissionCurve) tuples.
1274 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
1276 atmosphereTableName = lutCat[0][
'tablename']
1277 elevation = lutCat[0][
'elevation']
1278 atmLambda = lutCat[0][
'atmLambda']
1283 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1284 atmTable.loadTable()
1288 if atmTable
is None:
1291 modGen = fgcm.ModtranGenerator(elevation)
1292 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1293 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1294 except (ValueError, IOError)
as e:
1295 raise RuntimeError(
"FGCM look-up-table generated with modtran, "
1296 "but modtran not configured to run.")
from e
1298 zenith = np.degrees(np.arccos(1./atmCat[
'secZenith']))
1300 for i, visit
in enumerate(atmCat[
'visit']):
1301 if atmTable
is not None:
1303 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i][
'pmb'],
1304 pwv=atmCat[i][
'pwv'],
1306 tau=atmCat[i][
'tau'],
1307 alpha=atmCat[i][
'alpha'],
1309 ctranslamstd=[atmCat[i][
'cTrans'],
1310 atmCat[i][
'lamStd']])
1313 modAtm = modGen(pmb=atmCat[i][
'pmb'],
1314 pwv=atmCat[i][
'pwv'],
1316 tau=atmCat[i][
'tau'],
1317 alpha=atmCat[i][
'alpha'],
1319 lambdaRange=lambdaRange,
1320 lambdaStep=lambdaStep,
1321 ctranslamstd=[atmCat[i][
'cTrans'],
1322 atmCat[i][
'lamStd']])
1323 atmVals = modAtm[
'COMBINED']
1326 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1327 wavelengths=atmLambda,
1328 throughputAtMin=atmVals[0],
1329 throughputAtMax=atmVals[-1])
1331 yield (int(visit), curve)
def computeApproxPixelAreaFields(camera)