23 """Make the final fgcmcal output products.
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
41 from astropy
import units
43 import lsst.daf.base
as dafBase
44 import lsst.pex.config
as pexConfig
45 import lsst.pipe.base
as pipeBase
46 from lsst.pipe.base
import connectionTypes
47 from lsst.afw.image
import TransmissionCurve
48 from lsst.meas.algorithms
import LoadIndexedReferenceObjectsTask
49 from lsst.meas.algorithms
import ReferenceObjectLoader
50 from lsst.pipe.tasks.photoCal
import PhotoCalTask
52 import lsst.afw.image
as afwImage
53 import lsst.afw.math
as afwMath
54 import lsst.afw.table
as afwTable
55 from lsst.meas.algorithms
import IndexerRegistry
56 from lsst.meas.algorithms
import DatasetConfig
57 from lsst.meas.algorithms.ingestIndexReferenceTask
import addRefCatMetadata
59 from .utilities
import computeApproxPixelAreaFields
60 from .utilities
import lookupStaticCalibrations
64 __all__ = [
'FgcmOutputProductsConfig',
'FgcmOutputProductsTask',
'FgcmOutputProductsRunner']
68 dimensions=(
"instrument",),
69 defaultTemplates={
"cycleNumber":
"0"}):
70 camera = connectionTypes.PrerequisiteInput(
71 doc=
"Camera instrument",
73 storageClass=
"Camera",
74 dimensions=(
"instrument",),
75 lookupFunction=lookupStaticCalibrations,
79 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
80 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
81 "chromatic corrections."),
82 name=
"fgcmLookUpTable",
83 storageClass=
"Catalog",
84 dimensions=(
"instrument",),
88 fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
89 doc=
"Catalog of visit information for fgcm",
90 name=
"fgcmVisitCatalog",
91 storageClass=
"Catalog",
92 dimensions=(
"instrument",),
96 fgcmStandardStars = connectionTypes.PrerequisiteInput(
97 doc=
"Catalog of standard star data from fgcm fit",
98 name=
"fgcmStandardStars{cycleNumber}",
99 storageClass=
"SimpleCatalog",
100 dimensions=(
"instrument",),
104 fgcmZeropoints = connectionTypes.PrerequisiteInput(
105 doc=
"Catalog of zeropoints from fgcm fit",
106 name=
"fgcmZeropoints{cycleNumber}",
107 storageClass=
"Catalog",
108 dimensions=(
"instrument",),
112 fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
113 doc=
"Catalog of atmosphere parameters from fgcm fit",
114 name=
"fgcmAtmosphereParameters{cycleNumber}",
115 storageClass=
"Catalog",
116 dimensions=(
"instrument",),
120 refCat = connectionTypes.PrerequisiteInput(
121 doc=
"Reference catalog to use for photometric calibration",
123 storageClass=
"SimpleCatalog",
124 dimensions=(
"skypix",),
129 fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
130 doc=
"Config used to build FGCM input stars",
131 name=
"fgcmBuildStarsTable_config",
132 storageClass=
"Config",
135 fgcmPhotoCalib = connectionTypes.Output(
136 doc=(
"Per-visit photometric calibrations derived from fgcm calibration. "
137 "These catalogs use detector id for the id and are sorted for "
138 "fast lookups of a detector."),
139 name=
"fgcmPhotoCalibCatalog",
140 storageClass=
"ExposureCatalog",
141 dimensions=(
"instrument",
"visit",),
145 fgcmTransmissionAtmosphere = connectionTypes.Output(
146 doc=
"Per-visit atmosphere transmission files produced from fgcm calibration",
147 name=
"transmission_atmosphere_fgcm",
148 storageClass=
"TransmissionCurve",
149 dimensions=(
"instrument",
154 fgcmOffsets = connectionTypes.Output(
155 doc=
"Per-band offsets computed from doReferenceCalibration",
156 name=
"fgcmReferenceCalibrationOffsets",
157 storageClass=
"Catalog",
158 dimensions=(
"instrument",),
162 def __init__(self, *, config=None):
163 super().__init__(config=config)
165 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
166 raise ValueError(
"cycleNumber must be of integer format")
167 if config.connections.refCat != config.refObjLoader.ref_dataset_name:
168 raise ValueError(
"connections.refCat must be the same as refObjLoader.ref_dataset_name")
170 if config.doRefcatOutput:
171 raise ValueError(
"FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
173 if not config.doReferenceCalibration:
174 self.prerequisiteInputs.remove(
"refCat")
175 if not config.doAtmosphereOutput:
176 self.prerequisiteInputs.remove(
"fgcmAtmosphereParameters")
177 if not config.doZeropointOutput:
178 self.prerequisiteInputs.remove(
"fgcmZeropoints")
179 if not config.doReferenceCalibration:
180 self.outputs.remove(
"fgcmOffsets")
183 class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
184 pipelineConnections=FgcmOutputProductsConnections):
185 """Config for FgcmOutputProductsTask"""
187 cycleNumber = pexConfig.Field(
188 doc=
"Final fit cycle from FGCM fit",
195 doReferenceCalibration = pexConfig.Field(
196 doc=(
"Transfer 'absolute' calibration from reference catalog? "
197 "This afterburner step is unnecessary if reference stars "
198 "were used in the full fit in FgcmFitCycleTask."),
202 doRefcatOutput = pexConfig.Field(
203 doc=
"Output standard stars in reference catalog format",
207 doAtmosphereOutput = pexConfig.Field(
208 doc=
"Output atmospheres in transmission_atmosphere_fgcm format",
212 doZeropointOutput = pexConfig.Field(
213 doc=
"Output zeropoints in fgcm_photoCalib format",
217 doComposeWcsJacobian = pexConfig.Field(
218 doc=
"Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
222 doApplyMeanChromaticCorrection = pexConfig.Field(
223 doc=
"Apply the mean chromatic correction to the zeropoints?",
227 refObjLoader = pexConfig.ConfigurableField(
228 target=LoadIndexedReferenceObjectsTask,
229 doc=
"reference object loader for 'absolute' photometric calibration",
231 photoCal = pexConfig.ConfigurableField(
233 doc=
"task to perform 'absolute' calibration",
235 referencePixelizationNside = pexConfig.Field(
236 doc=
"Healpix nside to pixelize catalog to compare to reference catalog",
240 referencePixelizationMinStars = pexConfig.Field(
241 doc=(
"Minimum number of stars per healpix pixel to select for comparison"
242 "to the specified reference catalog"),
246 referenceMinMatch = pexConfig.Field(
247 doc=
"Minimum number of stars matched to reference catalog to be used in statistics",
251 referencePixelizationNPixels = pexConfig.Field(
252 doc=(
"Number of healpix pixels to sample to do comparison. "
253 "Doing too many will take a long time and not yield any more "
254 "precise results because the final number is the median offset "
255 "(per band) from the set of pixels."),
259 datasetConfig = pexConfig.ConfigField(
261 doc=
"Configuration for writing/reading ingested catalog",
264 def setDefaults(self):
265 pexConfig.Config.setDefaults(self)
275 self.photoCal.applyColorTerms =
False
276 self.photoCal.fluxField =
'instFlux'
277 self.photoCal.magErrFloor = 0.003
278 self.photoCal.match.referenceSelection.doSignalToNoise =
True
279 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
280 self.photoCal.match.sourceSelection.doSignalToNoise =
True
281 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
282 self.photoCal.match.sourceSelection.signalToNoise.fluxField =
'instFlux'
283 self.photoCal.match.sourceSelection.signalToNoise.errField =
'instFluxErr'
284 self.photoCal.match.sourceSelection.doFlags =
True
285 self.photoCal.match.sourceSelection.flags.good = []
286 self.photoCal.match.sourceSelection.flags.bad = [
'flag_badStar']
287 self.photoCal.match.sourceSelection.doUnresolved =
False
288 self.datasetConfig.ref_dataset_name =
'fgcm_stars'
289 self.datasetConfig.format_version = 1
295 self.connections.cycleNumber = str(self.cycleNumber)
298 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
299 """Subclass of TaskRunner for fgcmOutputProductsTask
301 fgcmOutputProductsTask.run() takes one argument, the butler, and
302 does not run on any data in the repository.
303 This runner does not use any parallelization.
307 def getTargetList(parsedCmd):
309 Return a list with one element, the butler.
311 return [parsedCmd.butler]
313 def __call__(self, butler):
317 butler: `lsst.daf.persistence.Butler`
321 exitStatus: `list` with `pipeBase.Struct`
322 exitStatus (0: success; 1: failure)
323 if self.doReturnResults also
324 results (`np.array` with absolute zeropoint offsets)
326 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
330 results = task.runDataRef(butler)
333 results = task.runDataRef(butler)
334 except Exception
as e:
336 task.log.fatal(
"Failed: %s" % e)
337 if not isinstance(e, pipeBase.TaskError):
338 traceback.print_exc(file=sys.stderr)
340 task.writeMetadata(butler)
342 if self.doReturnResults:
344 return [pipeBase.Struct(exitStatus=exitStatus,
347 return [pipeBase.Struct(exitStatus=exitStatus)]
349 def run(self, parsedCmd):
351 Run the task, with no multiprocessing
355 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
360 if self.precall(parsedCmd):
361 targetList = self.getTargetList(parsedCmd)
363 resultList = self(targetList[0])
368 class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
370 Output products from FGCM global calibration.
373 ConfigClass = FgcmOutputProductsConfig
374 RunnerClass = FgcmOutputProductsRunner
375 _DefaultName =
"fgcmOutputProducts"
377 def __init__(self, butler=None, **kwargs):
378 super().__init__(**kwargs)
381 def _getMetadataName(self):
384 def runQuantum(self, butlerQC, inputRefs, outputRefs):
386 dataRefDict[
'camera'] = butlerQC.get(inputRefs.camera)
387 dataRefDict[
'fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
388 dataRefDict[
'fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
389 dataRefDict[
'fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
391 if self.config.doZeropointOutput:
392 dataRefDict[
'fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
393 photoCalibRefDict = {photoCalibRef.dataId.byName()[
'visit']:
394 photoCalibRef
for photoCalibRef
in outputRefs.fgcmPhotoCalib}
396 if self.config.doAtmosphereOutput:
397 dataRefDict[
'fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
398 atmRefDict = {atmRef.dataId.byName()[
'visit']: atmRef
for
399 atmRef
in outputRefs.fgcmTransmissionAtmosphere}
401 if self.config.doReferenceCalibration:
402 refConfig = self.config.refObjLoader
403 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
404 for ref
in inputRefs.refCat],
405 refCats=butlerQC.get(inputRefs.refCat),
409 self.refObjLoader =
None
411 dataRefDict[
'fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
413 fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
414 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
416 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
417 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
418 "in fgcmBuildStarsTask.")
419 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
420 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
422 struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=
True)
425 if struct.photoCalibCatalogs
is not None:
426 self.log.info(
"Outputting photoCalib catalogs.")
427 for visit, expCatalog
in struct.photoCalibCatalogs:
428 butlerQC.put(expCatalog, photoCalibRefDict[visit])
429 self.log.info(
"Done outputting photoCalib catalogs.")
432 if struct.atmospheres
is not None:
433 self.log.info(
"Outputting atmosphere transmission files.")
434 for visit, atm
in struct.atmospheres:
435 butlerQC.put(atm, atmRefDict[visit])
436 self.log.info(
"Done outputting atmosphere files.")
438 if self.config.doReferenceCalibration:
440 schema = afwTable.Schema()
441 schema.addField(
'offset', type=np.float64,
442 doc=
"Post-process calibration offset (mag)")
443 offsetCat = afwTable.BaseCatalog(schema)
444 offsetCat.resize(len(struct.offsets))
445 offsetCat[
'offset'][:] = struct.offsets
447 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
452 def runDataRef(self, butler):
454 Make FGCM output products for use in the stack
458 butler: `lsst.daf.persistence.Butler`
460 Final fit cycle number, override config.
464 offsets: `lsst.pipe.base.Struct`
465 A structure with array of zeropoint offsets
470 Raised if any one of the following is true:
472 - butler cannot find "fgcmBuildStars_config" or
473 "fgcmBuildStarsTable_config".
474 - butler cannot find "fgcmFitCycle_config".
475 - "fgcmFitCycle_config" does not refer to
476 `self.config.cycleNumber`.
477 - butler cannot find "fgcmAtmosphereParameters" and
478 `self.config.doAtmosphereOutput` is `True`.
479 - butler cannot find "fgcmStandardStars" and
480 `self.config.doReferenceCalibration` is `True` or
481 `self.config.doRefcatOutput` is `True`.
482 - butler cannot find "fgcmZeropoints" and
483 `self.config.doZeropointOutput` is `True`.
485 if self.config.doReferenceCalibration:
487 self.makeSubtask(
"refObjLoader", butler=butler)
491 if not butler.datasetExists(
'fgcmBuildStarsTable_config')
and \
492 not butler.datasetExists(
'fgcmBuildStars_config'):
493 raise RuntimeError(
"Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
494 "which is prereq for fgcmOutputProducts")
496 if butler.datasetExists(
'fgcmBuildStarsTable_config'):
497 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStarsTable_config')
499 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStars_config')
500 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
501 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
502 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
504 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
505 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
506 "in fgcmBuildStarsTask.")
508 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
509 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
512 if (self.config.doAtmosphereOutput
513 and not butler.datasetExists(
'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
514 raise RuntimeError(f
"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
516 if not butler.datasetExists(
'fgcmStandardStars',
517 fgcmcycle=self.config.cycleNumber):
518 raise RuntimeError(
"Standard stars are missing for cycle %d." %
519 (self.config.cycleNumber))
521 if (self.config.doZeropointOutput
522 and (
not butler.datasetExists(
'fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
523 raise RuntimeError(
"Zeropoints are missing for cycle %d." %
524 (self.config.cycleNumber))
528 dataRefDict[
'camera'] = butler.get(
'camera')
529 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
530 dataRefDict[
'fgcmVisitCatalog'] = butler.dataRef(
'fgcmVisitCatalog')
531 dataRefDict[
'fgcmStandardStars'] = butler.dataRef(
'fgcmStandardStars',
532 fgcmcycle=self.config.cycleNumber)
534 if self.config.doZeropointOutput:
535 dataRefDict[
'fgcmZeropoints'] = butler.dataRef(
'fgcmZeropoints',
536 fgcmcycle=self.config.cycleNumber)
537 if self.config.doAtmosphereOutput:
538 dataRefDict[
'fgcmAtmosphereParameters'] = butler.dataRef(
'fgcmAtmosphereParameters',
539 fgcmcycle=self.config.cycleNumber)
541 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=
False)
543 if struct.photoCalibs
is not None:
544 self.log.info(
"Outputting photoCalib files.")
546 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
547 butler.put(photoCalib,
'fgcm_photoCalib',
548 dataId={visitDataRefName: visit,
549 ccdDataRefName: detector,
550 'filter': physicalFilter})
552 self.log.info(
"Done outputting photoCalib files.")
554 if struct.atmospheres
is not None:
555 self.log.info(
"Outputting atmosphere transmission files.")
556 for visit, atm
in struct.atmospheres:
557 butler.put(atm,
"transmission_atmosphere_fgcm",
558 dataId={visitDataRefName: visit})
559 self.log.info(
"Done outputting atmosphere transmissions.")
561 return pipeBase.Struct(offsets=struct.offsets)
563 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
564 """Run the output products task.
569 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
570 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
571 dataRef dictionary with keys:
574 Camera object (`lsst.afw.cameraGeom.Camera`)
575 ``"fgcmLookUpTable"``
576 dataRef for the FGCM look-up table.
577 ``"fgcmVisitCatalog"``
578 dataRef for visit summary catalog.
579 ``"fgcmStandardStars"``
580 dataRef for the output standard star catalog.
582 dataRef for the zeropoint data catalog.
583 ``"fgcmAtmosphereParameters"``
584 dataRef for the atmosphere parameter catalog.
585 ``"fgcmBuildStarsTableConfig"``
586 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
587 physicalFilterMap : `dict`
588 Dictionary of mappings from physical filter to FGCM band.
589 returnCatalogs : `bool`, optional
590 Return photoCalibs as per-visit exposure catalogs.
591 butler : `lsst.daf.persistence.Butler`, optional
592 Gen2 butler used for reference star outputs
596 retStruct : `lsst.pipe.base.Struct`
597 Output structure with keys:
599 offsets : `np.ndarray`
600 Final reference offsets, per band.
601 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
602 Generator that returns (visit, transmissionCurve) tuples.
603 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
604 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
605 (returned if returnCatalogs is False).
606 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
607 Generator that returns (visit, exposureCatalog) tuples.
608 (returned if returnCatalogs is True).
610 stdCat = dataRefDict[
'fgcmStandardStars'].get()
611 md = stdCat.getMetadata()
612 bands = md.getArray(
'BANDS')
614 if self.config.doReferenceCalibration:
615 offsets = self._computeReferenceOffsets(stdCat, bands)
617 offsets = np.zeros(len(bands))
620 if self.config.doRefcatOutput
and butler
is not None:
621 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
625 if self.config.doZeropointOutput:
626 zptCat = dataRefDict[
'fgcmZeropoints'].get()
627 visitCat = dataRefDict[
'fgcmVisitCatalog'].get()
629 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
630 physicalFilterMap, returnCatalogs=returnCatalogs)
634 if self.config.doAtmosphereOutput:
635 atmCat = dataRefDict[
'fgcmAtmosphereParameters'].get()
636 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
640 retStruct = pipeBase.Struct(offsets=offsets,
643 retStruct.photoCalibCatalogs = pcgen
645 retStruct.photoCalibs = pcgen
649 def generateTractOutputProducts(self, dataRefDict, tract,
650 visitCat, zptCat, atmCat, stdCat,
651 fgcmBuildStarsConfig,
655 Generate the output products for a given tract, as specified in the config.
657 This method is here to have an alternate entry-point for
663 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
664 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
665 dataRef dictionary with keys:
668 Camera object (`lsst.afw.cameraGeom.Camera`)
669 ``"fgcmLookUpTable"``
670 dataRef for the FGCM look-up table.
673 visitCat : `lsst.afw.table.BaseCatalog`
674 FGCM visitCat from `FgcmBuildStarsTask`
675 zptCat : `lsst.afw.table.BaseCatalog`
676 FGCM zeropoint catalog from `FgcmFitCycleTask`
677 atmCat : `lsst.afw.table.BaseCatalog`
678 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
679 stdCat : `lsst.afw.table.SimpleCatalog`
680 FGCM standard star catalog from `FgcmFitCycleTask`
681 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
682 Configuration object from `FgcmBuildStarsTask`
683 returnCatalogs : `bool`, optional
684 Return photoCalibs as per-visit exposure catalogs.
685 butler: `lsst.daf.persistence.Butler`, optional
686 Gen2 butler used for reference star outputs
690 retStruct : `lsst.pipe.base.Struct`
691 Output structure with keys:
693 offsets : `np.ndarray`
694 Final reference offsets, per band.
695 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
696 Generator that returns (visit, transmissionCurve) tuples.
697 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
698 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
699 (returned if returnCatalogs is False).
700 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
701 Generator that returns (visit, exposureCatalog) tuples.
702 (returned if returnCatalogs is True).
704 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
706 md = stdCat.getMetadata()
707 bands = md.getArray(
'BANDS')
709 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
710 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
711 "in fgcmBuildStarsTask.")
713 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
714 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
716 if self.config.doReferenceCalibration:
717 offsets = self._computeReferenceOffsets(stdCat, bands)
719 offsets = np.zeros(len(bands))
721 if self.config.doRefcatOutput
and butler
is not None:
723 datasetConfig = copy.copy(self.config.datasetConfig)
724 datasetConfig.ref_dataset_name =
'%s_%d' % (self.config.datasetConfig.ref_dataset_name,
726 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
728 if self.config.doZeropointOutput:
729 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
730 physicalFilterMap, returnCatalogs=returnCatalogs)
734 if self.config.doAtmosphereOutput:
735 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
739 retStruct = pipeBase.Struct(offsets=offsets,
742 retStruct.photoCalibCatalogs = pcgen
744 retStruct.photoCalibs = pcgen
748 def _computeReferenceOffsets(self, stdCat, bands):
750 Compute offsets relative to a reference catalog.
752 This method splits the star catalog into healpix pixels
753 and computes the calibration transfer for a sample of
754 these pixels to approximate the 'absolute' calibration
755 values (on for each band) to apply to transfer the
760 stdCat : `lsst.afw.table.SimpleCatalog`
762 bands : `list` [`str`]
763 List of band names from FGCM output
766 offsets : `numpy.array` of floats
767 Per band zeropoint offsets
773 minObs = stdCat[
'ngood'].min(axis=1)
775 goodStars = (minObs >= 1)
776 stdCat = stdCat[goodStars]
778 self.log.info(
"Found %d stars with at least 1 good observation in each band" %
788 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
789 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
790 sourceMapper.editOutputSchema().addField(
'instFlux', type=np.float64,
791 doc=
"instrumental flux (counts)")
792 sourceMapper.editOutputSchema().addField(
'instFluxErr', type=np.float64,
793 doc=
"instrumental flux error (counts)")
794 badStarKey = sourceMapper.editOutputSchema().addField(
'flag_badStar',
802 theta = np.pi/2. - stdCat[
'coord_dec']
803 phi = stdCat[
'coord_ra']
805 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
806 h, rev = esutil.stat.histogram(ipring, rev=
True)
808 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
810 self.log.info(
"Found %d pixels (nside=%d) with at least %d good stars" %
812 self.config.referencePixelizationNside,
813 self.config.referencePixelizationMinStars))
815 if gdpix.size < self.config.referencePixelizationNPixels:
816 self.log.warn(
"Found fewer good pixels (%d) than preferred in configuration (%d)" %
817 (gdpix.size, self.config.referencePixelizationNPixels))
820 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=
False)
822 results = np.zeros(gdpix.size, dtype=[(
'hpix',
'i4'),
823 (
'nstar',
'i4', len(bands)),
824 (
'nmatch',
'i4', len(bands)),
825 (
'zp',
'f4', len(bands)),
826 (
'zpErr',
'f4', len(bands))])
827 results[
'hpix'] = ipring[rev[rev[gdpix]]]
830 selected = np.zeros(len(stdCat), dtype=bool)
832 refFluxFields = [
None]*len(bands)
834 for p, pix
in enumerate(gdpix):
835 i1a = rev[rev[pix]: rev[pix + 1]]
843 for b, band
in enumerate(bands):
845 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
846 selected, refFluxFields)
847 results[
'nstar'][p, b] = len(i1a)
848 results[
'nmatch'][p, b] = len(struct.arrays.refMag)
849 results[
'zp'][p, b] = struct.zp
850 results[
'zpErr'][p, b] = struct.sigma
853 offsets = np.zeros(len(bands))
855 for b, band
in enumerate(bands):
857 ok, = np.where(results[
'nmatch'][:, b] >= self.config.referenceMinMatch)
858 offsets[b] = np.median(results[
'zp'][ok, b])
861 madSigma = 1.4826*np.median(np.abs(results[
'zp'][ok, b] - offsets[b]))
862 self.log.info(
"Reference catalog offset for %s band: %.12f +/- %.12f" %
863 (band, offsets[b], madSigma))
867 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
868 b, band, stdCat, selected, refFluxFields):
870 Compute the zeropoint offset between the fgcm stdCat and the reference
871 stars for one pixel in one band
875 sourceMapper: `lsst.afw.table.SchemaMapper`
876 Mapper to go from stdCat to calibratable catalog
877 badStarKey: `lsst.afw.table.Key`
878 Key for the field with bad stars
880 Index of the band in the star catalog
882 Name of band for reference catalog
883 stdCat: `lsst.afw.table.SimpleCatalog`
885 selected: `numpy.array(dtype=bool)`
886 Boolean array of which stars are in the pixel
887 refFluxFields: `list`
888 List of names of flux fields for reference catalog
891 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
892 sourceCat.reserve(selected.sum())
893 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
894 sourceCat[
'instFlux'] = 10.**(stdCat[
'mag_std_noabs'][selected, b]/(-2.5))
895 sourceCat[
'instFluxErr'] = (np.log(10.)/2.5)*(stdCat[
'magErr_std'][selected, b]
896 * sourceCat[
'instFlux'])
900 badStar = (stdCat[
'mag_std_noabs'][selected, b] > 90.0)
901 for rec
in sourceCat[badStar]:
902 rec.set(badStarKey,
True)
904 exposure = afwImage.ExposureF()
905 exposure.setFilterLabel(afwImage.FilterLabel(band=band))
907 if refFluxFields[b]
is None:
910 ctr = stdCat[0].getCoord()
911 rad = 0.05*lsst.geom.degrees
912 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
913 refFluxFields[b] = refDataTest.fluxField
916 calConfig = copy.copy(self.config.photoCal.value)
917 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
918 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] +
'Err'
919 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
921 schema=sourceCat.getSchema())
923 struct = calTask.run(exposure, sourceCat)
927 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
929 Output standard stars in indexed reference catalog format.
930 This is not currently supported in Gen3.
934 butler : `lsst.daf.persistence.Butler`
935 stdCat : `lsst.afw.table.SimpleCatalog`
936 FGCM standard star catalog from fgcmFitCycleTask
937 offsets : `numpy.array` of floats
938 Per band zeropoint offsets
939 bands : `list` [`str`]
940 List of band names from FGCM output
941 datasetConfig : `lsst.meas.algorithms.DatasetConfig`
942 Config for reference dataset
945 self.log.info(
"Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
947 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
948 self.config.datasetConfig.indexer.active)
955 conv = stdCat[0][
'coord_ra'].asDegrees()/float(stdCat[0][
'coord_ra'])
956 indices = np.array(indexer.indexPoints(stdCat[
'coord_ra']*conv,
957 stdCat[
'coord_dec']*conv))
959 formattedCat = self._formatCatalog(stdCat, offsets, bands)
962 dataId = indexer.makeDataId(
'master_schema',
963 datasetConfig.ref_dataset_name)
964 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
965 addRefCatMetadata(masterCat)
966 butler.put(masterCat,
'ref_cat', dataId=dataId)
969 h, rev = esutil.stat.histogram(indices, rev=
True)
970 gd, = np.where(h > 0)
971 selected = np.zeros(len(formattedCat), dtype=bool)
973 i1a = rev[rev[i]: rev[i + 1]]
982 dataId = indexer.makeDataId(indices[i1a[0]],
983 datasetConfig.ref_dataset_name)
984 butler.put(formattedCat[selected],
'ref_cat', dataId=dataId)
987 dataId = indexer.makeDataId(
None, datasetConfig.ref_dataset_name)
988 butler.put(datasetConfig,
'ref_cat_config', dataId=dataId)
990 self.log.info(
"Done outputting standard stars.")
992 def _formatCatalog(self, fgcmStarCat, offsets, bands):
994 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
998 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
999 SimpleCatalog as output by fgcmcal
1000 offsets : `list` with len(self.bands) entries
1001 Zeropoint offsets to apply
1002 bands : `list` [`str`]
1003 List of band names from FGCM output
1007 formattedCat: `lsst.afw.table.SimpleCatalog`
1008 SimpleCatalog suitable for using as a reference catalog
1011 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1012 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1016 sourceMapper.addMinimalSchema(minSchema)
1018 sourceMapper.editOutputSchema().addField(
'%s_nGood' % (band), type=np.int32)
1019 sourceMapper.editOutputSchema().addField(
'%s_nTotal' % (band), type=np.int32)
1020 sourceMapper.editOutputSchema().addField(
'%s_nPsfCandidate' % (band), type=np.int32)
1022 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1023 formattedCat.reserve(len(fgcmStarCat))
1024 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1028 for b, band
in enumerate(bands):
1029 mag = fgcmStarCat[
'mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1032 flux = (mag*units.ABmag).to_value(units.nJy)
1033 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat[
'magErr_std'][:, b].astype(np.float64)
1035 formattedCat[
'%s_flux' % (band)][:] = flux
1036 formattedCat[
'%s_fluxErr' % (band)][:] = fluxErr
1037 formattedCat[
'%s_nGood' % (band)][:] = fgcmStarCat[
'ngood'][:, b]
1038 formattedCat[
'%s_nTotal' % (band)][:] = fgcmStarCat[
'ntotal'][:, b]
1039 formattedCat[
'%s_nPsfCandidate' % (band)][:] = fgcmStarCat[
'npsfcand'][:, b]
1041 addRefCatMetadata(formattedCat)
1045 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1046 physicalFilterMap, returnCatalogs=True,
1048 """Output the zeropoints in fgcm_photoCalib format.
1052 camera : `lsst.afw.cameraGeom.Camera`
1053 Camera from the butler.
1054 zptCat : `lsst.afw.table.BaseCatalog`
1055 FGCM zeropoint catalog from `FgcmFitCycleTask`.
1056 visitCat : `lsst.afw.table.BaseCatalog`
1057 FGCM visitCat from `FgcmBuildStarsTask`.
1058 offsets : `numpy.array`
1059 Float array of absolute calibration offsets, one for each filter.
1060 bands : `list` [`str`]
1061 List of band names from FGCM output.
1062 physicalFilterMap : `dict`
1063 Dictionary of mappings from physical filter to FGCM band.
1064 returnCatalogs : `bool`, optional
1065 Return photoCalibs as per-visit exposure catalogs.
1066 tract: `int`, optional
1067 Tract number to output. Default is None (global calibration)
1071 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1072 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1073 (returned if returnCatalogs is False).
1074 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1075 Generator that returns (visit, exposureCatalog) tuples.
1076 (returned if returnCatalogs is True).
1081 cannot_compute = fgcm.fgcmUtilities.zpFlagDict[
'CANNOT_COMPUTE_ZEROPOINT']
1082 selected = (((zptCat[
'fgcmFlag'] & cannot_compute) == 0)
1083 & (zptCat[
'fgcmZptVar'] > 0.0))
1086 badVisits = np.unique(zptCat[
'visit'][~selected])
1087 goodVisits = np.unique(zptCat[
'visit'][selected])
1088 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1089 for allBadVisit
in allBadVisits:
1090 self.log.warn(f
'No suitable photoCalib for visit {allBadVisit}')
1094 for f
in physicalFilterMap:
1096 if physicalFilterMap[f]
in bands:
1097 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1101 for ccdIndex, detector
in enumerate(camera):
1102 ccdMapping[detector.getId()] = ccdIndex
1106 for rec
in visitCat:
1107 scalingMapping[rec[
'visit']] = rec[
'scaling']
1109 if self.config.doComposeWcsJacobian:
1114 zptVisitCatalog =
None
1116 metadata = dafBase.PropertyList()
1117 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
1118 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
1120 for rec
in zptCat[selected]:
1122 scaling = scalingMapping[rec[
'visit']][ccdMapping[rec[
'detector']]]
1129 postCalibrationOffset = offsetMapping[rec[
'filtername']]
1130 if self.config.doApplyMeanChromaticCorrection:
1131 postCalibrationOffset += rec[
'fgcmDeltaChrom']
1133 fgcmSuperStarField = self._getChebyshevBoundedField(rec[
'fgcmfZptSstarCheb'],
1134 rec[
'fgcmfZptChebXyMax'])
1136 fgcmZptField = self._getChebyshevBoundedField((rec[
'fgcmfZptCheb']*units.AB).to_value(units.nJy),
1137 rec[
'fgcmfZptChebXyMax'],
1138 offset=postCalibrationOffset,
1141 if self.config.doComposeWcsJacobian:
1143 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec[
'detector']],
1149 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1152 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1153 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec[
'fgcmZptVar'])
1154 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1155 calibrationErr=calibErr,
1156 calibration=fgcmField,
1159 if not returnCatalogs:
1161 yield (int(rec[
'visit']), int(rec[
'detector']), rec[
'filtername'], photoCalib)
1164 if rec[
'visit'] != lastVisit:
1169 zptVisitCatalog.sort()
1170 yield (int(lastVisit), zptVisitCatalog)
1173 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1174 zptExpCatSchema.addField(
'visit', type=
'I', doc=
'Visit number')
1177 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1178 zptVisitCatalog.setMetadata(metadata)
1180 lastVisit = int(rec[
'visit'])
1182 catRecord = zptVisitCatalog.addNew()
1183 catRecord[
'id'] = int(rec[
'detector'])
1184 catRecord[
'visit'] = rec[
'visit']
1185 catRecord.setPhotoCalib(photoCalib)
1190 zptVisitCatalog.sort()
1191 yield (int(lastVisit), zptVisitCatalog)
1193 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1195 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1200 coefficients: `numpy.array`
1201 Flattened array of chebyshev coefficients
1202 xyMax: `list` of length 2
1203 Maximum x and y of the chebyshev bounding box
1204 offset: `float`, optional
1205 Absolute calibration offset. Default is 0.0
1206 scaling: `float`, optional
1207 Flat scaling value from fgcmBuildStars. Default is 1.0
1211 boundedField: `lsst.afw.math.ChebyshevBoundedField`
1214 orderPlus1 = int(np.sqrt(coefficients.size))
1215 pars = np.zeros((orderPlus1, orderPlus1))
1217 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1218 lsst.geom.Point2I(*xyMax))
1220 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1221 * (10.**(offset/-2.5))*scaling)
1223 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1227 def _outputAtmospheres(self, dataRefDict, atmCat):
1229 Output the atmospheres.
1233 dataRefDict : `dict`
1234 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1235 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1236 dataRef dictionary with keys:
1238 ``"fgcmLookUpTable"``
1239 dataRef for the FGCM look-up table.
1240 atmCat : `lsst.afw.table.BaseCatalog`
1241 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1245 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1246 Generator that returns (visit, transmissionCurve) tuples.
1249 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
1251 atmosphereTableName = lutCat[0][
'tablename']
1252 elevation = lutCat[0][
'elevation']
1253 atmLambda = lutCat[0][
'atmLambda']
1258 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1259 atmTable.loadTable()
1263 if atmTable
is None:
1266 modGen = fgcm.ModtranGenerator(elevation)
1267 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1268 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1269 except (ValueError, IOError)
as e:
1270 raise RuntimeError(
"FGCM look-up-table generated with modtran, "
1271 "but modtran not configured to run.")
from e
1273 zenith = np.degrees(np.arccos(1./atmCat[
'secZenith']))
1275 for i, visit
in enumerate(atmCat[
'visit']):
1276 if atmTable
is not None:
1278 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i][
'pmb'],
1279 pwv=atmCat[i][
'pwv'],
1281 tau=atmCat[i][
'tau'],
1282 alpha=atmCat[i][
'alpha'],
1284 ctranslamstd=[atmCat[i][
'cTrans'],
1285 atmCat[i][
'lamStd']])
1288 modAtm = modGen(pmb=atmCat[i][
'pmb'],
1289 pwv=atmCat[i][
'pwv'],
1291 tau=atmCat[i][
'tau'],
1292 alpha=atmCat[i][
'alpha'],
1294 lambdaRange=lambdaRange,
1295 lambdaStep=lambdaStep,
1296 ctranslamstd=[atmCat[i][
'cTrans'],
1297 atmCat[i][
'lamStd']])
1298 atmVals = modAtm[
'COMBINED']
1301 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1302 wavelengths=atmLambda,
1303 throughputAtMin=atmVals[0],
1304 throughputAtMax=atmVals[-1])
1306 yield (int(visit), curve)
def computeApproxPixelAreaFields(camera)