23 """Make the final fgcmcal output products.
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
41 from astropy
import units
43 import lsst.daf.base
as dafBase
44 import lsst.pex.config
as pexConfig
45 import lsst.pipe.base
as pipeBase
46 from lsst.pipe.base
import connectionTypes
47 from lsst.afw.image
import TransmissionCurve
48 from lsst.meas.algorithms
import LoadIndexedReferenceObjectsTask
49 from lsst.meas.algorithms
import ReferenceObjectLoader
50 from lsst.pipe.tasks.photoCal
import PhotoCalTask
52 import lsst.afw.image
as afwImage
53 import lsst.afw.math
as afwMath
54 import lsst.afw.table
as afwTable
55 from lsst.meas.algorithms
import IndexerRegistry
56 from lsst.meas.algorithms
import DatasetConfig
57 from lsst.meas.algorithms.ingestIndexReferenceTask
import addRefCatMetadata
59 from .utilities
import computeApproxPixelAreaFields
60 from .utilities
import lookupStaticCalibrations
64 __all__ = [
'FgcmOutputProductsConfig',
'FgcmOutputProductsTask',
'FgcmOutputProductsRunner']
68 dimensions=(
"instrument",),
69 defaultTemplates={
"cycleNumber":
"0"}):
70 camera = connectionTypes.PrerequisiteInput(
71 doc=
"Camera instrument",
73 storageClass=
"Camera",
74 dimensions=(
"instrument",),
75 lookupFunction=lookupStaticCalibrations,
79 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
80 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
81 "chromatic corrections."),
82 name=
"fgcmLookUpTable",
83 storageClass=
"Catalog",
84 dimensions=(
"instrument",),
88 fgcmVisitCatalog = connectionTypes.Input(
89 doc=
"Catalog of visit information for fgcm",
90 name=
"fgcmVisitCatalog",
91 storageClass=
"Catalog",
92 dimensions=(
"instrument",),
96 fgcmStandardStars = connectionTypes.Input(
97 doc=
"Catalog of standard star data from fgcm fit",
98 name=
"fgcmStandardStars{cycleNumber}",
99 storageClass=
"SimpleCatalog",
100 dimensions=(
"instrument",),
104 fgcmZeropoints = connectionTypes.Input(
105 doc=
"Catalog of zeropoints from fgcm fit",
106 name=
"fgcmZeropoints{cycleNumber}",
107 storageClass=
"Catalog",
108 dimensions=(
"instrument",),
112 fgcmAtmosphereParameters = connectionTypes.Input(
113 doc=
"Catalog of atmosphere parameters from fgcm fit",
114 name=
"fgcmAtmosphereParameters{cycleNumber}",
115 storageClass=
"Catalog",
116 dimensions=(
"instrument",),
120 refCat = connectionTypes.PrerequisiteInput(
121 doc=
"Reference catalog to use for photometric calibration",
123 storageClass=
"SimpleCatalog",
124 dimensions=(
"skypix",),
129 fgcmPhotoCalib = connectionTypes.Output(
130 doc=(
"Per-visit photometric calibrations derived from fgcm calibration. "
131 "These catalogs use detector id for the id and are sorted for "
132 "fast lookups of a detector."),
133 name=
"fgcmPhotoCalibCatalog",
134 storageClass=
"ExposureCatalog",
135 dimensions=(
"instrument",
"visit",),
139 fgcmTransmissionAtmosphere = connectionTypes.Output(
140 doc=
"Per-visit atmosphere transmission files produced from fgcm calibration",
141 name=
"transmission_atmosphere_fgcm",
142 storageClass=
"TransmissionCurve",
143 dimensions=(
"instrument",
148 fgcmOffsets = connectionTypes.Output(
149 doc=
"Per-band offsets computed from doReferenceCalibration",
150 name=
"fgcmReferenceCalibrationOffsets",
151 storageClass=
"Catalog",
152 dimensions=(
"instrument",),
156 def __init__(self, *, config=None):
157 super().__init__(config=config)
159 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
160 raise ValueError(
"cycleNumber must be of integer format")
161 if config.connections.refCat != config.refObjLoader.ref_dataset_name:
162 raise ValueError(
"connections.refCat must be the same as refObjLoader.ref_dataset_name")
164 if config.doRefcatOutput:
165 raise ValueError(
"FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
167 if not config.doReferenceCalibration:
168 self.prerequisiteInputs.remove(
"refCat")
169 if not config.doAtmosphereOutput:
170 self.inputs.remove(
"fgcmAtmosphereParameters")
171 if not config.doZeropointOutput:
172 self.inputs.remove(
"fgcmZeropoints")
173 if not config.doReferenceCalibration:
174 self.outputs.remove(
"fgcmOffsets")
177 class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
178 pipelineConnections=FgcmOutputProductsConnections):
179 """Config for FgcmOutputProductsTask"""
181 cycleNumber = pexConfig.Field(
182 doc=
"Final fit cycle from FGCM fit",
186 physicalFilterMap = pexConfig.DictField(
187 doc=
"Mapping from 'physicalFilter' to band.",
194 doReferenceCalibration = pexConfig.Field(
195 doc=(
"Transfer 'absolute' calibration from reference catalog? "
196 "This afterburner step is unnecessary if reference stars "
197 "were used in the full fit in FgcmFitCycleTask."),
201 doRefcatOutput = pexConfig.Field(
202 doc=
"Output standard stars in reference catalog format",
206 doAtmosphereOutput = pexConfig.Field(
207 doc=
"Output atmospheres in transmission_atmosphere_fgcm format",
211 doZeropointOutput = pexConfig.Field(
212 doc=
"Output zeropoints in fgcm_photoCalib format",
216 doComposeWcsJacobian = pexConfig.Field(
217 doc=
"Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
221 doApplyMeanChromaticCorrection = pexConfig.Field(
222 doc=
"Apply the mean chromatic correction to the zeropoints?",
226 refObjLoader = pexConfig.ConfigurableField(
227 target=LoadIndexedReferenceObjectsTask,
228 doc=
"reference object loader for 'absolute' photometric calibration",
230 photoCal = pexConfig.ConfigurableField(
232 doc=
"task to perform 'absolute' calibration",
234 referencePixelizationNside = pexConfig.Field(
235 doc=
"Healpix nside to pixelize catalog to compare to reference catalog",
239 referencePixelizationMinStars = pexConfig.Field(
240 doc=(
"Minimum number of stars per healpix pixel to select for comparison"
241 "to the specified reference catalog"),
245 referenceMinMatch = pexConfig.Field(
246 doc=
"Minimum number of stars matched to reference catalog to be used in statistics",
250 referencePixelizationNPixels = pexConfig.Field(
251 doc=(
"Number of healpix pixels to sample to do comparison. "
252 "Doing too many will take a long time and not yield any more "
253 "precise results because the final number is the median offset "
254 "(per band) from the set of pixels."),
258 datasetConfig = pexConfig.ConfigField(
260 doc=
"Configuration for writing/reading ingested catalog",
263 def setDefaults(self):
264 pexConfig.Config.setDefaults(self)
274 self.photoCal.applyColorTerms =
False
275 self.photoCal.fluxField =
'instFlux'
276 self.photoCal.magErrFloor = 0.003
277 self.photoCal.match.referenceSelection.doSignalToNoise =
True
278 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
279 self.photoCal.match.sourceSelection.doSignalToNoise =
True
280 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
281 self.photoCal.match.sourceSelection.signalToNoise.fluxField =
'instFlux'
282 self.photoCal.match.sourceSelection.signalToNoise.errField =
'instFluxErr'
283 self.photoCal.match.sourceSelection.doFlags =
True
284 self.photoCal.match.sourceSelection.flags.good = []
285 self.photoCal.match.sourceSelection.flags.bad = [
'flag_badStar']
286 self.photoCal.match.sourceSelection.doUnresolved =
False
287 self.datasetConfig.ref_dataset_name =
'fgcm_stars'
288 self.datasetConfig.format_version = 1
294 self.connections.cycleNumber = str(self.cycleNumber)
297 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
298 """Subclass of TaskRunner for fgcmOutputProductsTask
300 fgcmOutputProductsTask.run() takes one argument, the butler, and
301 does not run on any data in the repository.
302 This runner does not use any parallelization.
306 def getTargetList(parsedCmd):
308 Return a list with one element, the butler.
310 return [parsedCmd.butler]
312 def __call__(self, butler):
316 butler: `lsst.daf.persistence.Butler`
320 exitStatus: `list` with `pipeBase.Struct`
321 exitStatus (0: success; 1: failure)
322 if self.doReturnResults also
323 results (`np.array` with absolute zeropoint offsets)
325 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
329 results = task.runDataRef(butler)
332 results = task.runDataRef(butler)
333 except Exception
as e:
335 task.log.fatal(
"Failed: %s" % e)
336 if not isinstance(e, pipeBase.TaskError):
337 traceback.print_exc(file=sys.stderr)
339 task.writeMetadata(butler)
341 if self.doReturnResults:
343 return [pipeBase.Struct(exitStatus=exitStatus,
346 return [pipeBase.Struct(exitStatus=exitStatus)]
348 def run(self, parsedCmd):
350 Run the task, with no multiprocessing
354 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
359 if self.precall(parsedCmd):
360 targetList = self.getTargetList(parsedCmd)
362 resultList = self(targetList[0])
367 class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
369 Output products from FGCM global calibration.
372 ConfigClass = FgcmOutputProductsConfig
373 RunnerClass = FgcmOutputProductsRunner
374 _DefaultName =
"fgcmOutputProducts"
376 def __init__(self, butler=None, **kwargs):
377 super().__init__(**kwargs)
380 def _getMetadataName(self):
383 def runQuantum(self, butlerQC, inputRefs, outputRefs):
385 dataRefDict[
'camera'] = butlerQC.get(inputRefs.camera)
386 dataRefDict[
'fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
387 dataRefDict[
'fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
388 dataRefDict[
'fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
390 if self.config.doZeropointOutput:
391 dataRefDict[
'fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
392 photoCalibRefDict = {photoCalibRef.dataId.byName()[
'visit']:
393 photoCalibRef
for photoCalibRef
in outputRefs.fgcmPhotoCalib}
395 if self.config.doAtmosphereOutput:
396 dataRefDict[
'fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
397 atmRefDict = {atmRef.dataId.byName()[
'visit']: atmRef
for
398 atmRef
in outputRefs.fgcmTransmissionAtmosphere}
400 if self.config.doReferenceCalibration:
401 refConfig = self.config.refObjLoader
402 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
403 for ref
in inputRefs.refCat],
404 refCats=butlerQC.get(inputRefs.refCat),
408 self.refObjLoader =
None
410 struct = self.run(dataRefDict, self.config.physicalFilterMap, returnCatalogs=
True)
413 if struct.photoCalibCatalogs
is not None:
414 self.log.info(
"Outputting photoCalib catalogs.")
415 for visit, expCatalog
in struct.photoCalibCatalogs:
416 butlerQC.put(expCatalog, photoCalibRefDict[visit])
417 self.log.info(
"Done outputting photoCalib catalogs.")
420 if struct.atmospheres
is not None:
421 self.log.info(
"Outputting atmosphere transmission files.")
422 for visit, atm
in struct.atmospheres:
423 butlerQC.put(atm, atmRefDict[visit])
424 self.log.info(
"Done outputting atmosphere files.")
426 if self.config.doReferenceCalibration:
428 schema = afwTable.Schema()
429 schema.addField(
'offset', type=np.float64,
430 doc=
"Post-process calibration offset (mag)")
431 offsetCat = afwTable.BaseCatalog(schema)
432 offsetCat.resize(len(struct.offsets))
433 offsetCat[
'offset'][:] = struct.offsets
435 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
440 def runDataRef(self, butler):
442 Make FGCM output products for use in the stack
446 butler: `lsst.daf.persistence.Butler`
448 Final fit cycle number, override config.
452 offsets: `lsst.pipe.base.Struct`
453 A structure with array of zeropoint offsets
458 Raised if any one of the following is true:
460 - butler cannot find "fgcmBuildStars_config" or
461 "fgcmBuildStarsTable_config".
462 - butler cannot find "fgcmFitCycle_config".
463 - "fgcmFitCycle_config" does not refer to
464 `self.config.cycleNumber`.
465 - butler cannot find "fgcmAtmosphereParameters" and
466 `self.config.doAtmosphereOutput` is `True`.
467 - butler cannot find "fgcmStandardStars" and
468 `self.config.doReferenceCalibration` is `True` or
469 `self.config.doRefcatOutput` is `True`.
470 - butler cannot find "fgcmZeropoints" and
471 `self.config.doZeropointOutput` is `True`.
473 if self.config.doReferenceCalibration:
475 self.makeSubtask(
"refObjLoader", butler=butler)
479 if not butler.datasetExists(
'fgcmBuildStarsTable_config')
and \
480 not butler.datasetExists(
'fgcmBuildStars_config'):
481 raise RuntimeError(
"Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
482 "which is prereq for fgcmOutputProducts")
484 if butler.datasetExists(
'fgcmBuildStarsTable_config'):
485 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStarsTable_config')
487 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStars_config')
488 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
489 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
490 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
492 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
493 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
494 "in fgcmBuildStarsTask.")
496 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
497 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
500 if (self.config.doAtmosphereOutput
501 and not butler.datasetExists(
'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
502 raise RuntimeError(f
"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
504 if not butler.datasetExists(
'fgcmStandardStars',
505 fgcmcycle=self.config.cycleNumber):
506 raise RuntimeError(
"Standard stars are missing for cycle %d." %
507 (self.config.cycleNumber))
509 if (self.config.doZeropointOutput
510 and (
not butler.datasetExists(
'fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
511 raise RuntimeError(
"Zeropoints are missing for cycle %d." %
512 (self.config.cycleNumber))
516 dataRefDict[
'camera'] = butler.get(
'camera')
517 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
518 dataRefDict[
'fgcmVisitCatalog'] = butler.dataRef(
'fgcmVisitCatalog')
519 dataRefDict[
'fgcmStandardStars'] = butler.dataRef(
'fgcmStandardStars',
520 fgcmcycle=self.config.cycleNumber)
522 if self.config.doZeropointOutput:
523 dataRefDict[
'fgcmZeropoints'] = butler.dataRef(
'fgcmZeropoints',
524 fgcmcycle=self.config.cycleNumber)
525 if self.config.doAtmosphereOutput:
526 dataRefDict[
'fgcmAtmosphereParameters'] = butler.dataRef(
'fgcmAtmosphereParameters',
527 fgcmcycle=self.config.cycleNumber)
529 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=
False)
531 if struct.photoCalibs
is not None:
532 self.log.info(
"Outputting photoCalib files.")
534 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
535 butler.put(photoCalib,
'fgcm_photoCalib',
536 dataId={visitDataRefName: visit,
537 ccdDataRefName: detector,
538 'filter': physicalFilter})
540 self.log.info(
"Done outputting photoCalib files.")
542 if struct.atmospheres
is not None:
543 self.log.info(
"Outputting atmosphere transmission files.")
544 for visit, atm
in struct.atmospheres:
545 butler.put(atm,
"transmission_atmosphere_fgcm",
546 dataId={visitDataRefName: visit})
547 self.log.info(
"Done outputting atmosphere transmissions.")
549 return pipeBase.Struct(offsets=struct.offsets)
551 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
552 """Run the output products task.
557 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
558 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
559 dataRef dictionary with keys:
562 Camera object (`lsst.afw.cameraGeom.Camera`)
563 ``"fgcmLookUpTable"``
564 dataRef for the FGCM look-up table.
565 ``"fgcmVisitCatalog"``
566 dataRef for visit summary catalog.
567 ``"fgcmStandardStars"``
568 dataRef for the output standard star catalog.
570 dataRef for the zeropoint data catalog.
571 ``"fgcmAtmosphereParameters"``
572 dataRef for the atmosphere parameter catalog.
573 ``"fgcmBuildStarsTableConfig"``
574 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
575 physicalFilterMap : `dict`
576 Dictionary of mappings from physical filter to FGCM band.
577 returnCatalogs : `bool`, optional
578 Return photoCalibs as per-visit exposure catalogs.
579 butler : `lsst.daf.persistence.Butler`, optional
580 Gen2 butler used for reference star outputs
584 retStruct : `lsst.pipe.base.Struct`
585 Output structure with keys:
587 offsets : `np.ndarray`
588 Final reference offsets, per band.
589 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
590 Generator that returns (visit, transmissionCurve) tuples.
591 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
592 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
593 (returned if returnCatalogs is False).
594 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
595 Generator that returns (visit, exposureCatalog) tuples.
596 (returned if returnCatalogs is True).
598 stdCat = dataRefDict[
'fgcmStandardStars'].get()
599 md = stdCat.getMetadata()
600 bands = md.getArray(
'BANDS')
602 if self.config.doReferenceCalibration:
603 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
604 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands)
606 offsets = np.zeros(len(bands))
609 if self.config.doRefcatOutput
and butler
is not None:
610 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
614 if self.config.doZeropointOutput:
615 zptCat = dataRefDict[
'fgcmZeropoints'].get()
616 visitCat = dataRefDict[
'fgcmVisitCatalog'].get()
618 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
619 physicalFilterMap, returnCatalogs=returnCatalogs)
623 if self.config.doAtmosphereOutput:
624 atmCat = dataRefDict[
'fgcmAtmosphereParameters'].get()
625 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
629 retStruct = pipeBase.Struct(offsets=offsets,
632 retStruct.photoCalibCatalogs = pcgen
634 retStruct.photoCalibs = pcgen
638 def generateTractOutputProducts(self, dataRefDict, tract,
639 visitCat, zptCat, atmCat, stdCat,
640 fgcmBuildStarsConfig,
644 Generate the output products for a given tract, as specified in the config.
646 This method is here to have an alternate entry-point for
652 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
653 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
654 dataRef dictionary with keys:
657 Camera object (`lsst.afw.cameraGeom.Camera`)
658 ``"fgcmLookUpTable"``
659 dataRef for the FGCM look-up table.
662 visitCat : `lsst.afw.table.BaseCatalog`
663 FGCM visitCat from `FgcmBuildStarsTask`
664 zptCat : `lsst.afw.table.BaseCatalog`
665 FGCM zeropoint catalog from `FgcmFitCycleTask`
666 atmCat : `lsst.afw.table.BaseCatalog`
667 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
668 stdCat : `lsst.afw.table.SimpleCatalog`
669 FGCM standard star catalog from `FgcmFitCycleTask`
670 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
671 Configuration object from `FgcmBuildStarsTask`
672 returnCatalogs : `bool`, optional
673 Return photoCalibs as per-visit exposure catalogs.
674 butler: `lsst.daf.persistence.Butler`, optional
675 Gen2 butler used for reference star outputs
679 retStruct : `lsst.pipe.base.Struct`
680 Output structure with keys:
682 offsets : `np.ndarray`
683 Final reference offsets, per band.
684 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
685 Generator that returns (visit, transmissionCurve) tuples.
686 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
687 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
688 (returned if returnCatalogs is False).
689 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
690 Generator that returns (visit, exposureCatalog) tuples.
691 (returned if returnCatalogs is True).
693 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
695 md = stdCat.getMetadata()
696 bands = md.getArray(
'BANDS')
698 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
699 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
700 "in fgcmBuildStarsTask.")
702 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
703 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
705 if self.config.doReferenceCalibration:
706 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
707 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap)
709 offsets = np.zeros(len(bands))
711 if self.config.doRefcatOutput
and butler
is not None:
713 datasetConfig = copy.copy(self.config.datasetConfig)
714 datasetConfig.ref_dataset_name =
'%s_%d' % (self.config.datasetConfig.ref_dataset_name,
716 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
718 if self.config.doZeropointOutput:
719 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
720 physicalFilterMap, returnCatalogs=returnCatalogs)
724 if self.config.doAtmosphereOutput:
725 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
729 retStruct = pipeBase.Struct(offsets=offsets,
732 retStruct.photoCalibCatalogs = pcgen
734 retStruct.photoCalibs = pcgen
738 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
740 Compute offsets relative to a reference catalog.
742 This method splits the star catalog into healpix pixels
743 and computes the calibration transfer for a sample of
744 these pixels to approximate the 'absolute' calibration
745 values (on for each band) to apply to transfer the
750 stdCat : `lsst.afw.table.SimpleCatalog`
752 lutCat : `lsst.afw.table.SimpleCatalog`
754 physicalFilterMap : `dict`
755 Dictionary of mappings from physical filter to FGCM band.
756 bands : `list` [`str`]
757 List of band names from FGCM output
760 offsets : `numpy.array` of floats
761 Per band zeropoint offsets
767 minObs = stdCat[
'ngood'].min(axis=1)
769 goodStars = (minObs >= 1)
770 stdCat = stdCat[goodStars]
772 self.log.info(
"Found %d stars with at least 1 good observation in each band" %
779 lutPhysicalFilters = lutCat[0][
'physicalFilters'].split(
',')
780 lutStdPhysicalFilters = lutCat[0][
'stdPhysicalFilters'].split(
',')
781 physicalFilterMapBands = list(physicalFilterMap.values())
782 physicalFilterMapFilters = list(physicalFilterMap.keys())
786 physicalFilterMapIndex = physicalFilterMapBands.index(band)
787 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
789 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
790 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
791 filterLabels.append(afwImage.FilterLabel(band=band,
792 physical=stdPhysicalFilter))
801 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
802 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
803 sourceMapper.editOutputSchema().addField(
'instFlux', type=np.float64,
804 doc=
"instrumental flux (counts)")
805 sourceMapper.editOutputSchema().addField(
'instFluxErr', type=np.float64,
806 doc=
"instrumental flux error (counts)")
807 badStarKey = sourceMapper.editOutputSchema().addField(
'flag_badStar',
815 theta = np.pi/2. - stdCat[
'coord_dec']
816 phi = stdCat[
'coord_ra']
818 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
819 h, rev = esutil.stat.histogram(ipring, rev=
True)
821 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
823 self.log.info(
"Found %d pixels (nside=%d) with at least %d good stars" %
825 self.config.referencePixelizationNside,
826 self.config.referencePixelizationMinStars))
828 if gdpix.size < self.config.referencePixelizationNPixels:
829 self.log.warn(
"Found fewer good pixels (%d) than preferred in configuration (%d)" %
830 (gdpix.size, self.config.referencePixelizationNPixels))
833 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=
False)
835 results = np.zeros(gdpix.size, dtype=[(
'hpix',
'i4'),
836 (
'nstar',
'i4', len(bands)),
837 (
'nmatch',
'i4', len(bands)),
838 (
'zp',
'f4', len(bands)),
839 (
'zpErr',
'f4', len(bands))])
840 results[
'hpix'] = ipring[rev[rev[gdpix]]]
843 selected = np.zeros(len(stdCat), dtype=bool)
845 refFluxFields = [
None]*len(bands)
847 for p_index, pix
in enumerate(gdpix):
848 i1a = rev[rev[pix]: rev[pix + 1]]
856 for b_index, filterLabel
in enumerate(filterLabels):
857 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index,
859 selected, refFluxFields)
860 results[
'nstar'][p_index, b_index] = len(i1a)
861 results[
'nmatch'][p_index, b_index] = len(struct.arrays.refMag)
862 results[
'zp'][p_index, b_index] = struct.zp
863 results[
'zpErr'][p_index, b_index] = struct.sigma
866 offsets = np.zeros(len(bands))
868 for b_index, band
in enumerate(bands):
870 ok, = np.where(results[
'nmatch'][:, b_index] >= self.config.referenceMinMatch)
871 offsets[b_index] = np.median(results[
'zp'][ok, b_index])
874 madSigma = 1.4826*np.median(np.abs(results[
'zp'][ok, b_index] - offsets[b_index]))
875 self.log.info(
"Reference catalog offset for %s band: %.12f +/- %.12f",
876 band, offsets[b_index], madSigma)
880 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
881 b_index, filterLabel, stdCat, selected, refFluxFields):
883 Compute the zeropoint offset between the fgcm stdCat and the reference
884 stars for one pixel in one band
888 sourceMapper : `lsst.afw.table.SchemaMapper`
889 Mapper to go from stdCat to calibratable catalog
890 badStarKey : `lsst.afw.table.Key`
891 Key for the field with bad stars
893 Index of the band in the star catalog
894 filterLabel : `lsst.afw.image.FilterLabel`
895 filterLabel with band and physical filter
896 stdCat : `lsst.afw.table.SimpleCatalog`
898 selected : `numpy.array(dtype=bool)`
899 Boolean array of which stars are in the pixel
900 refFluxFields : `list`
901 List of names of flux fields for reference catalog
904 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
905 sourceCat.reserve(selected.sum())
906 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
907 sourceCat[
'instFlux'] = 10.**(stdCat[
'mag_std_noabs'][selected, b_index]/(-2.5))
908 sourceCat[
'instFluxErr'] = (np.log(10.)/2.5)*(stdCat[
'magErr_std'][selected, b_index]
909 * sourceCat[
'instFlux'])
913 badStar = (stdCat[
'mag_std_noabs'][selected, b_index] > 90.0)
914 for rec
in sourceCat[badStar]:
915 rec.set(badStarKey,
True)
917 exposure = afwImage.ExposureF()
918 exposure.setFilterLabel(filterLabel)
920 if refFluxFields[b_index]
is None:
923 ctr = stdCat[0].getCoord()
924 rad = 0.05*lsst.geom.degrees
925 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel)
926 refFluxFields[b_index] = refDataTest.fluxField
929 calConfig = copy.copy(self.config.photoCal.value)
930 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index]
931 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] +
'Err'
932 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
934 schema=sourceCat.getSchema())
936 struct = calTask.run(exposure, sourceCat)
940 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
942 Output standard stars in indexed reference catalog format.
943 This is not currently supported in Gen3.
947 butler : `lsst.daf.persistence.Butler`
948 stdCat : `lsst.afw.table.SimpleCatalog`
949 FGCM standard star catalog from fgcmFitCycleTask
950 offsets : `numpy.array` of floats
951 Per band zeropoint offsets
952 bands : `list` [`str`]
953 List of band names from FGCM output
954 datasetConfig : `lsst.meas.algorithms.DatasetConfig`
955 Config for reference dataset
958 self.log.info(
"Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
960 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
961 self.config.datasetConfig.indexer.active)
968 conv = stdCat[0][
'coord_ra'].asDegrees()/float(stdCat[0][
'coord_ra'])
969 indices = np.array(indexer.indexPoints(stdCat[
'coord_ra']*conv,
970 stdCat[
'coord_dec']*conv))
972 formattedCat = self._formatCatalog(stdCat, offsets, bands)
975 dataId = indexer.makeDataId(
'master_schema',
976 datasetConfig.ref_dataset_name)
977 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
978 addRefCatMetadata(masterCat)
979 butler.put(masterCat,
'ref_cat', dataId=dataId)
982 h, rev = esutil.stat.histogram(indices, rev=
True)
983 gd, = np.where(h > 0)
984 selected = np.zeros(len(formattedCat), dtype=bool)
986 i1a = rev[rev[i]: rev[i + 1]]
995 dataId = indexer.makeDataId(indices[i1a[0]],
996 datasetConfig.ref_dataset_name)
997 butler.put(formattedCat[selected],
'ref_cat', dataId=dataId)
1000 dataId = indexer.makeDataId(
None, datasetConfig.ref_dataset_name)
1001 butler.put(datasetConfig,
'ref_cat_config', dataId=dataId)
1003 self.log.info(
"Done outputting standard stars.")
1005 def _formatCatalog(self, fgcmStarCat, offsets, bands):
1007 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
1011 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
1012 SimpleCatalog as output by fgcmcal
1013 offsets : `list` with len(self.bands) entries
1014 Zeropoint offsets to apply
1015 bands : `list` [`str`]
1016 List of band names from FGCM output
1020 formattedCat: `lsst.afw.table.SimpleCatalog`
1021 SimpleCatalog suitable for using as a reference catalog
1024 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1025 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1029 sourceMapper.addMinimalSchema(minSchema)
1031 sourceMapper.editOutputSchema().addField(
'%s_nGood' % (band), type=np.int32)
1032 sourceMapper.editOutputSchema().addField(
'%s_nTotal' % (band), type=np.int32)
1033 sourceMapper.editOutputSchema().addField(
'%s_nPsfCandidate' % (band), type=np.int32)
1035 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1036 formattedCat.reserve(len(fgcmStarCat))
1037 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1041 for b, band
in enumerate(bands):
1042 mag = fgcmStarCat[
'mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1045 flux = (mag*units.ABmag).to_value(units.nJy)
1046 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat[
'magErr_std'][:, b].astype(np.float64)
1048 formattedCat[
'%s_flux' % (band)][:] = flux
1049 formattedCat[
'%s_fluxErr' % (band)][:] = fluxErr
1050 formattedCat[
'%s_nGood' % (band)][:] = fgcmStarCat[
'ngood'][:, b]
1051 formattedCat[
'%s_nTotal' % (band)][:] = fgcmStarCat[
'ntotal'][:, b]
1052 formattedCat[
'%s_nPsfCandidate' % (band)][:] = fgcmStarCat[
'npsfcand'][:, b]
1054 addRefCatMetadata(formattedCat)
1058 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1059 physicalFilterMap, returnCatalogs=True,
1061 """Output the zeropoints in fgcm_photoCalib format.
1065 camera : `lsst.afw.cameraGeom.Camera`
1066 Camera from the butler.
1067 zptCat : `lsst.afw.table.BaseCatalog`
1068 FGCM zeropoint catalog from `FgcmFitCycleTask`.
1069 visitCat : `lsst.afw.table.BaseCatalog`
1070 FGCM visitCat from `FgcmBuildStarsTask`.
1071 offsets : `numpy.array`
1072 Float array of absolute calibration offsets, one for each filter.
1073 bands : `list` [`str`]
1074 List of band names from FGCM output.
1075 physicalFilterMap : `dict`
1076 Dictionary of mappings from physical filter to FGCM band.
1077 returnCatalogs : `bool`, optional
1078 Return photoCalibs as per-visit exposure catalogs.
1079 tract: `int`, optional
1080 Tract number to output. Default is None (global calibration)
1084 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1085 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1086 (returned if returnCatalogs is False).
1087 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1088 Generator that returns (visit, exposureCatalog) tuples.
1089 (returned if returnCatalogs is True).
1094 cannot_compute = fgcm.fgcmUtilities.zpFlagDict[
'CANNOT_COMPUTE_ZEROPOINT']
1095 selected = (((zptCat[
'fgcmFlag'] & cannot_compute) == 0)
1096 & (zptCat[
'fgcmZptVar'] > 0.0))
1099 badVisits = np.unique(zptCat[
'visit'][~selected])
1100 goodVisits = np.unique(zptCat[
'visit'][selected])
1101 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1102 for allBadVisit
in allBadVisits:
1103 self.log.warn(f
'No suitable photoCalib for visit {allBadVisit}')
1107 for f
in physicalFilterMap:
1109 if physicalFilterMap[f]
in bands:
1110 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1114 for ccdIndex, detector
in enumerate(camera):
1115 ccdMapping[detector.getId()] = ccdIndex
1119 for rec
in visitCat:
1120 scalingMapping[rec[
'visit']] = rec[
'scaling']
1122 if self.config.doComposeWcsJacobian:
1127 zptVisitCatalog =
None
1129 metadata = dafBase.PropertyList()
1130 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
1131 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
1133 for rec
in zptCat[selected]:
1135 scaling = scalingMapping[rec[
'visit']][ccdMapping[rec[
'detector']]]
1142 postCalibrationOffset = offsetMapping[rec[
'filtername']]
1143 if self.config.doApplyMeanChromaticCorrection:
1144 postCalibrationOffset += rec[
'fgcmDeltaChrom']
1146 fgcmSuperStarField = self._getChebyshevBoundedField(rec[
'fgcmfZptSstarCheb'],
1147 rec[
'fgcmfZptChebXyMax'])
1149 fgcmZptField = self._getChebyshevBoundedField((rec[
'fgcmfZptCheb']*units.AB).to_value(units.nJy),
1150 rec[
'fgcmfZptChebXyMax'],
1151 offset=postCalibrationOffset,
1154 if self.config.doComposeWcsJacobian:
1156 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec[
'detector']],
1162 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1165 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1166 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec[
'fgcmZptVar'])
1167 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1168 calibrationErr=calibErr,
1169 calibration=fgcmField,
1172 if not returnCatalogs:
1174 yield (int(rec[
'visit']), int(rec[
'detector']), rec[
'filtername'], photoCalib)
1177 if rec[
'visit'] != lastVisit:
1182 zptVisitCatalog.sort()
1183 yield (int(lastVisit), zptVisitCatalog)
1186 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1187 zptExpCatSchema.addField(
'visit', type=
'I', doc=
'Visit number')
1190 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1191 zptVisitCatalog.setMetadata(metadata)
1193 lastVisit = int(rec[
'visit'])
1195 catRecord = zptVisitCatalog.addNew()
1196 catRecord[
'id'] = int(rec[
'detector'])
1197 catRecord[
'visit'] = rec[
'visit']
1198 catRecord.setPhotoCalib(photoCalib)
1203 zptVisitCatalog.sort()
1204 yield (int(lastVisit), zptVisitCatalog)
1206 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1208 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1213 coefficients: `numpy.array`
1214 Flattened array of chebyshev coefficients
1215 xyMax: `list` of length 2
1216 Maximum x and y of the chebyshev bounding box
1217 offset: `float`, optional
1218 Absolute calibration offset. Default is 0.0
1219 scaling: `float`, optional
1220 Flat scaling value from fgcmBuildStars. Default is 1.0
1224 boundedField: `lsst.afw.math.ChebyshevBoundedField`
1227 orderPlus1 = int(np.sqrt(coefficients.size))
1228 pars = np.zeros((orderPlus1, orderPlus1))
1230 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1231 lsst.geom.Point2I(*xyMax))
1233 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1234 * (10.**(offset/-2.5))*scaling)
1236 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1240 def _outputAtmospheres(self, dataRefDict, atmCat):
1242 Output the atmospheres.
1246 dataRefDict : `dict`
1247 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1248 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1249 dataRef dictionary with keys:
1251 ``"fgcmLookUpTable"``
1252 dataRef for the FGCM look-up table.
1253 atmCat : `lsst.afw.table.BaseCatalog`
1254 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1258 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1259 Generator that returns (visit, transmissionCurve) tuples.
1262 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
1264 atmosphereTableName = lutCat[0][
'tablename']
1265 elevation = lutCat[0][
'elevation']
1266 atmLambda = lutCat[0][
'atmLambda']
1271 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1272 atmTable.loadTable()
1276 if atmTable
is None:
1279 modGen = fgcm.ModtranGenerator(elevation)
1280 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1281 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1282 except (ValueError, IOError)
as e:
1283 raise RuntimeError(
"FGCM look-up-table generated with modtran, "
1284 "but modtran not configured to run.")
from e
1286 zenith = np.degrees(np.arccos(1./atmCat[
'secZenith']))
1288 for i, visit
in enumerate(atmCat[
'visit']):
1289 if atmTable
is not None:
1291 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i][
'pmb'],
1292 pwv=atmCat[i][
'pwv'],
1294 tau=atmCat[i][
'tau'],
1295 alpha=atmCat[i][
'alpha'],
1297 ctranslamstd=[atmCat[i][
'cTrans'],
1298 atmCat[i][
'lamStd']])
1301 modAtm = modGen(pmb=atmCat[i][
'pmb'],
1302 pwv=atmCat[i][
'pwv'],
1304 tau=atmCat[i][
'tau'],
1305 alpha=atmCat[i][
'alpha'],
1307 lambdaRange=lambdaRange,
1308 lambdaStep=lambdaStep,
1309 ctranslamstd=[atmCat[i][
'cTrans'],
1310 atmCat[i][
'lamStd']])
1311 atmVals = modAtm[
'COMBINED']
1314 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1315 wavelengths=atmLambda,
1316 throughputAtMin=atmVals[0],
1317 throughputAtMax=atmVals[-1])
1319 yield (int(visit), curve)
def computeApproxPixelAreaFields(camera)