23 """Make the final fgcmcal output products.
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
41 from astropy
import units
43 import lsst.pex.config
as pexConfig
44 import lsst.pipe.base
as pipeBase
45 from lsst.pipe.base
import connectionTypes
46 from lsst.afw.image
import TransmissionCurve
47 from lsst.meas.algorithms
import LoadIndexedReferenceObjectsTask
48 from lsst.meas.algorithms
import ReferenceObjectLoader
49 from lsst.pipe.tasks.photoCal
import PhotoCalTask
51 import lsst.afw.image
as afwImage
52 import lsst.afw.math
as afwMath
53 import lsst.afw.table
as afwTable
54 from lsst.meas.algorithms
import IndexerRegistry
55 from lsst.meas.algorithms
import DatasetConfig
56 from lsst.meas.algorithms.ingestIndexReferenceTask
import addRefCatMetadata
58 from .utilities
import computeApproxPixelAreaFields
59 from .utilities
import lookupStaticCalibrations
63 __all__ = [
'FgcmOutputProductsConfig',
'FgcmOutputProductsTask',
'FgcmOutputProductsRunner']
67 dimensions=(
"instrument",),
68 defaultTemplates={
"cycleNumber":
"0"}):
69 camera = connectionTypes.PrerequisiteInput(
70 doc=
"Camera instrument",
72 storageClass=
"Camera",
73 dimensions=(
"instrument",),
74 lookupFunction=lookupStaticCalibrations,
78 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
79 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
80 "chromatic corrections."),
81 name=
"fgcmLookUpTable",
82 storageClass=
"Catalog",
83 dimensions=(
"instrument",),
87 fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
88 doc=
"Catalog of visit information for fgcm",
89 name=
"fgcmVisitCatalog",
90 storageClass=
"Catalog",
91 dimensions=(
"instrument",),
95 fgcmStandardStars = connectionTypes.PrerequisiteInput(
96 doc=
"Catalog of standard star data from fgcm fit",
97 name=
"fgcmStandardStars{cycleNumber}",
98 storageClass=
"SimpleCatalog",
99 dimensions=(
"instrument",),
103 fgcmZeropoints = connectionTypes.PrerequisiteInput(
104 doc=
"Catalog of zeropoints from fgcm fit",
105 name=
"fgcmZeropoints{cycleNumber}",
106 storageClass=
"Catalog",
107 dimensions=(
"instrument",),
111 fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
112 doc=
"Catalog of atmosphere parameters from fgcm fit",
113 name=
"fgcmAtmosphereParameters{cycleNumber}",
114 storageClass=
"Catalog",
115 dimensions=(
"instrument",),
119 refCat = connectionTypes.PrerequisiteInput(
120 doc=
"Reference catalog to use for photometric calibration",
122 storageClass=
"SimpleCatalog",
123 dimensions=(
"skypix",),
128 fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
129 doc=
"Config used to build FGCM input stars",
130 name=
"fgcmBuildStarsTable_config",
131 storageClass=
"Config",
134 fgcmPhotoCalib = connectionTypes.Output(
135 doc=
"Per-visit photoCalib exposure catalogs produced from fgcm calibration",
136 name=
"fgcmPhotoCalibCatalog",
137 storageClass=
"ExposureCatalog",
138 dimensions=(
"instrument",
"visit",),
142 fgcmTransmissionAtmosphere = connectionTypes.Output(
143 doc=
"Per-visit atmosphere transmission files produced from fgcm calibration",
144 name=
"transmission_atmosphere_fgcm",
145 storageClass=
"TransmissionCurve",
146 dimensions=(
"instrument",
151 fgcmOffsets = connectionTypes.Output(
152 doc=
"Per-band offsets computed from doReferenceCalibration",
153 name=
"fgcmReferenceCalibrationOffsets",
154 storageClass=
"Catalog",
155 dimensions=(
"instrument",),
159 def __init__(self, *, config=None):
160 super().__init__(config=config)
162 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
163 raise ValueError(
"cycleNumber must be of integer format")
164 if config.connections.refCat != config.refObjLoader.ref_dataset_name:
165 raise ValueError(
"connections.refCat must be the same as refObjLoader.ref_dataset_name")
167 if config.doRefcatOutput:
168 raise ValueError(
"FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
170 if not config.doReferenceCalibration:
171 self.prerequisiteInputs.remove(
"refCat")
172 if not config.doAtmosphereOutput:
173 self.prerequisiteInputs.remove(
"fgcmAtmosphereParameters")
174 if not config.doZeropointOutput:
175 self.prerequisiteInputs.remove(
"fgcmZeropoints")
176 if not config.doReferenceCalibration:
177 self.outputs.remove(
"fgcmOffsets")
180 class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
181 pipelineConnections=FgcmOutputProductsConnections):
182 """Config for FgcmOutputProductsTask"""
184 cycleNumber = pexConfig.Field(
185 doc=
"Final fit cycle from FGCM fit",
192 doReferenceCalibration = pexConfig.Field(
193 doc=(
"Transfer 'absolute' calibration from reference catalog? "
194 "This afterburner step is unnecessary if reference stars "
195 "were used in the full fit in FgcmFitCycleTask."),
199 doRefcatOutput = pexConfig.Field(
200 doc=
"Output standard stars in reference catalog format",
204 doAtmosphereOutput = pexConfig.Field(
205 doc=
"Output atmospheres in transmission_atmosphere_fgcm format",
209 doZeropointOutput = pexConfig.Field(
210 doc=
"Output zeropoints in fgcm_photoCalib format",
214 doComposeWcsJacobian = pexConfig.Field(
215 doc=
"Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
219 doApplyMeanChromaticCorrection = pexConfig.Field(
220 doc=
"Apply the mean chromatic correction to the zeropoints?",
224 refObjLoader = pexConfig.ConfigurableField(
225 target=LoadIndexedReferenceObjectsTask,
226 doc=
"reference object loader for 'absolute' photometric calibration",
228 photoCal = pexConfig.ConfigurableField(
230 doc=
"task to perform 'absolute' calibration",
232 referencePixelizationNside = pexConfig.Field(
233 doc=
"Healpix nside to pixelize catalog to compare to reference catalog",
237 referencePixelizationMinStars = pexConfig.Field(
238 doc=(
"Minimum number of stars per healpix pixel to select for comparison"
239 "to the specified reference catalog"),
243 referenceMinMatch = pexConfig.Field(
244 doc=
"Minimum number of stars matched to reference catalog to be used in statistics",
248 referencePixelizationNPixels = pexConfig.Field(
249 doc=(
"Number of healpix pixels to sample to do comparison. "
250 "Doing too many will take a long time and not yield any more "
251 "precise results because the final number is the median offset "
252 "(per band) from the set of pixels."),
256 datasetConfig = pexConfig.ConfigField(
258 doc=
"Configuration for writing/reading ingested catalog",
261 def setDefaults(self):
262 pexConfig.Config.setDefaults(self)
272 self.photoCal.applyColorTerms =
False
273 self.photoCal.fluxField =
'instFlux'
274 self.photoCal.magErrFloor = 0.003
275 self.photoCal.match.referenceSelection.doSignalToNoise =
True
276 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
277 self.photoCal.match.sourceSelection.doSignalToNoise =
True
278 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
279 self.photoCal.match.sourceSelection.signalToNoise.fluxField =
'instFlux'
280 self.photoCal.match.sourceSelection.signalToNoise.errField =
'instFluxErr'
281 self.photoCal.match.sourceSelection.doFlags =
True
282 self.photoCal.match.sourceSelection.flags.good = []
283 self.photoCal.match.sourceSelection.flags.bad = [
'flag_badStar']
284 self.photoCal.match.sourceSelection.doUnresolved =
False
285 self.datasetConfig.ref_dataset_name =
'fgcm_stars'
286 self.datasetConfig.format_version = 1
292 self.connections.cycleNumber = str(self.cycleNumber)
295 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
296 """Subclass of TaskRunner for fgcmOutputProductsTask
298 fgcmOutputProductsTask.run() takes one argument, the butler, and
299 does not run on any data in the repository.
300 This runner does not use any parallelization.
304 def getTargetList(parsedCmd):
306 Return a list with one element, the butler.
308 return [parsedCmd.butler]
310 def __call__(self, butler):
314 butler: `lsst.daf.persistence.Butler`
318 exitStatus: `list` with `pipeBase.Struct`
319 exitStatus (0: success; 1: failure)
320 if self.doReturnResults also
321 results (`np.array` with absolute zeropoint offsets)
323 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
327 results = task.runDataRef(butler)
330 results = task.runDataRef(butler)
331 except Exception
as e:
333 task.log.fatal(
"Failed: %s" % e)
334 if not isinstance(e, pipeBase.TaskError):
335 traceback.print_exc(file=sys.stderr)
337 task.writeMetadata(butler)
339 if self.doReturnResults:
341 return [pipeBase.Struct(exitStatus=exitStatus,
344 return [pipeBase.Struct(exitStatus=exitStatus)]
346 def run(self, parsedCmd):
348 Run the task, with no multiprocessing
352 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
357 if self.precall(parsedCmd):
358 targetList = self.getTargetList(parsedCmd)
360 resultList = self(targetList[0])
365 class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
367 Output products from FGCM global calibration.
370 ConfigClass = FgcmOutputProductsConfig
371 RunnerClass = FgcmOutputProductsRunner
372 _DefaultName =
"fgcmOutputProducts"
374 def __init__(self, butler=None, **kwargs):
375 super().__init__(**kwargs)
378 def _getMetadataName(self):
381 def runQuantum(self, butlerQC, inputRefs, outputRefs):
383 dataRefDict[
'camera'] = butlerQC.get(inputRefs.camera)
384 dataRefDict[
'fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
385 dataRefDict[
'fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
386 dataRefDict[
'fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
388 if self.config.doZeropointOutput:
389 dataRefDict[
'fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
390 photoCalibRefDict = {photoCalibRef.dataId.byName()[
'visit']:
391 photoCalibRef
for photoCalibRef
in outputRefs.fgcmPhotoCalib}
393 if self.config.doAtmosphereOutput:
394 dataRefDict[
'fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
395 atmRefDict = {atmRef.dataId.byName()[
'visit']: atmRef
for
396 atmRef
in outputRefs.fgcmTransmissionAtmosphere}
398 if self.config.doReferenceCalibration:
399 refConfig = self.config.refObjLoader
400 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
401 for ref
in inputRefs.refCat],
402 refCats=butlerQC.get(inputRefs.refCat),
406 self.refObjLoader =
None
408 dataRefDict[
'fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
410 fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
411 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
413 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
414 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
415 "in fgcmBuildStarsTask.")
416 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
417 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
419 struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=
True)
422 if struct.photoCalibCatalogs
is not None:
423 self.log.info(
"Outputting photoCalib catalogs.")
424 for visit, expCatalog
in struct.photoCalibCatalogs:
425 butlerQC.put(expCatalog, photoCalibRefDict[visit])
426 self.log.info(
"Done outputting photoCalib catalogs.")
429 if struct.atmospheres
is not None:
430 self.log.info(
"Outputting atmosphere transmission files.")
431 for visit, atm
in struct.atmospheres:
432 butlerQC.put(atm, atmRefDict[visit])
433 self.log.info(
"Done outputting atmosphere files.")
435 if self.config.doReferenceCalibration:
437 schema = afwTable.Schema()
438 schema.addField(
'offset', type=np.float64,
439 doc=
"Post-process calibration offset (mag)")
440 offsetCat = afwTable.BaseCatalog(schema)
441 offsetCat.resize(len(struct.offsets))
442 offsetCat[
'offset'][:] = struct.offsets
444 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
449 def runDataRef(self, butler):
451 Make FGCM output products for use in the stack
455 butler: `lsst.daf.persistence.Butler`
457 Final fit cycle number, override config.
461 offsets: `lsst.pipe.base.Struct`
462 A structure with array of zeropoint offsets
467 Raised if any one of the following is true:
469 - butler cannot find "fgcmBuildStars_config" or
470 "fgcmBuildStarsTable_config".
471 - butler cannot find "fgcmFitCycle_config".
472 - "fgcmFitCycle_config" does not refer to
473 `self.config.cycleNumber`.
474 - butler cannot find "fgcmAtmosphereParameters" and
475 `self.config.doAtmosphereOutput` is `True`.
476 - butler cannot find "fgcmStandardStars" and
477 `self.config.doReferenceCalibration` is `True` or
478 `self.config.doRefcatOutput` is `True`.
479 - butler cannot find "fgcmZeropoints" and
480 `self.config.doZeropointOutput` is `True`.
482 if self.config.doReferenceCalibration:
484 self.makeSubtask(
"refObjLoader", butler=butler)
488 if not butler.datasetExists(
'fgcmBuildStarsTable_config')
and \
489 not butler.datasetExists(
'fgcmBuildStars_config'):
490 raise RuntimeError(
"Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
491 "which is prereq for fgcmOutputProducts")
493 if butler.datasetExists(
'fgcmBuildStarsTable_config'):
494 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStarsTable_config')
496 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStars_config')
497 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
498 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
499 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
501 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
502 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
503 "in fgcmBuildStarsTask.")
505 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
506 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
509 if (self.config.doAtmosphereOutput
510 and not butler.datasetExists(
'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
511 raise RuntimeError(f
"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
513 if not butler.datasetExists(
'fgcmStandardStars',
514 fgcmcycle=self.config.cycleNumber):
515 raise RuntimeError(
"Standard stars are missing for cycle %d." %
516 (self.config.cycleNumber))
518 if (self.config.doZeropointOutput
519 and (
not butler.datasetExists(
'fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
520 raise RuntimeError(
"Zeropoints are missing for cycle %d." %
521 (self.config.cycleNumber))
525 dataRefDict[
'camera'] = butler.get(
'camera')
526 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
527 dataRefDict[
'fgcmVisitCatalog'] = butler.dataRef(
'fgcmVisitCatalog')
528 dataRefDict[
'fgcmStandardStars'] = butler.dataRef(
'fgcmStandardStars',
529 fgcmcycle=self.config.cycleNumber)
531 if self.config.doZeropointOutput:
532 dataRefDict[
'fgcmZeropoints'] = butler.dataRef(
'fgcmZeropoints',
533 fgcmcycle=self.config.cycleNumber)
534 if self.config.doAtmosphereOutput:
535 dataRefDict[
'fgcmAtmosphereParameters'] = butler.dataRef(
'fgcmAtmosphereParameters',
536 fgcmcycle=self.config.cycleNumber)
538 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=
False)
540 if struct.photoCalibs
is not None:
541 self.log.info(
"Outputting photoCalib files.")
543 for visit, detector, physicalFilter, photoCalib
in struct.photoCalibs:
544 butler.put(photoCalib,
'fgcm_photoCalib',
545 dataId={visitDataRefName: visit,
546 ccdDataRefName: detector,
547 'filter': physicalFilter})
549 self.log.info(
"Done outputting photoCalib files.")
551 if struct.atmospheres
is not None:
552 self.log.info(
"Outputting atmosphere transmission files.")
553 for visit, atm
in struct.atmospheres:
554 butler.put(atm,
"transmission_atmosphere_fgcm",
555 dataId={visitDataRefName: visit})
556 self.log.info(
"Done outputting atmosphere transmissions.")
558 return pipeBase.Struct(offsets=struct.offsets)
560 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
561 """Run the output products task.
566 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
567 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
568 dataRef dictionary with keys:
571 Camera object (`lsst.afw.cameraGeom.Camera`)
572 ``"fgcmLookUpTable"``
573 dataRef for the FGCM look-up table.
574 ``"fgcmVisitCatalog"``
575 dataRef for visit summary catalog.
576 ``"fgcmStandardStars"``
577 dataRef for the output standard star catalog.
579 dataRef for the zeropoint data catalog.
580 ``"fgcmAtmosphereParameters"``
581 dataRef for the atmosphere parameter catalog.
582 ``"fgcmBuildStarsTableConfig"``
583 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
584 physicalFilterMap : `dict`
585 Dictionary of mappings from physical filter to FGCM band.
586 returnCatalogs : `bool`, optional
587 Return photoCalibs as per-visit exposure catalogs.
588 butler : `lsst.daf.persistence.Butler`, optional
589 Gen2 butler used for reference star outputs
593 retStruct : `lsst.pipe.base.Struct`
594 Output structure with keys:
596 offsets : `np.ndarray`
597 Final reference offsets, per band.
598 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
599 Generator that returns (visit, transmissionCurve) tuples.
600 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
601 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
602 (returned if returnCatalogs is False).
603 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
604 Generator that returns (visit, exposureCatalog) tuples.
605 (returned if returnCatalogs is True).
607 stdCat = dataRefDict[
'fgcmStandardStars'].get()
608 md = stdCat.getMetadata()
609 bands = md.getArray(
'BANDS')
611 if self.config.doReferenceCalibration:
612 offsets = self._computeReferenceOffsets(stdCat, bands)
614 offsets = np.zeros(len(bands))
617 if self.config.doRefcatOutput
and butler
is not None:
618 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
622 if self.config.doZeropointOutput:
623 zptCat = dataRefDict[
'fgcmZeropoints'].get()
624 visitCat = dataRefDict[
'fgcmVisitCatalog'].get()
626 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
627 physicalFilterMap, returnCatalogs=returnCatalogs)
631 if self.config.doAtmosphereOutput:
632 atmCat = dataRefDict[
'fgcmAtmosphereParameters'].get()
633 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
637 retStruct = pipeBase.Struct(offsets=offsets,
640 retStruct.photoCalibCatalogs = pcgen
642 retStruct.photoCalibs = pcgen
646 def generateTractOutputProducts(self, dataRefDict, tract,
647 visitCat, zptCat, atmCat, stdCat,
648 fgcmBuildStarsConfig,
652 Generate the output products for a given tract, as specified in the config.
654 This method is here to have an alternate entry-point for
660 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
661 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
662 dataRef dictionary with keys:
665 Camera object (`lsst.afw.cameraGeom.Camera`)
666 ``"fgcmLookUpTable"``
667 dataRef for the FGCM look-up table.
670 visitCat : `lsst.afw.table.BaseCatalog`
671 FGCM visitCat from `FgcmBuildStarsTask`
672 zptCat : `lsst.afw.table.BaseCatalog`
673 FGCM zeropoint catalog from `FgcmFitCycleTask`
674 atmCat : `lsst.afw.table.BaseCatalog`
675 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
676 stdCat : `lsst.afw.table.SimpleCatalog`
677 FGCM standard star catalog from `FgcmFitCycleTask`
678 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
679 Configuration object from `FgcmBuildStarsTask`
680 returnCatalogs : `bool`, optional
681 Return photoCalibs as per-visit exposure catalogs.
682 butler: `lsst.daf.persistence.Butler`, optional
683 Gen2 butler used for reference star outputs
687 retStruct : `lsst.pipe.base.Struct`
688 Output structure with keys:
690 offsets : `np.ndarray`
691 Final reference offsets, per band.
692 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
693 Generator that returns (visit, transmissionCurve) tuples.
694 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
695 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
696 (returned if returnCatalogs is False).
697 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
698 Generator that returns (visit, exposureCatalog) tuples.
699 (returned if returnCatalogs is True).
701 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
703 md = stdCat.getMetadata()
704 bands = md.getArray(
'BANDS')
706 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
707 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
708 "in fgcmBuildStarsTask.")
710 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
711 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
713 if self.config.doReferenceCalibration:
714 offsets = self._computeReferenceOffsets(stdCat, bands)
716 offsets = np.zeros(len(bands))
718 if self.config.doRefcatOutput
and butler
is not None:
720 datasetConfig = copy.copy(self.config.datasetConfig)
721 datasetConfig.ref_dataset_name =
'%s_%d' % (self.config.datasetConfig.ref_dataset_name,
723 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
725 if self.config.doZeropointOutput:
726 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
727 physicalFilterMap, returnCatalogs=returnCatalogs)
731 if self.config.doAtmosphereOutput:
732 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
736 retStruct = pipeBase.Struct(offsets=offsets,
739 retStruct.photoCalibCatalogs = pcgen
741 retStruct.photoCalibs = pcgen
745 def _computeReferenceOffsets(self, stdCat, bands):
747 Compute offsets relative to a reference catalog.
749 This method splits the star catalog into healpix pixels
750 and computes the calibration transfer for a sample of
751 these pixels to approximate the 'absolute' calibration
752 values (on for each band) to apply to transfer the
757 stdCat : `lsst.afw.table.SimpleCatalog`
759 bands : `list` [`str`]
760 List of band names from FGCM output
763 offsets : `numpy.array` of floats
764 Per band zeropoint offsets
770 minObs = stdCat[
'ngood'].min(axis=1)
772 goodStars = (minObs >= 1)
773 stdCat = stdCat[goodStars]
775 self.log.info(
"Found %d stars with at least 1 good observation in each band" %
785 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
786 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
787 sourceMapper.editOutputSchema().addField(
'instFlux', type=np.float64,
788 doc=
"instrumental flux (counts)")
789 sourceMapper.editOutputSchema().addField(
'instFluxErr', type=np.float64,
790 doc=
"instrumental flux error (counts)")
791 badStarKey = sourceMapper.editOutputSchema().addField(
'flag_badStar',
799 theta = np.pi/2. - stdCat[
'coord_dec']
800 phi = stdCat[
'coord_ra']
802 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
803 h, rev = esutil.stat.histogram(ipring, rev=
True)
805 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
807 self.log.info(
"Found %d pixels (nside=%d) with at least %d good stars" %
809 self.config.referencePixelizationNside,
810 self.config.referencePixelizationMinStars))
812 if gdpix.size < self.config.referencePixelizationNPixels:
813 self.log.warn(
"Found fewer good pixels (%d) than preferred in configuration (%d)" %
814 (gdpix.size, self.config.referencePixelizationNPixels))
817 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=
False)
819 results = np.zeros(gdpix.size, dtype=[(
'hpix',
'i4'),
820 (
'nstar',
'i4', len(bands)),
821 (
'nmatch',
'i4', len(bands)),
822 (
'zp',
'f4', len(bands)),
823 (
'zpErr',
'f4', len(bands))])
824 results[
'hpix'] = ipring[rev[rev[gdpix]]]
827 selected = np.zeros(len(stdCat), dtype=np.bool)
829 refFluxFields = [
None]*len(bands)
831 for p, pix
in enumerate(gdpix):
832 i1a = rev[rev[pix]: rev[pix + 1]]
840 for b, band
in enumerate(bands):
842 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
843 selected, refFluxFields)
844 results[
'nstar'][p, b] = len(i1a)
845 results[
'nmatch'][p, b] = len(struct.arrays.refMag)
846 results[
'zp'][p, b] = struct.zp
847 results[
'zpErr'][p, b] = struct.sigma
850 offsets = np.zeros(len(bands))
852 for b, band
in enumerate(bands):
854 ok, = np.where(results[
'nmatch'][:, b] >= self.config.referenceMinMatch)
855 offsets[b] = np.median(results[
'zp'][ok, b])
858 madSigma = 1.4826*np.median(np.abs(results[
'zp'][ok, b] - offsets[b]))
859 self.log.info(
"Reference catalog offset for %s band: %.12f +/- %.12f" %
860 (band, offsets[b], madSigma))
864 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
865 b, band, stdCat, selected, refFluxFields):
867 Compute the zeropoint offset between the fgcm stdCat and the reference
868 stars for one pixel in one band
872 sourceMapper: `lsst.afw.table.SchemaMapper`
873 Mapper to go from stdCat to calibratable catalog
874 badStarKey: `lsst.afw.table.Key`
875 Key for the field with bad stars
877 Index of the band in the star catalog
879 Name of band for reference catalog
880 stdCat: `lsst.afw.table.SimpleCatalog`
882 selected: `numpy.array(dtype=np.bool)`
883 Boolean array of which stars are in the pixel
884 refFluxFields: `list`
885 List of names of flux fields for reference catalog
888 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
889 sourceCat.reserve(selected.sum())
890 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
891 sourceCat[
'instFlux'] = 10.**(stdCat[
'mag_std_noabs'][selected, b]/(-2.5))
892 sourceCat[
'instFluxErr'] = (np.log(10.)/2.5)*(stdCat[
'magErr_std'][selected, b]
893 * sourceCat[
'instFlux'])
897 badStar = (stdCat[
'mag_std_noabs'][selected, b] > 90.0)
898 for rec
in sourceCat[badStar]:
899 rec.set(badStarKey,
True)
901 exposure = afwImage.ExposureF()
902 exposure.setFilter(afwImage.Filter(band))
904 if refFluxFields[b]
is None:
907 ctr = stdCat[0].getCoord()
908 rad = 0.05*lsst.geom.degrees
909 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
910 refFluxFields[b] = refDataTest.fluxField
913 calConfig = copy.copy(self.config.photoCal.value)
914 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
915 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] +
'Err'
916 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
918 schema=sourceCat.getSchema())
920 struct = calTask.run(exposure, sourceCat)
924 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
926 Output standard stars in indexed reference catalog format.
927 This is not currently supported in Gen3.
931 butler : `lsst.daf.persistence.Butler`
932 stdCat : `lsst.afw.table.SimpleCatalog`
933 FGCM standard star catalog from fgcmFitCycleTask
934 offsets : `numpy.array` of floats
935 Per band zeropoint offsets
936 bands : `list` [`str`]
937 List of band names from FGCM output
938 datasetConfig : `lsst.meas.algorithms.DatasetConfig`
939 Config for reference dataset
942 self.log.info(
"Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
944 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
945 self.config.datasetConfig.indexer.active)
952 conv = stdCat[0][
'coord_ra'].asDegrees()/float(stdCat[0][
'coord_ra'])
953 indices = np.array(indexer.indexPoints(stdCat[
'coord_ra']*conv,
954 stdCat[
'coord_dec']*conv))
956 formattedCat = self._formatCatalog(stdCat, offsets, bands)
959 dataId = indexer.makeDataId(
'master_schema',
960 datasetConfig.ref_dataset_name)
961 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
962 addRefCatMetadata(masterCat)
963 butler.put(masterCat,
'ref_cat', dataId=dataId)
966 h, rev = esutil.stat.histogram(indices, rev=
True)
967 gd, = np.where(h > 0)
968 selected = np.zeros(len(formattedCat), dtype=np.bool)
970 i1a = rev[rev[i]: rev[i + 1]]
979 dataId = indexer.makeDataId(indices[i1a[0]],
980 datasetConfig.ref_dataset_name)
981 butler.put(formattedCat[selected],
'ref_cat', dataId=dataId)
984 dataId = indexer.makeDataId(
None, datasetConfig.ref_dataset_name)
985 butler.put(datasetConfig,
'ref_cat_config', dataId=dataId)
987 self.log.info(
"Done outputting standard stars.")
989 def _formatCatalog(self, fgcmStarCat, offsets, bands):
991 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
995 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
996 SimpleCatalog as output by fgcmcal
997 offsets : `list` with len(self.bands) entries
998 Zeropoint offsets to apply
999 bands : `list` [`str`]
1000 List of band names from FGCM output
1004 formattedCat: `lsst.afw.table.SimpleCatalog`
1005 SimpleCatalog suitable for using as a reference catalog
1008 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1009 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1013 sourceMapper.addMinimalSchema(minSchema)
1015 sourceMapper.editOutputSchema().addField(
'%s_nGood' % (band), type=np.int32)
1016 sourceMapper.editOutputSchema().addField(
'%s_nTotal' % (band), type=np.int32)
1017 sourceMapper.editOutputSchema().addField(
'%s_nPsfCandidate' % (band), type=np.int32)
1019 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1020 formattedCat.reserve(len(fgcmStarCat))
1021 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1025 for b, band
in enumerate(bands):
1026 mag = fgcmStarCat[
'mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1029 flux = (mag*units.ABmag).to_value(units.nJy)
1030 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat[
'magErr_std'][:, b].astype(np.float64)
1032 formattedCat[
'%s_flux' % (band)][:] = flux
1033 formattedCat[
'%s_fluxErr' % (band)][:] = fluxErr
1034 formattedCat[
'%s_nGood' % (band)][:] = fgcmStarCat[
'ngood'][:, b]
1035 formattedCat[
'%s_nTotal' % (band)][:] = fgcmStarCat[
'ntotal'][:, b]
1036 formattedCat[
'%s_nPsfCandidate' % (band)][:] = fgcmStarCat[
'npsfcand'][:, b]
1038 addRefCatMetadata(formattedCat)
1042 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1043 physicalFilterMap, returnCatalogs=True,
1045 """Output the zeropoints in fgcm_photoCalib format.
1049 camera : `lsst.afw.cameraGeom.Camera`
1050 Camera from the butler.
1051 zptCat : `lsst.afw.table.BaseCatalog`
1052 FGCM zeropoint catalog from `FgcmFitCycleTask`.
1053 visitCat : `lsst.afw.table.BaseCatalog`
1054 FGCM visitCat from `FgcmBuildStarsTask`.
1055 offsets : `numpy.array`
1056 Float array of absolute calibration offsets, one for each filter.
1057 bands : `list` [`str`]
1058 List of band names from FGCM output.
1059 physicalFilterMap : `dict`
1060 Dictionary of mappings from physical filter to FGCM band.
1061 returnCatalogs : `bool`, optional
1062 Return photoCalibs as per-visit exposure catalogs.
1063 tract: `int`, optional
1064 Tract number to output. Default is None (global calibration)
1068 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1069 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1070 (returned if returnCatalogs is False).
1071 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1072 Generator that returns (visit, exposureCatalog) tuples.
1073 (returned if returnCatalogs is True).
1078 cannot_compute = fgcm.fgcmUtilities.zpFlagDict[
'CANNOT_COMPUTE_ZEROPOINT']
1079 selected = (((zptCat[
'fgcmFlag'] & cannot_compute) == 0)
1080 & (zptCat[
'fgcmZptVar'] > 0.0))
1083 badVisits = np.unique(zptCat[
'visit'][~selected])
1084 goodVisits = np.unique(zptCat[
'visit'][selected])
1085 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1086 for allBadVisit
in allBadVisits:
1087 self.log.warn(f
'No suitable photoCalib for visit {allBadVisit}')
1091 for f
in physicalFilterMap:
1093 if physicalFilterMap[f]
in bands:
1094 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1098 for ccdIndex, detector
in enumerate(camera):
1099 ccdMapping[detector.getId()] = ccdIndex
1103 for rec
in visitCat:
1104 scalingMapping[rec[
'visit']] = rec[
'scaling']
1106 if self.config.doComposeWcsJacobian:
1112 zptVisitCatalog =
None
1113 for rec
in zptCat[selected]:
1116 scaling = scalingMapping[rec[
'visit']][ccdMapping[rec[
'detector']]]
1123 postCalibrationOffset = offsetMapping[rec[
'filtername']]
1124 if self.config.doApplyMeanChromaticCorrection:
1125 postCalibrationOffset += rec[
'fgcmDeltaChrom']
1127 fgcmSuperStarField = self._getChebyshevBoundedField(rec[
'fgcmfZptSstarCheb'],
1128 rec[
'fgcmfZptChebXyMax'])
1130 fgcmZptField = self._getChebyshevBoundedField((rec[
'fgcmfZptCheb']*units.AB).to_value(units.nJy),
1131 rec[
'fgcmfZptChebXyMax'],
1132 offset=postCalibrationOffset,
1135 if self.config.doComposeWcsJacobian:
1137 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec[
'detector']],
1143 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1146 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1147 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec[
'fgcmZptVar'])
1148 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1149 calibrationErr=calibErr,
1150 calibration=fgcmField,
1153 if not returnCatalogs:
1155 yield (int(rec[
'visit']), int(rec[
'detector']), rec[
'filtername'], photoCalib)
1158 if rec[
'visit'] != lastVisit:
1162 yield (int(lastVisit), zptVisitCatalog)
1165 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1166 zptExpCatSchema.addField(
'visit', type=
'I', doc=
'Visit number')
1167 zptExpCatSchema.addField(
'detector_id', type=
'I', doc=
'Detector number')
1170 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1171 zptVisitCatalog.resize(len(camera))
1172 zptVisitCatalog[
'visit'] = rec[
'visit']
1174 zptVisitCatalog[
'detector_id'] = -1
1179 lastVisit = int(rec[
'visit'])
1181 zptVisitCatalog[zptCounter].setPhotoCalib(photoCalib)
1182 zptVisitCatalog[zptCounter][
'detector_id'] = int(rec[
'detector'])
1188 yield (int(lastVisit), zptVisitCatalog)
1190 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1192 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1197 coefficients: `numpy.array`
1198 Flattened array of chebyshev coefficients
1199 xyMax: `list` of length 2
1200 Maximum x and y of the chebyshev bounding box
1201 offset: `float`, optional
1202 Absolute calibration offset. Default is 0.0
1203 scaling: `float`, optional
1204 Flat scaling value from fgcmBuildStars. Default is 1.0
1208 boundedField: `lsst.afw.math.ChebyshevBoundedField`
1211 orderPlus1 = int(np.sqrt(coefficients.size))
1212 pars = np.zeros((orderPlus1, orderPlus1))
1214 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1215 lsst.geom.Point2I(*xyMax))
1217 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1218 * (10.**(offset/-2.5))*scaling)
1220 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1224 def _outputAtmospheres(self, dataRefDict, atmCat):
1226 Output the atmospheres.
1230 dataRefDict : `dict`
1231 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1232 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1233 dataRef dictionary with keys:
1235 ``"fgcmLookUpTable"``
1236 dataRef for the FGCM look-up table.
1237 atmCat : `lsst.afw.table.BaseCatalog`
1238 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1242 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1243 Generator that returns (visit, transmissionCurve) tuples.
1246 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
1248 atmosphereTableName = lutCat[0][
'tablename']
1249 elevation = lutCat[0][
'elevation']
1250 atmLambda = lutCat[0][
'atmLambda']
1255 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1256 atmTable.loadTable()
1260 if atmTable
is None:
1263 modGen = fgcm.ModtranGenerator(elevation)
1264 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1265 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1266 except (ValueError, IOError)
as e:
1267 raise RuntimeError(
"FGCM look-up-table generated with modtran, "
1268 "but modtran not configured to run.")
from e
1270 zenith = np.degrees(np.arccos(1./atmCat[
'secZenith']))
1272 for i, visit
in enumerate(atmCat[
'visit']):
1273 if atmTable
is not None:
1275 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i][
'pmb'],
1276 pwv=atmCat[i][
'pwv'],
1278 tau=atmCat[i][
'tau'],
1279 alpha=atmCat[i][
'alpha'],
1281 ctranslamstd=[atmCat[i][
'cTrans'],
1282 atmCat[i][
'lamStd']])
1285 modAtm = modGen(pmb=atmCat[i][
'pmb'],
1286 pwv=atmCat[i][
'pwv'],
1288 tau=atmCat[i][
'tau'],
1289 alpha=atmCat[i][
'alpha'],
1291 lambdaRange=lambdaRange,
1292 lambdaStep=lambdaStep,
1293 ctranslamstd=[atmCat[i][
'cTrans'],
1294 atmCat[i][
'lamStd']])
1295 atmVals = modAtm[
'COMBINED']
1298 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1299 wavelengths=atmLambda,
1300 throughputAtMin=atmVals[0],
1301 throughputAtMax=atmVals[-1])
1303 yield (int(visit), curve)
def computeApproxPixelAreaFields(camera)