23 """Make the final fgcmcal output products.
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
41 from astropy
import units
43 import lsst.pex.config
as pexConfig
44 import lsst.pipe.base
as pipeBase
45 from lsst.pipe.base
import connectionTypes
46 from lsst.afw.image
import TransmissionCurve
47 from lsst.meas.algorithms
import LoadIndexedReferenceObjectsTask
48 from lsst.meas.algorithms
import ReferenceObjectLoader
49 from lsst.pipe.tasks.photoCal
import PhotoCalTask
51 import lsst.afw.image
as afwImage
52 import lsst.afw.math
as afwMath
53 import lsst.afw.table
as afwTable
54 from lsst.meas.algorithms
import IndexerRegistry
55 from lsst.meas.algorithms
import DatasetConfig
56 from lsst.meas.algorithms.ingestIndexReferenceTask
import addRefCatMetadata
58 from .utilities
import computeApproxPixelAreaFields
59 from .utilities
import lookupStaticCalibrations
63 __all__ = [
'FgcmOutputProductsConfig',
'FgcmOutputProductsTask',
'FgcmOutputProductsRunner']
67 dimensions=(
"instrument",),
68 defaultTemplates={
"cycleNumber":
"0"}):
69 camera = connectionTypes.PrerequisiteInput(
70 doc=
"Camera instrument",
72 storageClass=
"Camera",
73 dimensions=(
"instrument",),
74 lookupFunction=lookupStaticCalibrations,
78 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
79 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
80 "chromatic corrections."),
81 name=
"fgcmLookUpTable",
82 storageClass=
"Catalog",
83 dimensions=(
"instrument",),
87 fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
88 doc=
"Catalog of visit information for fgcm",
89 name=
"fgcmVisitCatalog",
90 storageClass=
"Catalog",
91 dimensions=(
"instrument",),
95 fgcmStandardStars = connectionTypes.PrerequisiteInput(
96 doc=
"Catalog of standard star data from fgcm fit",
97 name=
"fgcmStandardStars{cycleNumber}",
98 storageClass=
"SimpleCatalog",
99 dimensions=(
"instrument",),
103 fgcmZeropoints = connectionTypes.PrerequisiteInput(
104 doc=
"Catalog of zeropoints from fgcm fit",
105 name=
"fgcmZeropoints{cycleNumber}",
106 storageClass=
"Catalog",
107 dimensions=(
"instrument",),
111 fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
112 doc=
"Catalog of atmosphere parameters from fgcm fit",
113 name=
"fgcmAtmosphereParameters{cycleNumber}",
114 storageClass=
"Catalog",
115 dimensions=(
"instrument",),
119 refCat = connectionTypes.PrerequisiteInput(
120 doc=
"Reference catalog to use for photometric calibration",
122 storageClass=
"SimpleCatalog",
123 dimensions=(
"skypix",),
128 fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
129 doc=
"Config used to build FGCM input stars",
130 name=
"fgcmBuildStarsTable_config",
131 storageClass=
"Config",
134 fgcmPhotoCalib = connectionTypes.Output(
135 doc=
"Per-visit photoCalib exposure catalogs produced from fgcm calibration",
136 name=
"fgcmPhotoCalibCatalog",
137 storageClass=
"ExposureCatalog",
138 dimensions=(
"instrument",
"visit",),
142 fgcmTransmissionAtmosphere = connectionTypes.Output(
143 doc=
"Per-visit atmosphere transmission files produced from fgcm calibration",
144 name=
"transmission_atmosphere_fgcm",
145 storageClass=
"TransmissionCurve",
146 dimensions=(
"instrument",
151 fgcmOffsets = connectionTypes.Output(
152 doc=
"Per-band offsets computed from doReferenceCalibration",
153 name=
"fgcmReferenceCalibrationOffsets",
154 storageClass=
"Catalog",
155 dimensions=(
"instrument",),
159 def __init__(self, *, config=None):
160 super().__init__(config=config)
162 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
163 raise ValueError(
"cycleNumber must be of integer format")
164 if config.connections.refCat != config.refObjLoader.ref_dataset_name:
165 raise ValueError(
"connections.refCat must be the same as refObjLoader.ref_dataset_name")
167 if config.doRefcatOutput:
168 raise ValueError(
"FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
170 if not config.doReferenceCalibration:
171 self.prerequisiteInputs.remove(
"refCat")
172 if not config.doAtmosphereOutput:
173 self.prerequisiteInputs.remove(
"fgcmAtmosphereParameters")
174 if not config.doZeropointOutput:
175 self.prerequisiteInputs.remove(
"fgcmZeropoints")
176 if not config.doReferenceCalibration:
177 self.outputs.remove(
"fgcmOffsets")
180 class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
181 pipelineConnections=FgcmOutputProductsConnections):
182 """Config for FgcmOutputProductsTask"""
184 cycleNumber = pexConfig.Field(
185 doc=
"Final fit cycle from FGCM fit",
192 doReferenceCalibration = pexConfig.Field(
193 doc=(
"Transfer 'absolute' calibration from reference catalog? "
194 "This afterburner step is unnecessary if reference stars "
195 "were used in the full fit in FgcmFitCycleTask."),
199 doRefcatOutput = pexConfig.Field(
200 doc=
"Output standard stars in reference catalog format",
204 doAtmosphereOutput = pexConfig.Field(
205 doc=
"Output atmospheres in transmission_atmosphere_fgcm format",
209 doZeropointOutput = pexConfig.Field(
210 doc=
"Output zeropoints in fgcm_photoCalib format",
214 doComposeWcsJacobian = pexConfig.Field(
215 doc=
"Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
219 doApplyMeanChromaticCorrection = pexConfig.Field(
220 doc=
"Apply the mean chromatic correction to the zeropoints?",
224 refObjLoader = pexConfig.ConfigurableField(
225 target=LoadIndexedReferenceObjectsTask,
226 doc=
"reference object loader for 'absolute' photometric calibration",
228 photoCal = pexConfig.ConfigurableField(
230 doc=
"task to perform 'absolute' calibration",
232 referencePixelizationNside = pexConfig.Field(
233 doc=
"Healpix nside to pixelize catalog to compare to reference catalog",
237 referencePixelizationMinStars = pexConfig.Field(
238 doc=(
"Minimum number of stars per healpix pixel to select for comparison"
239 "to the specified reference catalog"),
243 referenceMinMatch = pexConfig.Field(
244 doc=
"Minimum number of stars matched to reference catalog to be used in statistics",
248 referencePixelizationNPixels = pexConfig.Field(
249 doc=(
"Number of healpix pixels to sample to do comparison. "
250 "Doing too many will take a long time and not yield any more "
251 "precise results because the final number is the median offset "
252 "(per band) from the set of pixels."),
256 datasetConfig = pexConfig.ConfigField(
258 doc=
"Configuration for writing/reading ingested catalog",
261 def setDefaults(self):
262 pexConfig.Config.setDefaults(self)
272 self.photoCal.applyColorTerms =
False
273 self.photoCal.fluxField =
'instFlux'
274 self.photoCal.magErrFloor = 0.003
275 self.photoCal.match.referenceSelection.doSignalToNoise =
True
276 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
277 self.photoCal.match.sourceSelection.doSignalToNoise =
True
278 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
279 self.photoCal.match.sourceSelection.signalToNoise.fluxField =
'instFlux'
280 self.photoCal.match.sourceSelection.signalToNoise.errField =
'instFluxErr'
281 self.photoCal.match.sourceSelection.doFlags =
True
282 self.photoCal.match.sourceSelection.flags.good = []
283 self.photoCal.match.sourceSelection.flags.bad = [
'flag_badStar']
284 self.photoCal.match.sourceSelection.doUnresolved =
False
285 self.datasetConfig.ref_dataset_name =
'fgcm_stars'
286 self.datasetConfig.format_version = 1
292 self.connections.cycleNumber = str(self.cycleNumber)
295 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
296 """Subclass of TaskRunner for fgcmOutputProductsTask
298 fgcmOutputProductsTask.run() takes one argument, the butler, and
299 does not run on any data in the repository.
300 This runner does not use any parallelization.
304 def getTargetList(parsedCmd):
306 Return a list with one element, the butler.
308 return [parsedCmd.butler]
310 def __call__(self, butler):
314 butler: `lsst.daf.persistence.Butler`
318 exitStatus: `list` with `pipeBase.Struct`
319 exitStatus (0: success; 1: failure)
320 if self.doReturnResults also
321 results (`np.array` with absolute zeropoint offsets)
323 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
327 results = task.runDataRef(butler)
330 results = task.runDataRef(butler)
331 except Exception
as e:
333 task.log.fatal(
"Failed: %s" % e)
334 if not isinstance(e, pipeBase.TaskError):
335 traceback.print_exc(file=sys.stderr)
337 task.writeMetadata(butler)
339 if self.doReturnResults:
341 return [pipeBase.Struct(exitStatus=exitStatus,
344 return [pipeBase.Struct(exitStatus=exitStatus)]
346 def run(self, parsedCmd):
348 Run the task, with no multiprocessing
352 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
357 if self.precall(parsedCmd):
358 targetList = self.getTargetList(parsedCmd)
360 resultList = self(targetList[0])
365 class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
367 Output products from FGCM global calibration.
370 ConfigClass = FgcmOutputProductsConfig
371 RunnerClass = FgcmOutputProductsRunner
372 _DefaultName =
"fgcmOutputProducts"
374 def __init__(self, butler=None, **kwargs):
375 super().__init__(**kwargs)
378 def _getMetadataName(self):
381 def runQuantum(self, butlerQC, inputRefs, outputRefs):
383 dataRefDict[
'camera'] = butlerQC.get(inputRefs.camera)
384 dataRefDict[
'fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
385 dataRefDict[
'fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
386 dataRefDict[
'fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
388 if self.config.doZeropointOutput:
389 dataRefDict[
'fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
390 photoCalibRefDict = {photoCalibRef.dataId.byName()[
'visit']:
391 photoCalibRef
for photoCalibRef
in outputRefs.fgcmPhotoCalib}
393 if self.config.doAtmosphereOutput:
394 dataRefDict[
'fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
395 atmRefDict = {atmRef.dataId.byName()[
'visit']: atmRef
for
396 atmRef
in outputRefs.fgcmTransmissionAtmosphere}
398 if self.config.doReferenceCalibration:
399 refConfig = self.config.refObjLoader
400 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
401 for ref
in inputRefs.refCat],
402 refCats=butlerQC.get(inputRefs.refCat),
406 self.refObjLoader =
None
408 dataRefDict[
'fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
410 fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
411 filterMap = fgcmBuildStarsConfig.filterMap
413 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
414 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
415 "in fgcmBuildStarsTask.")
416 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
417 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
419 struct = self.run(dataRefDict, filterMap, returnCatalogs=
True)
422 if struct.photoCalibCatalogs
is not None:
423 self.log.info(
"Outputting photoCalib catalogs.")
424 for visit, expCatalog
in struct.photoCalibCatalogs:
425 butlerQC.put(expCatalog, photoCalibRefDict[visit])
426 self.log.info(
"Done outputting photoCalib catalogs.")
429 if struct.atmospheres
is not None:
430 self.log.info(
"Outputting atmosphere transmission files.")
431 for visit, atm
in struct.atmospheres:
432 butlerQC.put(atm, atmRefDict[visit])
433 self.log.info(
"Done outputting atmosphere files.")
435 if self.config.doReferenceCalibration:
437 schema = afwTable.Schema()
438 schema.addField(
'offset', type=np.float64,
439 doc=
"Post-process calibration offset (mag)")
440 offsetCat = afwTable.BaseCatalog(schema)
441 offsetCat.resize(len(struct.offsets))
442 offsetCat[
'offset'][:] = struct.offsets
444 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
449 def runDataRef(self, butler):
451 Make FGCM output products for use in the stack
455 butler: `lsst.daf.persistence.Butler`
457 Final fit cycle number, override config.
461 offsets: `lsst.pipe.base.Struct`
462 A structure with array of zeropoint offsets
467 Raised if any one of the following is true:
469 - butler cannot find "fgcmBuildStars_config" or
470 "fgcmBuildStarsTable_config".
471 - butler cannot find "fgcmFitCycle_config".
472 - "fgcmFitCycle_config" does not refer to
473 `self.config.cycleNumber`.
474 - butler cannot find "fgcmAtmosphereParameters" and
475 `self.config.doAtmosphereOutput` is `True`.
476 - butler cannot find "fgcmStandardStars" and
477 `self.config.doReferenceCalibration` is `True` or
478 `self.config.doRefcatOutput` is `True`.
479 - butler cannot find "fgcmZeropoints" and
480 `self.config.doZeropointOutput` is `True`.
482 if self.config.doReferenceCalibration:
484 self.makeSubtask(
"refObjLoader", butler=butler)
488 if not butler.datasetExists(
'fgcmBuildStarsTable_config')
and \
489 not butler.datasetExists(
'fgcmBuildStars_config'):
490 raise RuntimeError(
"Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
491 "which is prereq for fgcmOutputProducts")
493 if butler.datasetExists(
'fgcmBuildStarsTable_config'):
494 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStarsTable_config')
496 fgcmBuildStarsConfig = butler.get(
'fgcmBuildStars_config')
497 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
498 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
499 filterMap = fgcmBuildStarsConfig.filterMap
501 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
502 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
503 "in fgcmBuildStarsTask.")
505 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
506 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
509 if (self.config.doAtmosphereOutput
510 and not butler.datasetExists(
'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
511 raise RuntimeError(f
"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
513 if not butler.datasetExists(
'fgcmStandardStars',
514 fgcmcycle=self.config.cycleNumber):
515 raise RuntimeError(
"Standard stars are missing for cycle %d." %
516 (self.config.cycleNumber))
518 if (self.config.doZeropointOutput
519 and (
not butler.datasetExists(
'fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
520 raise RuntimeError(
"Zeropoints are missing for cycle %d." %
521 (self.config.cycleNumber))
525 dataRefDict[
'camera'] = butler.get(
'camera')
526 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
527 dataRefDict[
'fgcmVisitCatalog'] = butler.dataRef(
'fgcmVisitCatalog')
528 dataRefDict[
'fgcmStandardStars'] = butler.dataRef(
'fgcmStandardStars',
529 fgcmcycle=self.config.cycleNumber)
531 if self.config.doZeropointOutput:
532 dataRefDict[
'fgcmZeropoints'] = butler.dataRef(
'fgcmZeropoints',
533 fgcmcycle=self.config.cycleNumber)
534 if self.config.doAtmosphereOutput:
535 dataRefDict[
'fgcmAtmosphereParameters'] = butler.dataRef(
'fgcmAtmosphereParameters',
536 fgcmcycle=self.config.cycleNumber)
538 struct = self.run(dataRefDict, filterMap, butler=butler, returnCatalogs=
False)
540 if struct.photoCalibs
is not None:
541 self.log.info(
"Outputting photoCalib files.")
544 for visit, detector, filtername, photoCalib
in struct.photoCalibs:
545 if filtername
not in filterMapping:
548 dataId = {visitDataRefName: visit,
549 ccdDataRefName: detector}
550 dataRef = butler.dataRef(
'raw', dataId=dataId)
551 filterMapping[filtername] = dataRef.dataId[
'filter']
553 butler.put(photoCalib,
'fgcm_photoCalib',
554 dataId={visitDataRefName: visit,
555 ccdDataRefName: detector,
556 'filter': filterMapping[filtername]})
558 self.log.info(
"Done outputting photoCalib files.")
560 if struct.atmospheres
is not None:
561 self.log.info(
"Outputting atmosphere transmission files.")
562 for visit, atm
in struct.atmospheres:
563 butler.put(atm,
"transmission_atmosphere_fgcm",
564 dataId={visitDataRefName: visit})
565 self.log.info(
"Done outputting atmosphere transmissions.")
567 return pipeBase.Struct(offsets=struct.offsets)
569 def run(self, dataRefDict, filterMap, returnCatalogs=True, butler=None):
570 """Run the output products task.
575 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
576 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
577 dataRef dictionary with keys:
580 Camera object (`lsst.afw.cameraGeom.Camera`)
581 ``"fgcmLookUpTable"``
582 dataRef for the FGCM look-up table.
583 ``"fgcmVisitCatalog"``
584 dataRef for visit summary catalog.
585 ``"fgcmStandardStars"``
586 dataRef for the output standard star catalog.
588 dataRef for the zeropoint data catalog.
589 ``"fgcmAtmosphereParameters"``
590 dataRef for the atmosphere parameter catalog.
591 ``"fgcmBuildStarsTableConfig"``
592 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
594 Dictionary of mappings from filter to FGCM band.
595 returnCatalogs : `bool`, optional
596 Return photoCalibs as per-visit exposure catalogs.
597 butler : `lsst.daf.persistence.Butler`, optional
598 Gen2 butler used for reference star outputs
602 retStruct : `lsst.pipe.base.Struct`
603 Output structure with keys:
605 offsets : `np.ndarray`
606 Final reference offsets, per band.
607 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
608 Generator that returns (visit, transmissionCurve) tuples.
609 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
610 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
611 (returned if returnCatalogs is False).
612 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
613 Generator that returns (visit, exposureCatalog) tuples.
614 (returned if returnCatalogs is True).
616 stdCat = dataRefDict[
'fgcmStandardStars'].get()
617 md = stdCat.getMetadata()
618 bands = md.getArray(
'BANDS')
620 if self.config.doReferenceCalibration:
621 offsets = self._computeReferenceOffsets(stdCat, bands)
623 offsets = np.zeros(len(bands))
626 if self.config.doRefcatOutput
and butler
is not None:
627 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
631 if self.config.doZeropointOutput:
632 zptCat = dataRefDict[
'fgcmZeropoints'].get()
633 visitCat = dataRefDict[
'fgcmVisitCatalog'].get()
635 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
636 filterMap, returnCatalogs=returnCatalogs)
640 if self.config.doAtmosphereOutput:
641 atmCat = dataRefDict[
'fgcmAtmosphereParameters'].get()
642 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
646 retStruct = pipeBase.Struct(offsets=offsets,
649 retStruct.photoCalibCatalogs = pcgen
651 retStruct.photoCalibs = pcgen
655 def generateTractOutputProducts(self, dataRefDict, tract,
656 visitCat, zptCat, atmCat, stdCat,
657 fgcmBuildStarsConfig,
661 Generate the output products for a given tract, as specified in the config.
663 This method is here to have an alternate entry-point for
669 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
670 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
671 dataRef dictionary with keys:
674 Camera object (`lsst.afw.cameraGeom.Camera`)
675 ``"fgcmLookUpTable"``
676 dataRef for the FGCM look-up table.
679 visitCat : `lsst.afw.table.BaseCatalog`
680 FGCM visitCat from `FgcmBuildStarsTask`
681 zptCat : `lsst.afw.table.BaseCatalog`
682 FGCM zeropoint catalog from `FgcmFitCycleTask`
683 atmCat : `lsst.afw.table.BaseCatalog`
684 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
685 stdCat : `lsst.afw.table.SimpleCatalog`
686 FGCM standard star catalog from `FgcmFitCycleTask`
687 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
688 Configuration object from `FgcmBuildStarsTask`
689 returnCatalogs : `bool`, optional
690 Return photoCalibs as per-visit exposure catalogs.
691 butler: `lsst.daf.persistence.Butler`, optional
692 Gen2 butler used for reference star outputs
696 retStruct : `lsst.pipe.base.Struct`
697 Output structure with keys:
699 offsets : `np.ndarray`
700 Final reference offsets, per band.
701 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
702 Generator that returns (visit, transmissionCurve) tuples.
703 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
704 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
705 (returned if returnCatalogs is False).
706 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
707 Generator that returns (visit, exposureCatalog) tuples.
708 (returned if returnCatalogs is True).
710 filterMap = fgcmBuildStarsConfig.filterMap
712 md = stdCat.getMetadata()
713 bands = md.getArray(
'BANDS')
715 if self.config.doComposeWcsJacobian
and not fgcmBuildStarsConfig.doApplyWcsJacobian:
716 raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
717 "in fgcmBuildStarsTask.")
719 if not self.config.doComposeWcsJacobian
and fgcmBuildStarsConfig.doApplyWcsJacobian:
720 self.log.warn(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
722 if self.config.doReferenceCalibration:
723 offsets = self._computeReferenceOffsets(stdCat, bands)
725 offsets = np.zeros(len(bands))
727 if self.config.doRefcatOutput
and butler
is not None:
729 datasetConfig = copy.copy(self.config.datasetConfig)
730 datasetConfig.ref_dataset_name =
'%s_%d' % (self.config.datasetConfig.ref_dataset_name,
732 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
734 if self.config.doZeropointOutput:
735 pcgen = self._outputZeropoints(dataRefDict[
'camera'], zptCat, visitCat, offsets, bands,
736 filterMap, returnCatalogs=returnCatalogs)
740 if self.config.doAtmosphereOutput:
741 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
745 retStruct = pipeBase.Struct(offsets=offsets,
748 retStruct.photoCalibCatalogs = pcgen
750 retStruct.photoCalibs = pcgen
754 def _computeReferenceOffsets(self, stdCat, bands):
756 Compute offsets relative to a reference catalog.
758 This method splits the star catalog into healpix pixels
759 and computes the calibration transfer for a sample of
760 these pixels to approximate the 'absolute' calibration
761 values (on for each band) to apply to transfer the
766 stdCat : `lsst.afw.table.SimpleCatalog`
768 bands : `list` [`str`]
769 List of band names from FGCM output
772 offsets : `numpy.array` of floats
773 Per band zeropoint offsets
779 minObs = stdCat[
'ngood'].min(axis=1)
781 goodStars = (minObs >= 1)
782 stdCat = stdCat[goodStars]
784 self.log.info(
"Found %d stars with at least 1 good observation in each band" %
794 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
795 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
796 sourceMapper.editOutputSchema().addField(
'instFlux', type=np.float64,
797 doc=
"instrumental flux (counts)")
798 sourceMapper.editOutputSchema().addField(
'instFluxErr', type=np.float64,
799 doc=
"instrumental flux error (counts)")
800 badStarKey = sourceMapper.editOutputSchema().addField(
'flag_badStar',
808 theta = np.pi/2. - stdCat[
'coord_dec']
809 phi = stdCat[
'coord_ra']
811 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
812 h, rev = esutil.stat.histogram(ipring, rev=
True)
814 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
816 self.log.info(
"Found %d pixels (nside=%d) with at least %d good stars" %
818 self.config.referencePixelizationNside,
819 self.config.referencePixelizationMinStars))
821 if gdpix.size < self.config.referencePixelizationNPixels:
822 self.log.warn(
"Found fewer good pixels (%d) than preferred in configuration (%d)" %
823 (gdpix.size, self.config.referencePixelizationNPixels))
826 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=
False)
828 results = np.zeros(gdpix.size, dtype=[(
'hpix',
'i4'),
829 (
'nstar',
'i4', len(bands)),
830 (
'nmatch',
'i4', len(bands)),
831 (
'zp',
'f4', len(bands)),
832 (
'zpErr',
'f4', len(bands))])
833 results[
'hpix'] = ipring[rev[rev[gdpix]]]
836 selected = np.zeros(len(stdCat), dtype=np.bool)
838 refFluxFields = [
None]*len(bands)
840 for p, pix
in enumerate(gdpix):
841 i1a = rev[rev[pix]: rev[pix + 1]]
849 for b, band
in enumerate(bands):
851 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
852 selected, refFluxFields)
853 results[
'nstar'][p, b] = len(i1a)
854 results[
'nmatch'][p, b] = len(struct.arrays.refMag)
855 results[
'zp'][p, b] = struct.zp
856 results[
'zpErr'][p, b] = struct.sigma
859 offsets = np.zeros(len(bands))
861 for b, band
in enumerate(bands):
863 ok, = np.where(results[
'nmatch'][:, b] >= self.config.referenceMinMatch)
864 offsets[b] = np.median(results[
'zp'][ok, b])
867 madSigma = 1.4826*np.median(np.abs(results[
'zp'][ok, b] - offsets[b]))
868 self.log.info(
"Reference catalog offset for %s band: %.12f +/- %.12f" %
869 (band, offsets[b], madSigma))
873 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
874 b, band, stdCat, selected, refFluxFields):
876 Compute the zeropoint offset between the fgcm stdCat and the reference
877 stars for one pixel in one band
881 sourceMapper: `lsst.afw.table.SchemaMapper`
882 Mapper to go from stdCat to calibratable catalog
883 badStarKey: `lsst.afw.table.Key`
884 Key for the field with bad stars
886 Index of the band in the star catalog
888 Name of band for reference catalog
889 stdCat: `lsst.afw.table.SimpleCatalog`
891 selected: `numpy.array(dtype=np.bool)`
892 Boolean array of which stars are in the pixel
893 refFluxFields: `list`
894 List of names of flux fields for reference catalog
897 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
898 sourceCat.reserve(selected.sum())
899 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
900 sourceCat[
'instFlux'] = 10.**(stdCat[
'mag_std_noabs'][selected, b]/(-2.5))
901 sourceCat[
'instFluxErr'] = (np.log(10.)/2.5)*(stdCat[
'magErr_std'][selected, b]
902 * sourceCat[
'instFlux'])
906 badStar = (stdCat[
'mag_std_noabs'][selected, b] > 90.0)
907 for rec
in sourceCat[badStar]:
908 rec.set(badStarKey,
True)
910 exposure = afwImage.ExposureF()
911 exposure.setFilter(afwImage.Filter(band))
913 if refFluxFields[b]
is None:
916 ctr = stdCat[0].getCoord()
917 rad = 0.05*lsst.geom.degrees
918 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
919 refFluxFields[b] = refDataTest.fluxField
922 calConfig = copy.copy(self.config.photoCal.value)
923 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
924 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] +
'Err'
925 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
927 schema=sourceCat.getSchema())
929 struct = calTask.run(exposure, sourceCat)
933 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
935 Output standard stars in indexed reference catalog format.
936 This is not currently supported in Gen3.
940 butler : `lsst.daf.persistence.Butler`
941 stdCat : `lsst.afw.table.SimpleCatalog`
942 FGCM standard star catalog from fgcmFitCycleTask
943 offsets : `numpy.array` of floats
944 Per band zeropoint offsets
945 bands : `list` [`str`]
946 List of band names from FGCM output
947 datasetConfig : `lsst.meas.algorithms.DatasetConfig`
948 Config for reference dataset
951 self.log.info(
"Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
953 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
954 self.config.datasetConfig.indexer.active)
961 conv = stdCat[0][
'coord_ra'].asDegrees()/float(stdCat[0][
'coord_ra'])
962 indices = np.array(indexer.indexPoints(stdCat[
'coord_ra']*conv,
963 stdCat[
'coord_dec']*conv))
965 formattedCat = self._formatCatalog(stdCat, offsets, bands)
968 dataId = indexer.makeDataId(
'master_schema',
969 datasetConfig.ref_dataset_name)
970 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
971 addRefCatMetadata(masterCat)
972 butler.put(masterCat,
'ref_cat', dataId=dataId)
975 h, rev = esutil.stat.histogram(indices, rev=
True)
976 gd, = np.where(h > 0)
977 selected = np.zeros(len(formattedCat), dtype=np.bool)
979 i1a = rev[rev[i]: rev[i + 1]]
988 dataId = indexer.makeDataId(indices[i1a[0]],
989 datasetConfig.ref_dataset_name)
990 butler.put(formattedCat[selected],
'ref_cat', dataId=dataId)
993 dataId = indexer.makeDataId(
None, datasetConfig.ref_dataset_name)
994 butler.put(datasetConfig,
'ref_cat_config', dataId=dataId)
996 self.log.info(
"Done outputting standard stars.")
998 def _formatCatalog(self, fgcmStarCat, offsets, bands):
1000 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
1004 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
1005 SimpleCatalog as output by fgcmcal
1006 offsets : `list` with len(self.bands) entries
1007 Zeropoint offsets to apply
1008 bands : `list` [`str`]
1009 List of band names from FGCM output
1013 formattedCat: `lsst.afw.table.SimpleCatalog`
1014 SimpleCatalog suitable for using as a reference catalog
1017 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1018 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1022 sourceMapper.addMinimalSchema(minSchema)
1024 sourceMapper.editOutputSchema().addField(
'%s_nGood' % (band), type=np.int32)
1025 sourceMapper.editOutputSchema().addField(
'%s_nTotal' % (band), type=np.int32)
1026 sourceMapper.editOutputSchema().addField(
'%s_nPsfCandidate' % (band), type=np.int32)
1028 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1029 formattedCat.reserve(len(fgcmStarCat))
1030 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1034 for b, band
in enumerate(bands):
1035 mag = fgcmStarCat[
'mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1038 flux = (mag*units.ABmag).to_value(units.nJy)
1039 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat[
'magErr_std'][:, b].astype(np.float64)
1041 formattedCat[
'%s_flux' % (band)][:] = flux
1042 formattedCat[
'%s_fluxErr' % (band)][:] = fluxErr
1043 formattedCat[
'%s_nGood' % (band)][:] = fgcmStarCat[
'ngood'][:, b]
1044 formattedCat[
'%s_nTotal' % (band)][:] = fgcmStarCat[
'ntotal'][:, b]
1045 formattedCat[
'%s_nPsfCandidate' % (band)][:] = fgcmStarCat[
'npsfcand'][:, b]
1047 addRefCatMetadata(formattedCat)
1051 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1052 filterMap, returnCatalogs=True,
1054 """Output the zeropoints in fgcm_photoCalib format.
1058 camera : `lsst.afw.cameraGeom.Camera`
1059 Camera from the butler.
1060 zptCat : `lsst.afw.table.BaseCatalog`
1061 FGCM zeropoint catalog from `FgcmFitCycleTask`.
1062 visitCat : `lsst.afw.table.BaseCatalog`
1063 FGCM visitCat from `FgcmBuildStarsTask`.
1064 offsets : `numpy.array`
1065 Float array of absolute calibration offsets, one for each filter.
1066 bands : `list` [`str`]
1067 List of band names from FGCM output.
1069 Dictionary of mappings from filter to FGCM band.
1070 returnCatalogs : `bool`, optional
1071 Return photoCalibs as per-visit exposure catalogs.
1072 tract: `int`, optional
1073 Tract number to output. Default is None (global calibration)
1077 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1078 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1079 (returned if returnCatalogs is False).
1080 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1081 Generator that returns (visit, exposureCatalog) tuples.
1082 (returned if returnCatalogs is True).
1087 cannot_compute = fgcm.fgcmUtilities.zpFlagDict[
'CANNOT_COMPUTE_ZEROPOINT']
1088 selected = (((zptCat[
'fgcmFlag'] & cannot_compute) == 0)
1089 & (zptCat[
'fgcmZptVar'] > 0.0))
1092 badVisits = np.unique(zptCat[
'visit'][~selected])
1093 goodVisits = np.unique(zptCat[
'visit'][selected])
1094 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1095 for allBadVisit
in allBadVisits:
1096 self.log.warn(f
'No suitable photoCalib for visit {allBadVisit}')
1102 if filterMap[f]
in bands:
1103 offsetMapping[f] = offsets[bands.index(filterMap[f])]
1107 for ccdIndex, detector
in enumerate(camera):
1108 ccdMapping[detector.getId()] = ccdIndex
1112 for rec
in visitCat:
1113 scalingMapping[rec[
'visit']] = rec[
'scaling']
1115 if self.config.doComposeWcsJacobian:
1121 zptVisitCatalog =
None
1122 for rec
in zptCat[selected]:
1125 scaling = scalingMapping[rec[
'visit']][ccdMapping[rec[
'detector']]]
1132 postCalibrationOffset = offsetMapping[rec[
'filtername']]
1133 if self.config.doApplyMeanChromaticCorrection:
1134 postCalibrationOffset += rec[
'fgcmDeltaChrom']
1136 fgcmSuperStarField = self._getChebyshevBoundedField(rec[
'fgcmfZptSstarCheb'],
1137 rec[
'fgcmfZptChebXyMax'])
1139 fgcmZptField = self._getChebyshevBoundedField((rec[
'fgcmfZptCheb']*units.AB).to_value(units.nJy),
1140 rec[
'fgcmfZptChebXyMax'],
1141 offset=postCalibrationOffset,
1144 if self.config.doComposeWcsJacobian:
1146 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec[
'detector']],
1152 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1155 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1156 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec[
'fgcmZptVar'])
1157 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1158 calibrationErr=calibErr,
1159 calibration=fgcmField,
1162 if not returnCatalogs:
1164 yield (int(rec[
'visit']), int(rec[
'detector']), rec[
'filtername'], photoCalib)
1167 if rec[
'visit'] != lastVisit:
1171 yield (int(lastVisit), zptVisitCatalog)
1174 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1175 zptExpCatSchema.addField(
'visit', type=
'I', doc=
'Visit number')
1176 zptExpCatSchema.addField(
'detector_id', type=
'I', doc=
'Detector number')
1179 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1180 zptVisitCatalog.resize(len(camera))
1185 lastVisit = int(rec[
'visit'])
1187 zptVisitCatalog[zptCounter].setPhotoCalib(photoCalib)
1188 zptVisitCatalog[zptCounter][
'visit'] = int(rec[
'visit'])
1189 zptVisitCatalog[zptCounter][
'detector_id'] = int(rec[
'detector'])
1195 yield (int(lastVisit), zptVisitCatalog)
1197 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1199 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1204 coefficients: `numpy.array`
1205 Flattened array of chebyshev coefficients
1206 xyMax: `list` of length 2
1207 Maximum x and y of the chebyshev bounding box
1208 offset: `float`, optional
1209 Absolute calibration offset. Default is 0.0
1210 scaling: `float`, optional
1211 Flat scaling value from fgcmBuildStars. Default is 1.0
1215 boundedField: `lsst.afw.math.ChebyshevBoundedField`
1218 orderPlus1 = int(np.sqrt(coefficients.size))
1219 pars = np.zeros((orderPlus1, orderPlus1))
1221 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1222 lsst.geom.Point2I(*xyMax))
1224 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1225 * (10.**(offset/-2.5))*scaling)
1227 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1231 def _outputAtmospheres(self, dataRefDict, atmCat):
1233 Output the atmospheres.
1237 dataRefDict : `dict`
1238 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1239 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1240 dataRef dictionary with keys:
1242 ``"fgcmLookUpTable"``
1243 dataRef for the FGCM look-up table.
1244 atmCat : `lsst.afw.table.BaseCatalog`
1245 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1249 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1250 Generator that returns (visit, transmissionCurve) tuples.
1253 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
1255 atmosphereTableName = lutCat[0][
'tablename']
1256 elevation = lutCat[0][
'elevation']
1257 atmLambda = lutCat[0][
'atmLambda']
1262 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1263 atmTable.loadTable()
1267 if atmTable
is None:
1270 modGen = fgcm.ModtranGenerator(elevation)
1271 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1272 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1273 except (ValueError, IOError)
as e:
1274 raise RuntimeError(
"FGCM look-up-table generated with modtran, "
1275 "but modtran not configured to run.")
from e
1277 zenith = np.degrees(np.arccos(1./atmCat[
'secZenith']))
1279 for i, visit
in enumerate(atmCat[
'visit']):
1280 if atmTable
is not None:
1282 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i][
'pmb'],
1283 pwv=atmCat[i][
'pwv'],
1285 tau=atmCat[i][
'tau'],
1286 alpha=atmCat[i][
'alpha'],
1288 ctranslamstd=[atmCat[i][
'cTrans'],
1289 atmCat[i][
'lamStd']])
1292 modAtm = modGen(pmb=atmCat[i][
'pmb'],
1293 pwv=atmCat[i][
'pwv'],
1295 tau=atmCat[i][
'tau'],
1296 alpha=atmCat[i][
'alpha'],
1298 lambdaRange=lambdaRange,
1299 lambdaStep=lambdaStep,
1300 ctranslamstd=[atmCat[i][
'cTrans'],
1301 atmCat[i][
'lamStd']])
1302 atmVals = modAtm[
'COMBINED']
1305 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1306 wavelengths=atmLambda,
1307 throughputAtMin=atmVals[0],
1308 throughputAtMax=atmVals[-1])
1310 yield (int(visit), curve)
def computeApproxPixelAreaFields(camera)