Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Make the final fgcmcal output products.
25This task takes the final output from fgcmFitCycle and produces the following
26outputs for use in the DM stack: the FGCM standard stars in a reference
27catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29task can transfer the 'absolute' calibration from a reference catalog
30to put the fgcm standard stars in units of Jansky. This is accomplished
31by matching stars in a sample of healpix pixels, and applying the median
32offset per band.
33"""
34import sys
35import traceback
36import copy
38import numpy as np
39import healpy as hp
40import esutil
41from astropy import units
43import lsst.daf.base as dafBase
44import lsst.pex.config as pexConfig
45import lsst.pipe.base as pipeBase
46from lsst.pipe.base import connectionTypes
47from lsst.afw.image import TransmissionCurve
48from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
49from lsst.meas.algorithms import ReferenceObjectLoader
50from lsst.pipe.tasks.photoCal import PhotoCalTask
51import lsst.geom
52import lsst.afw.image as afwImage
53import lsst.afw.math as afwMath
54import lsst.afw.table as afwTable
55from lsst.meas.algorithms import IndexerRegistry
56from lsst.meas.algorithms import DatasetConfig
57from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
59from .utilities import computeApproxPixelAreaFields
60from .utilities import lookupStaticCalibrations
62import fgcm
64__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
67class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
68 dimensions=("instrument",),
69 defaultTemplates={"cycleNumber": "0"}):
70 camera = connectionTypes.PrerequisiteInput(
71 doc="Camera instrument",
72 name="camera",
73 storageClass="Camera",
74 dimensions=("instrument",),
75 lookupFunction=lookupStaticCalibrations,
76 isCalibration=True,
77 )
79 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
80 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
81 "chromatic corrections."),
82 name="fgcmLookUpTable",
83 storageClass="Catalog",
84 dimensions=("instrument",),
85 deferLoad=True,
86 )
88 fgcmVisitCatalog = connectionTypes.Input(
89 doc="Catalog of visit information for fgcm",
90 name="fgcmVisitCatalog",
91 storageClass="Catalog",
92 dimensions=("instrument",),
93 deferLoad=True,
94 )
96 fgcmStandardStars = connectionTypes.Input(
97 doc="Catalog of standard star data from fgcm fit",
98 name="fgcmStandardStars{cycleNumber}",
99 storageClass="SimpleCatalog",
100 dimensions=("instrument",),
101 deferLoad=True,
102 )
104 fgcmZeropoints = connectionTypes.Input(
105 doc="Catalog of zeropoints from fgcm fit",
106 name="fgcmZeropoints{cycleNumber}",
107 storageClass="Catalog",
108 dimensions=("instrument",),
109 deferLoad=True,
110 )
112 fgcmAtmosphereParameters = connectionTypes.Input(
113 doc="Catalog of atmosphere parameters from fgcm fit",
114 name="fgcmAtmosphereParameters{cycleNumber}",
115 storageClass="Catalog",
116 dimensions=("instrument",),
117 deferLoad=True,
118 )
120 refCat = connectionTypes.PrerequisiteInput(
121 doc="Reference catalog to use for photometric calibration",
122 name="cal_ref_cat",
123 storageClass="SimpleCatalog",
124 dimensions=("skypix",),
125 deferLoad=True,
126 multiple=True,
127 )
129 fgcmPhotoCalib = connectionTypes.Output(
130 doc=("Per-visit photometric calibrations derived from fgcm calibration. "
131 "These catalogs use detector id for the id and are sorted for "
132 "fast lookups of a detector."),
133 name="fgcmPhotoCalibCatalog",
134 storageClass="ExposureCatalog",
135 dimensions=("instrument", "visit",),
136 multiple=True,
137 )
139 fgcmTransmissionAtmosphere = connectionTypes.Output(
140 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
141 name="transmission_atmosphere_fgcm",
142 storageClass="TransmissionCurve",
143 dimensions=("instrument",
144 "visit",),
145 multiple=True,
146 )
148 fgcmOffsets = connectionTypes.Output(
149 doc="Per-band offsets computed from doReferenceCalibration",
150 name="fgcmReferenceCalibrationOffsets",
151 storageClass="Catalog",
152 dimensions=("instrument",),
153 multiple=False,
154 )
156 def __init__(self, *, config=None):
157 super().__init__(config=config)
159 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
160 raise ValueError("cycleNumber must be of integer format")
161 if config.connections.refCat != config.refObjLoader.ref_dataset_name:
162 raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name")
164 if config.doRefcatOutput:
165 raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
167 if not config.doReferenceCalibration:
168 self.prerequisiteInputs.remove("refCat")
169 if not config.doAtmosphereOutput:
170 self.inputs.remove("fgcmAtmosphereParameters")
171 if not config.doZeropointOutput:
172 self.inputs.remove("fgcmZeropoints")
173 if not config.doReferenceCalibration:
174 self.outputs.remove("fgcmOffsets")
177class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
178 pipelineConnections=FgcmOutputProductsConnections):
179 """Config for FgcmOutputProductsTask"""
181 cycleNumber = pexConfig.Field(
182 doc="Final fit cycle from FGCM fit",
183 dtype=int,
184 default=None,
185 )
186 physicalFilterMap = pexConfig.DictField(
187 doc="Mapping from 'physicalFilter' to band.",
188 keytype=str,
189 itemtype=str,
190 default={},
191 )
192 # The following fields refer to calibrating from a reference
193 # catalog, but in the future this might need to be expanded
194 doReferenceCalibration = pexConfig.Field(
195 doc=("Transfer 'absolute' calibration from reference catalog? "
196 "This afterburner step is unnecessary if reference stars "
197 "were used in the full fit in FgcmFitCycleTask."),
198 dtype=bool,
199 default=False,
200 )
201 doRefcatOutput = pexConfig.Field(
202 doc="Output standard stars in reference catalog format",
203 dtype=bool,
204 default=True,
205 )
206 doAtmosphereOutput = pexConfig.Field(
207 doc="Output atmospheres in transmission_atmosphere_fgcm format",
208 dtype=bool,
209 default=True,
210 )
211 doZeropointOutput = pexConfig.Field(
212 doc="Output zeropoints in fgcm_photoCalib format",
213 dtype=bool,
214 default=True,
215 )
216 doComposeWcsJacobian = pexConfig.Field(
217 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
218 dtype=bool,
219 default=True,
220 )
221 doApplyMeanChromaticCorrection = pexConfig.Field(
222 doc="Apply the mean chromatic correction to the zeropoints?",
223 dtype=bool,
224 default=True,
225 )
226 refObjLoader = pexConfig.ConfigurableField(
227 target=LoadIndexedReferenceObjectsTask,
228 doc="reference object loader for 'absolute' photometric calibration",
229 )
230 photoCal = pexConfig.ConfigurableField(
231 target=PhotoCalTask,
232 doc="task to perform 'absolute' calibration",
233 )
234 referencePixelizationNside = pexConfig.Field(
235 doc="Healpix nside to pixelize catalog to compare to reference catalog",
236 dtype=int,
237 default=64,
238 )
239 referencePixelizationMinStars = pexConfig.Field(
240 doc=("Minimum number of stars per healpix pixel to select for comparison"
241 "to the specified reference catalog"),
242 dtype=int,
243 default=200,
244 )
245 referenceMinMatch = pexConfig.Field(
246 doc="Minimum number of stars matched to reference catalog to be used in statistics",
247 dtype=int,
248 default=50,
249 )
250 referencePixelizationNPixels = pexConfig.Field(
251 doc=("Number of healpix pixels to sample to do comparison. "
252 "Doing too many will take a long time and not yield any more "
253 "precise results because the final number is the median offset "
254 "(per band) from the set of pixels."),
255 dtype=int,
256 default=100,
257 )
258 datasetConfig = pexConfig.ConfigField(
259 dtype=DatasetConfig,
260 doc="Configuration for writing/reading ingested catalog",
261 )
263 def setDefaults(self):
264 pexConfig.Config.setDefaults(self)
266 # In order to transfer the "absolute" calibration from a reference
267 # catalog to the relatively calibrated FGCM standard stars (one number
268 # per band), we use the PhotoCalTask to match stars in a sample of healpix
269 # pixels. These basic settings ensure that only well-measured, good stars
270 # from the source and reference catalogs are used for the matching.
272 # applyColorTerms needs to be False if doReferenceCalibration is False,
273 # as is the new default after DM-16702
274 self.photoCal.applyColorTerms = False
275 self.photoCal.fluxField = 'instFlux'
276 self.photoCal.magErrFloor = 0.003
277 self.photoCal.match.referenceSelection.doSignalToNoise = True
278 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
279 self.photoCal.match.sourceSelection.doSignalToNoise = True
280 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
281 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
282 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
283 self.photoCal.match.sourceSelection.doFlags = True
284 self.photoCal.match.sourceSelection.flags.good = []
285 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
286 self.photoCal.match.sourceSelection.doUnresolved = False
287 self.datasetConfig.ref_dataset_name = 'fgcm_stars'
288 self.datasetConfig.format_version = 1
290 def validate(self):
291 super().validate()
293 # Force the connections to conform with cycleNumber
294 self.connections.cycleNumber = str(self.cycleNumber)
297class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
298 """Subclass of TaskRunner for fgcmOutputProductsTask
300 fgcmOutputProductsTask.run() takes one argument, the butler, and
301 does not run on any data in the repository.
302 This runner does not use any parallelization.
303 """
305 @staticmethod
306 def getTargetList(parsedCmd):
307 """
308 Return a list with one element, the butler.
309 """
310 return [parsedCmd.butler]
312 def __call__(self, butler):
313 """
314 Parameters
315 ----------
316 butler: `lsst.daf.persistence.Butler`
318 Returns
319 -------
320 exitStatus: `list` with `pipeBase.Struct`
321 exitStatus (0: success; 1: failure)
322 if self.doReturnResults also
323 results (`np.array` with absolute zeropoint offsets)
324 """
325 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
327 exitStatus = 0
328 if self.doRaise:
329 results = task.runDataRef(butler)
330 else:
331 try:
332 results = task.runDataRef(butler)
333 except Exception as e:
334 exitStatus = 1
335 task.log.fatal("Failed: %s" % e)
336 if not isinstance(e, pipeBase.TaskError):
337 traceback.print_exc(file=sys.stderr)
339 task.writeMetadata(butler)
341 if self.doReturnResults:
342 # The results here are the zeropoint offsets for each band
343 return [pipeBase.Struct(exitStatus=exitStatus,
344 results=results)]
345 else:
346 return [pipeBase.Struct(exitStatus=exitStatus)]
348 def run(self, parsedCmd):
349 """
350 Run the task, with no multiprocessing
352 Parameters
353 ----------
354 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
355 """
357 resultList = []
359 if self.precall(parsedCmd):
360 targetList = self.getTargetList(parsedCmd)
361 # make sure that we only get 1
362 resultList = self(targetList[0])
364 return resultList
367class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
368 """
369 Output products from FGCM global calibration.
370 """
372 ConfigClass = FgcmOutputProductsConfig
373 RunnerClass = FgcmOutputProductsRunner
374 _DefaultName = "fgcmOutputProducts"
376 def __init__(self, butler=None, **kwargs):
377 super().__init__(**kwargs)
379 # no saving of metadata for now
380 def _getMetadataName(self):
381 return None
383 def runQuantum(self, butlerQC, inputRefs, outputRefs):
384 dataRefDict = {}
385 dataRefDict['camera'] = butlerQC.get(inputRefs.camera)
386 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
387 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
388 dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
390 if self.config.doZeropointOutput:
391 dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
392 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
393 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
395 if self.config.doAtmosphereOutput:
396 dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
397 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
398 atmRef in outputRefs.fgcmTransmissionAtmosphere}
400 if self.config.doReferenceCalibration:
401 refConfig = self.config.refObjLoader
402 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
403 for ref in inputRefs.refCat],
404 refCats=butlerQC.get(inputRefs.refCat),
405 config=refConfig,
406 log=self.log)
407 else:
408 self.refObjLoader = None
410 struct = self.run(dataRefDict, self.config.physicalFilterMap, returnCatalogs=True)
412 # Output the photoCalib exposure catalogs
413 if struct.photoCalibCatalogs is not None:
414 self.log.info("Outputting photoCalib catalogs.")
415 for visit, expCatalog in struct.photoCalibCatalogs:
416 butlerQC.put(expCatalog, photoCalibRefDict[visit])
417 self.log.info("Done outputting photoCalib catalogs.")
419 # Output the atmospheres
420 if struct.atmospheres is not None:
421 self.log.info("Outputting atmosphere transmission files.")
422 for visit, atm in struct.atmospheres:
423 butlerQC.put(atm, atmRefDict[visit])
424 self.log.info("Done outputting atmosphere files.")
426 if self.config.doReferenceCalibration:
427 # Turn offset into simple catalog for persistence if necessary
428 schema = afwTable.Schema()
429 schema.addField('offset', type=np.float64,
430 doc="Post-process calibration offset (mag)")
431 offsetCat = afwTable.BaseCatalog(schema)
432 offsetCat.resize(len(struct.offsets))
433 offsetCat['offset'][:] = struct.offsets
435 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
437 return
439 @pipeBase.timeMethod
440 def runDataRef(self, butler):
441 """
442 Make FGCM output products for use in the stack
444 Parameters
445 ----------
446 butler: `lsst.daf.persistence.Butler`
447 cycleNumber: `int`
448 Final fit cycle number, override config.
450 Returns
451 -------
452 offsets: `lsst.pipe.base.Struct`
453 A structure with array of zeropoint offsets
455 Raises
456 ------
457 RuntimeError:
458 Raised if any one of the following is true:
460 - butler cannot find "fgcmBuildStars_config" or
461 "fgcmBuildStarsTable_config".
462 - butler cannot find "fgcmFitCycle_config".
463 - "fgcmFitCycle_config" does not refer to
464 `self.config.cycleNumber`.
465 - butler cannot find "fgcmAtmosphereParameters" and
466 `self.config.doAtmosphereOutput` is `True`.
467 - butler cannot find "fgcmStandardStars" and
468 `self.config.doReferenceCalibration` is `True` or
469 `self.config.doRefcatOutput` is `True`.
470 - butler cannot find "fgcmZeropoints" and
471 `self.config.doZeropointOutput` is `True`.
472 """
473 if self.config.doReferenceCalibration:
474 # We need the ref obj loader to get the flux field
475 self.makeSubtask("refObjLoader", butler=butler)
477 # Check to make sure that the fgcmBuildStars config exists, to retrieve
478 # the visit and ccd dataset tags
479 if not butler.datasetExists('fgcmBuildStarsTable_config') and \
480 not butler.datasetExists('fgcmBuildStars_config'):
481 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
482 "which is prereq for fgcmOutputProducts")
484 if butler.datasetExists('fgcmBuildStarsTable_config'):
485 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
486 else:
487 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
488 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
489 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
490 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
492 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
493 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
494 "in fgcmBuildStarsTask.")
496 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
497 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
499 # And make sure that the atmosphere was output properly
500 if (self.config.doAtmosphereOutput
501 and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
502 raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
504 if not butler.datasetExists('fgcmStandardStars',
505 fgcmcycle=self.config.cycleNumber):
506 raise RuntimeError("Standard stars are missing for cycle %d." %
507 (self.config.cycleNumber))
509 if (self.config.doZeropointOutput
510 and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
511 raise RuntimeError("Zeropoints are missing for cycle %d." %
512 (self.config.cycleNumber))
514 dataRefDict = {}
515 # This is the _actual_ camera
516 dataRefDict['camera'] = butler.get('camera')
517 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
518 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
519 dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars',
520 fgcmcycle=self.config.cycleNumber)
522 if self.config.doZeropointOutput:
523 dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints',
524 fgcmcycle=self.config.cycleNumber)
525 if self.config.doAtmosphereOutput:
526 dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters',
527 fgcmcycle=self.config.cycleNumber)
529 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False)
531 if struct.photoCalibs is not None:
532 self.log.info("Outputting photoCalib files.")
534 for visit, detector, physicalFilter, photoCalib in struct.photoCalibs:
535 butler.put(photoCalib, 'fgcm_photoCalib',
536 dataId={visitDataRefName: visit,
537 ccdDataRefName: detector,
538 'filter': physicalFilter})
540 self.log.info("Done outputting photoCalib files.")
542 if struct.atmospheres is not None:
543 self.log.info("Outputting atmosphere transmission files.")
544 for visit, atm in struct.atmospheres:
545 butler.put(atm, "transmission_atmosphere_fgcm",
546 dataId={visitDataRefName: visit})
547 self.log.info("Done outputting atmosphere transmissions.")
549 return pipeBase.Struct(offsets=struct.offsets)
551 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
552 """Run the output products task.
554 Parameters
555 ----------
556 dataRefDict : `dict`
557 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
558 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
559 dataRef dictionary with keys:
561 ``"camera"``
562 Camera object (`lsst.afw.cameraGeom.Camera`)
563 ``"fgcmLookUpTable"``
564 dataRef for the FGCM look-up table.
565 ``"fgcmVisitCatalog"``
566 dataRef for visit summary catalog.
567 ``"fgcmStandardStars"``
568 dataRef for the output standard star catalog.
569 ``"fgcmZeropoints"``
570 dataRef for the zeropoint data catalog.
571 ``"fgcmAtmosphereParameters"``
572 dataRef for the atmosphere parameter catalog.
573 ``"fgcmBuildStarsTableConfig"``
574 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
575 physicalFilterMap : `dict`
576 Dictionary of mappings from physical filter to FGCM band.
577 returnCatalogs : `bool`, optional
578 Return photoCalibs as per-visit exposure catalogs.
579 butler : `lsst.daf.persistence.Butler`, optional
580 Gen2 butler used for reference star outputs
582 Returns
583 -------
584 retStruct : `lsst.pipe.base.Struct`
585 Output structure with keys:
587 offsets : `np.ndarray`
588 Final reference offsets, per band.
589 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
590 Generator that returns (visit, transmissionCurve) tuples.
591 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
592 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
593 (returned if returnCatalogs is False).
594 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
595 Generator that returns (visit, exposureCatalog) tuples.
596 (returned if returnCatalogs is True).
597 """
598 stdCat = dataRefDict['fgcmStandardStars'].get()
599 md = stdCat.getMetadata()
600 bands = md.getArray('BANDS')
602 if self.config.doReferenceCalibration:
603 lutCat = dataRefDict['fgcmLookUpTable'].get()
604 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands)
605 else:
606 offsets = np.zeros(len(bands))
608 # This is Gen2 only, and requires the butler.
609 if self.config.doRefcatOutput and butler is not None:
610 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
612 del stdCat
614 if self.config.doZeropointOutput:
615 zptCat = dataRefDict['fgcmZeropoints'].get()
616 visitCat = dataRefDict['fgcmVisitCatalog'].get()
618 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
619 physicalFilterMap, returnCatalogs=returnCatalogs)
620 else:
621 pcgen = None
623 if self.config.doAtmosphereOutput:
624 atmCat = dataRefDict['fgcmAtmosphereParameters'].get()
625 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
626 else:
627 atmgen = None
629 retStruct = pipeBase.Struct(offsets=offsets,
630 atmospheres=atmgen)
631 if returnCatalogs:
632 retStruct.photoCalibCatalogs = pcgen
633 else:
634 retStruct.photoCalibs = pcgen
636 return retStruct
638 def generateTractOutputProducts(self, dataRefDict, tract,
639 visitCat, zptCat, atmCat, stdCat,
640 fgcmBuildStarsConfig,
641 returnCatalogs=True,
642 butler=None):
643 """
644 Generate the output products for a given tract, as specified in the config.
646 This method is here to have an alternate entry-point for
647 FgcmCalibrateTract.
649 Parameters
650 ----------
651 dataRefDict : `dict`
652 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
653 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
654 dataRef dictionary with keys:
656 ``"camera"``
657 Camera object (`lsst.afw.cameraGeom.Camera`)
658 ``"fgcmLookUpTable"``
659 dataRef for the FGCM look-up table.
660 tract : `int`
661 Tract number
662 visitCat : `lsst.afw.table.BaseCatalog`
663 FGCM visitCat from `FgcmBuildStarsTask`
664 zptCat : `lsst.afw.table.BaseCatalog`
665 FGCM zeropoint catalog from `FgcmFitCycleTask`
666 atmCat : `lsst.afw.table.BaseCatalog`
667 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
668 stdCat : `lsst.afw.table.SimpleCatalog`
669 FGCM standard star catalog from `FgcmFitCycleTask`
670 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
671 Configuration object from `FgcmBuildStarsTask`
672 returnCatalogs : `bool`, optional
673 Return photoCalibs as per-visit exposure catalogs.
674 butler: `lsst.daf.persistence.Butler`, optional
675 Gen2 butler used for reference star outputs
677 Returns
678 -------
679 retStruct : `lsst.pipe.base.Struct`
680 Output structure with keys:
682 offsets : `np.ndarray`
683 Final reference offsets, per band.
684 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
685 Generator that returns (visit, transmissionCurve) tuples.
686 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
687 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
688 (returned if returnCatalogs is False).
689 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
690 Generator that returns (visit, exposureCatalog) tuples.
691 (returned if returnCatalogs is True).
692 """
693 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
695 md = stdCat.getMetadata()
696 bands = md.getArray('BANDS')
698 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
699 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
700 "in fgcmBuildStarsTask.")
702 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
703 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
705 if self.config.doReferenceCalibration:
706 lutCat = dataRefDict['fgcmLookUpTable'].get()
707 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap)
708 else:
709 offsets = np.zeros(len(bands))
711 if self.config.doRefcatOutput and butler is not None:
712 # Create a special config that has the tract number in it
713 datasetConfig = copy.copy(self.config.datasetConfig)
714 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
715 tract)
716 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
718 if self.config.doZeropointOutput:
719 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
720 physicalFilterMap, returnCatalogs=returnCatalogs)
721 else:
722 pcgen = None
724 if self.config.doAtmosphereOutput:
725 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
726 else:
727 atmgen = None
729 retStruct = pipeBase.Struct(offsets=offsets,
730 atmospheres=atmgen)
731 if returnCatalogs:
732 retStruct.photoCalibCatalogs = pcgen
733 else:
734 retStruct.photoCalibs = pcgen
736 return retStruct
738 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
739 """
740 Compute offsets relative to a reference catalog.
742 This method splits the star catalog into healpix pixels
743 and computes the calibration transfer for a sample of
744 these pixels to approximate the 'absolute' calibration
745 values (on for each band) to apply to transfer the
746 absolute scale.
748 Parameters
749 ----------
750 stdCat : `lsst.afw.table.SimpleCatalog`
751 FGCM standard stars
752 lutCat : `lsst.afw.table.SimpleCatalog`
753 FGCM Look-up table
754 physicalFilterMap : `dict`
755 Dictionary of mappings from physical filter to FGCM band.
756 bands : `list` [`str`]
757 List of band names from FGCM output
758 Returns
759 -------
760 offsets : `numpy.array` of floats
761 Per band zeropoint offsets
762 """
764 # Only use stars that are observed in all the bands that were actually used
765 # This will ensure that we use the same healpix pixels for the absolute
766 # calibration of each band.
767 minObs = stdCat['ngood'].min(axis=1)
769 goodStars = (minObs >= 1)
770 stdCat = stdCat[goodStars]
772 self.log.info("Found %d stars with at least 1 good observation in each band" %
773 (len(stdCat)))
775 # Associate each band with the appropriate physicalFilter and make
776 # filterLabels
777 filterLabels = []
779 lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',')
780 lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',')
781 physicalFilterMapBands = list(physicalFilterMap.values())
782 physicalFilterMapFilters = list(physicalFilterMap.keys())
783 for band in bands:
784 # Find a physical filter associated from the band by doing
785 # a reverse lookup on the physicalFilterMap dict
786 physicalFilterMapIndex = physicalFilterMapBands.index(band)
787 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
788 # Find the appropriate fgcm standard physicalFilter
789 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
790 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
791 filterLabels.append(afwImage.FilterLabel(band=band,
792 physical=stdPhysicalFilter))
794 # We have to make a table for each pixel with flux/fluxErr
795 # This is a temporary table generated for input to the photoCal task.
796 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
797 # have had chromatic corrections applied to get to the standard system
798 # specified by the atmosphere/instrumental parameters), nor are they
799 # in Jansky (since they don't have a proper absolute calibration: the overall
800 # zeropoint is estimated from the telescope size, etc.)
801 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
802 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
803 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
804 doc="instrumental flux (counts)")
805 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
806 doc="instrumental flux error (counts)")
807 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
808 type='Flag',
809 doc="bad flag")
811 # Split up the stars
812 # Note that there is an assumption here that the ra/dec coords stored
813 # on-disk are in radians, and therefore that starObs['coord_ra'] /
814 # starObs['coord_dec'] return radians when used as an array of numpy float64s.
815 theta = np.pi/2. - stdCat['coord_dec']
816 phi = stdCat['coord_ra']
818 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
819 h, rev = esutil.stat.histogram(ipring, rev=True)
821 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
823 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
824 (gdpix.size,
825 self.config.referencePixelizationNside,
826 self.config.referencePixelizationMinStars))
828 if gdpix.size < self.config.referencePixelizationNPixels:
829 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
830 (gdpix.size, self.config.referencePixelizationNPixels))
831 else:
832 # Sample out the pixels we want to use
833 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
835 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
836 ('nstar', 'i4', len(bands)),
837 ('nmatch', 'i4', len(bands)),
838 ('zp', 'f4', len(bands)),
839 ('zpErr', 'f4', len(bands))])
840 results['hpix'] = ipring[rev[rev[gdpix]]]
842 # We need a boolean index to deal with catalogs...
843 selected = np.zeros(len(stdCat), dtype=bool)
845 refFluxFields = [None]*len(bands)
847 for p_index, pix in enumerate(gdpix):
848 i1a = rev[rev[pix]: rev[pix + 1]]
850 # the stdCat afwTable can only be indexed with boolean arrays,
851 # and not numpy index arrays (see DM-16497). This little trick
852 # converts the index array into a boolean array
853 selected[:] = False
854 selected[i1a] = True
856 for b_index, filterLabel in enumerate(filterLabels):
857 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index,
858 filterLabel, stdCat,
859 selected, refFluxFields)
860 results['nstar'][p_index, b_index] = len(i1a)
861 results['nmatch'][p_index, b_index] = len(struct.arrays.refMag)
862 results['zp'][p_index, b_index] = struct.zp
863 results['zpErr'][p_index, b_index] = struct.sigma
865 # And compute the summary statistics
866 offsets = np.zeros(len(bands))
868 for b_index, band in enumerate(bands):
869 # make configurable
870 ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch)
871 offsets[b_index] = np.median(results['zp'][ok, b_index])
872 # use median absolute deviation to estimate Normal sigma
873 # see https://en.wikipedia.org/wiki/Median_absolute_deviation
874 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index]))
875 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f",
876 band, offsets[b_index], madSigma)
878 return offsets
880 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
881 b_index, filterLabel, stdCat, selected, refFluxFields):
882 """
883 Compute the zeropoint offset between the fgcm stdCat and the reference
884 stars for one pixel in one band
886 Parameters
887 ----------
888 sourceMapper : `lsst.afw.table.SchemaMapper`
889 Mapper to go from stdCat to calibratable catalog
890 badStarKey : `lsst.afw.table.Key`
891 Key for the field with bad stars
892 b_index : `int`
893 Index of the band in the star catalog
894 filterLabel : `lsst.afw.image.FilterLabel`
895 filterLabel with band and physical filter
896 stdCat : `lsst.afw.table.SimpleCatalog`
897 FGCM standard stars
898 selected : `numpy.array(dtype=bool)`
899 Boolean array of which stars are in the pixel
900 refFluxFields : `list`
901 List of names of flux fields for reference catalog
902 """
904 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
905 sourceCat.reserve(selected.sum())
906 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
907 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5))
908 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index]
909 * sourceCat['instFlux'])
910 # Make sure we only use stars that have valid measurements
911 # (This is perhaps redundant with requirements above that the
912 # stars be observed in all bands, but it can't hurt)
913 badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0)
914 for rec in sourceCat[badStar]:
915 rec.set(badStarKey, True)
917 exposure = afwImage.ExposureF()
918 exposure.setFilterLabel(filterLabel)
920 if refFluxFields[b_index] is None:
921 # Need to find the flux field in the reference catalog
922 # to work around limitations of DirectMatch in PhotoCal
923 ctr = stdCat[0].getCoord()
924 rad = 0.05*lsst.geom.degrees
925 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel)
926 refFluxFields[b_index] = refDataTest.fluxField
928 # Make a copy of the config so that we can modify it
929 calConfig = copy.copy(self.config.photoCal.value)
930 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index]
931 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err'
932 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
933 config=calConfig,
934 schema=sourceCat.getSchema())
936 struct = calTask.run(exposure, sourceCat)
938 return struct
940 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
941 """
942 Output standard stars in indexed reference catalog format.
943 This is not currently supported in Gen3.
945 Parameters
946 ----------
947 butler : `lsst.daf.persistence.Butler`
948 stdCat : `lsst.afw.table.SimpleCatalog`
949 FGCM standard star catalog from fgcmFitCycleTask
950 offsets : `numpy.array` of floats
951 Per band zeropoint offsets
952 bands : `list` [`str`]
953 List of band names from FGCM output
954 datasetConfig : `lsst.meas.algorithms.DatasetConfig`
955 Config for reference dataset
956 """
958 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
960 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
961 self.config.datasetConfig.indexer.active)
963 # We determine the conversion from the native units (typically radians) to
964 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
965 # numpy arrays rather than Angles, which would we approximately 600x slower.
966 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
967 # (as Angles) for input
968 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
969 indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv,
970 stdCat['coord_dec']*conv))
972 formattedCat = self._formatCatalog(stdCat, offsets, bands)
974 # Write the master schema
975 dataId = indexer.makeDataId('master_schema',
976 datasetConfig.ref_dataset_name)
977 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
978 addRefCatMetadata(masterCat)
979 butler.put(masterCat, 'ref_cat', dataId=dataId)
981 # Break up the pixels using a histogram
982 h, rev = esutil.stat.histogram(indices, rev=True)
983 gd, = np.where(h > 0)
984 selected = np.zeros(len(formattedCat), dtype=bool)
985 for i in gd:
986 i1a = rev[rev[i]: rev[i + 1]]
988 # the formattedCat afwTable can only be indexed with boolean arrays,
989 # and not numpy index arrays (see DM-16497). This little trick
990 # converts the index array into a boolean array
991 selected[:] = False
992 selected[i1a] = True
994 # Write the individual pixel
995 dataId = indexer.makeDataId(indices[i1a[0]],
996 datasetConfig.ref_dataset_name)
997 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
999 # And save the dataset configuration
1000 dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name)
1001 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
1003 self.log.info("Done outputting standard stars.")
1005 def _formatCatalog(self, fgcmStarCat, offsets, bands):
1006 """
1007 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
1009 Parameters
1010 ----------
1011 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
1012 SimpleCatalog as output by fgcmcal
1013 offsets : `list` with len(self.bands) entries
1014 Zeropoint offsets to apply
1015 bands : `list` [`str`]
1016 List of band names from FGCM output
1018 Returns
1019 -------
1020 formattedCat: `lsst.afw.table.SimpleCatalog`
1021 SimpleCatalog suitable for using as a reference catalog
1022 """
1024 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1025 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1026 addCentroid=False,
1027 addIsResolved=True,
1028 coordErrDim=0)
1029 sourceMapper.addMinimalSchema(minSchema)
1030 for band in bands:
1031 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
1032 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
1033 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
1035 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1036 formattedCat.reserve(len(fgcmStarCat))
1037 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1039 # Note that we don't have to set `resolved` because the default is False
1041 for b, band in enumerate(bands):
1042 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1043 # We want fluxes in nJy from calibrated AB magnitudes
1044 # (after applying offset). Updated after RFC-549 and RFC-575.
1045 flux = (mag*units.ABmag).to_value(units.nJy)
1046 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
1048 formattedCat['%s_flux' % (band)][:] = flux
1049 formattedCat['%s_fluxErr' % (band)][:] = fluxErr
1050 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
1051 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
1052 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
1054 addRefCatMetadata(formattedCat)
1056 return formattedCat
1058 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1059 physicalFilterMap, returnCatalogs=True,
1060 tract=None):
1061 """Output the zeropoints in fgcm_photoCalib format.
1063 Parameters
1064 ----------
1065 camera : `lsst.afw.cameraGeom.Camera`
1066 Camera from the butler.
1067 zptCat : `lsst.afw.table.BaseCatalog`
1068 FGCM zeropoint catalog from `FgcmFitCycleTask`.
1069 visitCat : `lsst.afw.table.BaseCatalog`
1070 FGCM visitCat from `FgcmBuildStarsTask`.
1071 offsets : `numpy.array`
1072 Float array of absolute calibration offsets, one for each filter.
1073 bands : `list` [`str`]
1074 List of band names from FGCM output.
1075 physicalFilterMap : `dict`
1076 Dictionary of mappings from physical filter to FGCM band.
1077 returnCatalogs : `bool`, optional
1078 Return photoCalibs as per-visit exposure catalogs.
1079 tract: `int`, optional
1080 Tract number to output. Default is None (global calibration)
1082 Returns
1083 -------
1084 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1085 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1086 (returned if returnCatalogs is False).
1087 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1088 Generator that returns (visit, exposureCatalog) tuples.
1089 (returned if returnCatalogs is True).
1090 """
1091 # Select visit/ccds where we have a calibration
1092 # This includes ccds where we were able to interpolate from neighboring
1093 # ccds.
1094 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
1095 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
1096 & (zptCat['fgcmZptVar'] > 0.0))
1098 # Log warnings for any visit which has no valid zeropoints
1099 badVisits = np.unique(zptCat['visit'][~selected])
1100 goodVisits = np.unique(zptCat['visit'][selected])
1101 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1102 for allBadVisit in allBadVisits:
1103 self.log.warn(f'No suitable photoCalib for visit {allBadVisit}')
1105 # Get a mapping from filtername to the offsets
1106 offsetMapping = {}
1107 for f in physicalFilterMap:
1108 # Not every filter in the map will necesarily have a band.
1109 if physicalFilterMap[f] in bands:
1110 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1112 # Get a mapping from "ccd" to the ccd index used for the scaling
1113 ccdMapping = {}
1114 for ccdIndex, detector in enumerate(camera):
1115 ccdMapping[detector.getId()] = ccdIndex
1117 # And a mapping to get the flat-field scaling values
1118 scalingMapping = {}
1119 for rec in visitCat:
1120 scalingMapping[rec['visit']] = rec['scaling']
1122 if self.config.doComposeWcsJacobian:
1123 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
1125 # The zptCat is sorted by visit, which is useful
1126 lastVisit = -1
1127 zptVisitCatalog = None
1129 metadata = dafBase.PropertyList()
1130 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
1131 metadata.add("COMMENT", "Only detectors with data have entries.")
1133 for rec in zptCat[selected]:
1134 # Retrieve overall scaling
1135 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
1137 # The postCalibrationOffset describe any zeropoint offsets
1138 # to apply after the fgcm calibration. The first part comes
1139 # from the reference catalog match (used in testing). The
1140 # second part comes from the mean chromatic correction
1141 # (if configured).
1142 postCalibrationOffset = offsetMapping[rec['filtername']]
1143 if self.config.doApplyMeanChromaticCorrection:
1144 postCalibrationOffset += rec['fgcmDeltaChrom']
1146 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
1147 rec['fgcmfZptChebXyMax'])
1148 # Convert from FGCM AB to nJy
1149 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
1150 rec['fgcmfZptChebXyMax'],
1151 offset=postCalibrationOffset,
1152 scaling=scaling)
1154 if self.config.doComposeWcsJacobian:
1156 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']],
1157 fgcmSuperStarField,
1158 fgcmZptField])
1159 else:
1160 # The photoCalib is just the product of the fgcmSuperStarField and the
1161 # fgcmZptField
1162 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1164 # The "mean" calibration will be set to the center of the ccd for reference
1165 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1166 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
1167 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1168 calibrationErr=calibErr,
1169 calibration=fgcmField,
1170 isConstant=False)
1172 if not returnCatalogs:
1173 # Return individual photoCalibs
1174 yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib)
1175 else:
1176 # Return full per-visit exposure catalogs
1177 if rec['visit'] != lastVisit:
1178 # This is a new visit. If the last visit was not -1, yield
1179 # the ExposureCatalog
1180 if lastVisit > -1:
1181 # ensure that the detectors are in sorted order, for fast lookups
1182 zptVisitCatalog.sort()
1183 yield (int(lastVisit), zptVisitCatalog)
1184 else:
1185 # We need to create a new schema
1186 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1187 zptExpCatSchema.addField('visit', type='I', doc='Visit number')
1189 # And start a new one
1190 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1191 zptVisitCatalog.setMetadata(metadata)
1193 lastVisit = int(rec['visit'])
1195 catRecord = zptVisitCatalog.addNew()
1196 catRecord['id'] = int(rec['detector'])
1197 catRecord['visit'] = rec['visit']
1198 catRecord.setPhotoCalib(photoCalib)
1200 # Final output of last exposure catalog
1201 if returnCatalogs:
1202 # ensure that the detectors are in sorted order, for fast lookups
1203 zptVisitCatalog.sort()
1204 yield (int(lastVisit), zptVisitCatalog)
1206 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1207 """
1208 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1209 and scaling.
1211 Parameters
1212 ----------
1213 coefficients: `numpy.array`
1214 Flattened array of chebyshev coefficients
1215 xyMax: `list` of length 2
1216 Maximum x and y of the chebyshev bounding box
1217 offset: `float`, optional
1218 Absolute calibration offset. Default is 0.0
1219 scaling: `float`, optional
1220 Flat scaling value from fgcmBuildStars. Default is 1.0
1222 Returns
1223 -------
1224 boundedField: `lsst.afw.math.ChebyshevBoundedField`
1225 """
1227 orderPlus1 = int(np.sqrt(coefficients.size))
1228 pars = np.zeros((orderPlus1, orderPlus1))
1230 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1231 lsst.geom.Point2I(*xyMax))
1233 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1234 * (10.**(offset/-2.5))*scaling)
1236 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1238 return boundedField
1240 def _outputAtmospheres(self, dataRefDict, atmCat):
1241 """
1242 Output the atmospheres.
1244 Parameters
1245 ----------
1246 dataRefDict : `dict`
1247 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1248 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1249 dataRef dictionary with keys:
1251 ``"fgcmLookUpTable"``
1252 dataRef for the FGCM look-up table.
1253 atmCat : `lsst.afw.table.BaseCatalog`
1254 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1256 Returns
1257 -------
1258 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1259 Generator that returns (visit, transmissionCurve) tuples.
1260 """
1261 # First, we need to grab the look-up table and key info
1262 lutCat = dataRefDict['fgcmLookUpTable'].get()
1264 atmosphereTableName = lutCat[0]['tablename']
1265 elevation = lutCat[0]['elevation']
1266 atmLambda = lutCat[0]['atmLambda']
1267 lutCat = None
1269 # Make the atmosphere table if possible
1270 try:
1271 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1272 atmTable.loadTable()
1273 except IOError:
1274 atmTable = None
1276 if atmTable is None:
1277 # Try to use MODTRAN instead
1278 try:
1279 modGen = fgcm.ModtranGenerator(elevation)
1280 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1281 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1282 except (ValueError, IOError) as e:
1283 raise RuntimeError("FGCM look-up-table generated with modtran, "
1284 "but modtran not configured to run.") from e
1286 zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
1288 for i, visit in enumerate(atmCat['visit']):
1289 if atmTable is not None:
1290 # Interpolate the atmosphere table
1291 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
1292 pwv=atmCat[i]['pwv'],
1293 o3=atmCat[i]['o3'],
1294 tau=atmCat[i]['tau'],
1295 alpha=atmCat[i]['alpha'],
1296 zenith=zenith[i],
1297 ctranslamstd=[atmCat[i]['cTrans'],
1298 atmCat[i]['lamStd']])
1299 else:
1300 # Run modtran
1301 modAtm = modGen(pmb=atmCat[i]['pmb'],
1302 pwv=atmCat[i]['pwv'],
1303 o3=atmCat[i]['o3'],
1304 tau=atmCat[i]['tau'],
1305 alpha=atmCat[i]['alpha'],
1306 zenith=zenith[i],
1307 lambdaRange=lambdaRange,
1308 lambdaStep=lambdaStep,
1309 ctranslamstd=[atmCat[i]['cTrans'],
1310 atmCat[i]['lamStd']])
1311 atmVals = modAtm['COMBINED']
1313 # Now need to create something to persist...
1314 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1315 wavelengths=atmLambda,
1316 throughputAtMin=atmVals[0],
1317 throughputAtMax=atmVals[-1])
1319 yield (int(visit), curve)