Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Make the final fgcmcal output products.
25This task takes the final output from fgcmFitCycle and produces the following
26outputs for use in the DM stack: the FGCM standard stars in a reference
27catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29task can transfer the 'absolute' calibration from a reference catalog
30to put the fgcm standard stars in units of Jansky. This is accomplished
31by matching stars in a sample of healpix pixels, and applying the median
32offset per band.
33"""
34import sys
35import traceback
36import copy
38import numpy as np
39import healpy as hp
40import esutil
41from astropy import units
43import lsst.daf.base as dafBase
44import lsst.pex.config as pexConfig
45import lsst.pipe.base as pipeBase
46from lsst.pipe.base import connectionTypes
47from lsst.afw.image import TransmissionCurve
48from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
49from lsst.meas.algorithms import ReferenceObjectLoader
50from lsst.pipe.tasks.photoCal import PhotoCalTask
51import lsst.geom
52import lsst.afw.image as afwImage
53import lsst.afw.math as afwMath
54import lsst.afw.table as afwTable
55from lsst.meas.algorithms import IndexerRegistry
56from lsst.meas.algorithms import DatasetConfig
57from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
59from .utilities import computeApproxPixelAreaFields
60from .utilities import lookupStaticCalibrations
62import fgcm
64__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
67class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
68 dimensions=("instrument",),
69 defaultTemplates={"cycleNumber": "0"}):
70 camera = connectionTypes.PrerequisiteInput(
71 doc="Camera instrument",
72 name="camera",
73 storageClass="Camera",
74 dimensions=("instrument",),
75 lookupFunction=lookupStaticCalibrations,
76 isCalibration=True,
77 )
79 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
80 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
81 "chromatic corrections."),
82 name="fgcmLookUpTable",
83 storageClass="Catalog",
84 dimensions=("instrument",),
85 deferLoad=True,
86 )
88 fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
89 doc="Catalog of visit information for fgcm",
90 name="fgcmVisitCatalog",
91 storageClass="Catalog",
92 dimensions=("instrument",),
93 deferLoad=True,
94 )
96 fgcmStandardStars = connectionTypes.PrerequisiteInput(
97 doc="Catalog of standard star data from fgcm fit",
98 name="fgcmStandardStars{cycleNumber}",
99 storageClass="SimpleCatalog",
100 dimensions=("instrument",),
101 deferLoad=True,
102 )
104 fgcmZeropoints = connectionTypes.PrerequisiteInput(
105 doc="Catalog of zeropoints from fgcm fit",
106 name="fgcmZeropoints{cycleNumber}",
107 storageClass="Catalog",
108 dimensions=("instrument",),
109 deferLoad=True,
110 )
112 fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
113 doc="Catalog of atmosphere parameters from fgcm fit",
114 name="fgcmAtmosphereParameters{cycleNumber}",
115 storageClass="Catalog",
116 dimensions=("instrument",),
117 deferLoad=True,
118 )
120 refCat = connectionTypes.PrerequisiteInput(
121 doc="Reference catalog to use for photometric calibration",
122 name="cal_ref_cat",
123 storageClass="SimpleCatalog",
124 dimensions=("skypix",),
125 deferLoad=True,
126 multiple=True,
127 )
129 fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
130 doc="Config used to build FGCM input stars",
131 name="fgcmBuildStarsTable_config",
132 storageClass="Config",
133 )
135 fgcmPhotoCalib = connectionTypes.Output(
136 doc=("Per-visit photometric calibrations derived from fgcm calibration. "
137 "These catalogs use detector id for the id and are sorted for "
138 "fast lookups of a detector."),
139 name="fgcmPhotoCalibCatalog",
140 storageClass="ExposureCatalog",
141 dimensions=("instrument", "visit",),
142 multiple=True,
143 )
145 fgcmTransmissionAtmosphere = connectionTypes.Output(
146 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
147 name="transmission_atmosphere_fgcm",
148 storageClass="TransmissionCurve",
149 dimensions=("instrument",
150 "visit",),
151 multiple=True,
152 )
154 fgcmOffsets = connectionTypes.Output(
155 doc="Per-band offsets computed from doReferenceCalibration",
156 name="fgcmReferenceCalibrationOffsets",
157 storageClass="Catalog",
158 dimensions=("instrument",),
159 multiple=False,
160 )
162 def __init__(self, *, config=None):
163 super().__init__(config=config)
165 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
166 raise ValueError("cycleNumber must be of integer format")
167 if config.connections.refCat != config.refObjLoader.ref_dataset_name:
168 raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name")
170 if config.doRefcatOutput:
171 raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
173 if not config.doReferenceCalibration:
174 self.prerequisiteInputs.remove("refCat")
175 if not config.doAtmosphereOutput:
176 self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
177 if not config.doZeropointOutput:
178 self.prerequisiteInputs.remove("fgcmZeropoints")
179 if not config.doReferenceCalibration:
180 self.outputs.remove("fgcmOffsets")
183class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
184 pipelineConnections=FgcmOutputProductsConnections):
185 """Config for FgcmOutputProductsTask"""
187 cycleNumber = pexConfig.Field(
188 doc="Final fit cycle from FGCM fit",
189 dtype=int,
190 default=None,
191 )
193 # The following fields refer to calibrating from a reference
194 # catalog, but in the future this might need to be expanded
195 doReferenceCalibration = pexConfig.Field(
196 doc=("Transfer 'absolute' calibration from reference catalog? "
197 "This afterburner step is unnecessary if reference stars "
198 "were used in the full fit in FgcmFitCycleTask."),
199 dtype=bool,
200 default=False,
201 )
202 doRefcatOutput = pexConfig.Field(
203 doc="Output standard stars in reference catalog format",
204 dtype=bool,
205 default=True,
206 )
207 doAtmosphereOutput = pexConfig.Field(
208 doc="Output atmospheres in transmission_atmosphere_fgcm format",
209 dtype=bool,
210 default=True,
211 )
212 doZeropointOutput = pexConfig.Field(
213 doc="Output zeropoints in fgcm_photoCalib format",
214 dtype=bool,
215 default=True,
216 )
217 doComposeWcsJacobian = pexConfig.Field(
218 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
219 dtype=bool,
220 default=True,
221 )
222 doApplyMeanChromaticCorrection = pexConfig.Field(
223 doc="Apply the mean chromatic correction to the zeropoints?",
224 dtype=bool,
225 default=True,
226 )
227 refObjLoader = pexConfig.ConfigurableField(
228 target=LoadIndexedReferenceObjectsTask,
229 doc="reference object loader for 'absolute' photometric calibration",
230 )
231 photoCal = pexConfig.ConfigurableField(
232 target=PhotoCalTask,
233 doc="task to perform 'absolute' calibration",
234 )
235 referencePixelizationNside = pexConfig.Field(
236 doc="Healpix nside to pixelize catalog to compare to reference catalog",
237 dtype=int,
238 default=64,
239 )
240 referencePixelizationMinStars = pexConfig.Field(
241 doc=("Minimum number of stars per healpix pixel to select for comparison"
242 "to the specified reference catalog"),
243 dtype=int,
244 default=200,
245 )
246 referenceMinMatch = pexConfig.Field(
247 doc="Minimum number of stars matched to reference catalog to be used in statistics",
248 dtype=int,
249 default=50,
250 )
251 referencePixelizationNPixels = pexConfig.Field(
252 doc=("Number of healpix pixels to sample to do comparison. "
253 "Doing too many will take a long time and not yield any more "
254 "precise results because the final number is the median offset "
255 "(per band) from the set of pixels."),
256 dtype=int,
257 default=100,
258 )
259 datasetConfig = pexConfig.ConfigField(
260 dtype=DatasetConfig,
261 doc="Configuration for writing/reading ingested catalog",
262 )
264 def setDefaults(self):
265 pexConfig.Config.setDefaults(self)
267 # In order to transfer the "absolute" calibration from a reference
268 # catalog to the relatively calibrated FGCM standard stars (one number
269 # per band), we use the PhotoCalTask to match stars in a sample of healpix
270 # pixels. These basic settings ensure that only well-measured, good stars
271 # from the source and reference catalogs are used for the matching.
273 # applyColorTerms needs to be False if doReferenceCalibration is False,
274 # as is the new default after DM-16702
275 self.photoCal.applyColorTerms = False
276 self.photoCal.fluxField = 'instFlux'
277 self.photoCal.magErrFloor = 0.003
278 self.photoCal.match.referenceSelection.doSignalToNoise = True
279 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
280 self.photoCal.match.sourceSelection.doSignalToNoise = True
281 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
282 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
283 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
284 self.photoCal.match.sourceSelection.doFlags = True
285 self.photoCal.match.sourceSelection.flags.good = []
286 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
287 self.photoCal.match.sourceSelection.doUnresolved = False
288 self.datasetConfig.ref_dataset_name = 'fgcm_stars'
289 self.datasetConfig.format_version = 1
291 def validate(self):
292 super().validate()
294 # Force the connections to conform with cycleNumber
295 self.connections.cycleNumber = str(self.cycleNumber)
298class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
299 """Subclass of TaskRunner for fgcmOutputProductsTask
301 fgcmOutputProductsTask.run() takes one argument, the butler, and
302 does not run on any data in the repository.
303 This runner does not use any parallelization.
304 """
306 @staticmethod
307 def getTargetList(parsedCmd):
308 """
309 Return a list with one element, the butler.
310 """
311 return [parsedCmd.butler]
313 def __call__(self, butler):
314 """
315 Parameters
316 ----------
317 butler: `lsst.daf.persistence.Butler`
319 Returns
320 -------
321 exitStatus: `list` with `pipeBase.Struct`
322 exitStatus (0: success; 1: failure)
323 if self.doReturnResults also
324 results (`np.array` with absolute zeropoint offsets)
325 """
326 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
328 exitStatus = 0
329 if self.doRaise:
330 results = task.runDataRef(butler)
331 else:
332 try:
333 results = task.runDataRef(butler)
334 except Exception as e:
335 exitStatus = 1
336 task.log.fatal("Failed: %s" % e)
337 if not isinstance(e, pipeBase.TaskError):
338 traceback.print_exc(file=sys.stderr)
340 task.writeMetadata(butler)
342 if self.doReturnResults:
343 # The results here are the zeropoint offsets for each band
344 return [pipeBase.Struct(exitStatus=exitStatus,
345 results=results)]
346 else:
347 return [pipeBase.Struct(exitStatus=exitStatus)]
349 def run(self, parsedCmd):
350 """
351 Run the task, with no multiprocessing
353 Parameters
354 ----------
355 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
356 """
358 resultList = []
360 if self.precall(parsedCmd):
361 targetList = self.getTargetList(parsedCmd)
362 # make sure that we only get 1
363 resultList = self(targetList[0])
365 return resultList
368class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
369 """
370 Output products from FGCM global calibration.
371 """
373 ConfigClass = FgcmOutputProductsConfig
374 RunnerClass = FgcmOutputProductsRunner
375 _DefaultName = "fgcmOutputProducts"
377 def __init__(self, butler=None, **kwargs):
378 super().__init__(**kwargs)
380 # no saving of metadata for now
381 def _getMetadataName(self):
382 return None
384 def runQuantum(self, butlerQC, inputRefs, outputRefs):
385 dataRefDict = {}
386 dataRefDict['camera'] = butlerQC.get(inputRefs.camera)
387 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
388 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
389 dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
391 if self.config.doZeropointOutput:
392 dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
393 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
394 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
396 if self.config.doAtmosphereOutput:
397 dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
398 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
399 atmRef in outputRefs.fgcmTransmissionAtmosphere}
401 if self.config.doReferenceCalibration:
402 refConfig = self.config.refObjLoader
403 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
404 for ref in inputRefs.refCat],
405 refCats=butlerQC.get(inputRefs.refCat),
406 config=refConfig,
407 log=self.log)
408 else:
409 self.refObjLoader = None
411 dataRefDict['fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
413 fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
414 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
416 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
417 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
418 "in fgcmBuildStarsTask.")
419 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
420 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
422 struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=True)
424 # Output the photoCalib exposure catalogs
425 if struct.photoCalibCatalogs is not None:
426 self.log.info("Outputting photoCalib catalogs.")
427 for visit, expCatalog in struct.photoCalibCatalogs:
428 butlerQC.put(expCatalog, photoCalibRefDict[visit])
429 self.log.info("Done outputting photoCalib catalogs.")
431 # Output the atmospheres
432 if struct.atmospheres is not None:
433 self.log.info("Outputting atmosphere transmission files.")
434 for visit, atm in struct.atmospheres:
435 butlerQC.put(atm, atmRefDict[visit])
436 self.log.info("Done outputting atmosphere files.")
438 if self.config.doReferenceCalibration:
439 # Turn offset into simple catalog for persistence if necessary
440 schema = afwTable.Schema()
441 schema.addField('offset', type=np.float64,
442 doc="Post-process calibration offset (mag)")
443 offsetCat = afwTable.BaseCatalog(schema)
444 offsetCat.resize(len(struct.offsets))
445 offsetCat['offset'][:] = struct.offsets
447 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
449 return
451 @pipeBase.timeMethod
452 def runDataRef(self, butler):
453 """
454 Make FGCM output products for use in the stack
456 Parameters
457 ----------
458 butler: `lsst.daf.persistence.Butler`
459 cycleNumber: `int`
460 Final fit cycle number, override config.
462 Returns
463 -------
464 offsets: `lsst.pipe.base.Struct`
465 A structure with array of zeropoint offsets
467 Raises
468 ------
469 RuntimeError:
470 Raised if any one of the following is true:
472 - butler cannot find "fgcmBuildStars_config" or
473 "fgcmBuildStarsTable_config".
474 - butler cannot find "fgcmFitCycle_config".
475 - "fgcmFitCycle_config" does not refer to
476 `self.config.cycleNumber`.
477 - butler cannot find "fgcmAtmosphereParameters" and
478 `self.config.doAtmosphereOutput` is `True`.
479 - butler cannot find "fgcmStandardStars" and
480 `self.config.doReferenceCalibration` is `True` or
481 `self.config.doRefcatOutput` is `True`.
482 - butler cannot find "fgcmZeropoints" and
483 `self.config.doZeropointOutput` is `True`.
484 """
485 if self.config.doReferenceCalibration:
486 # We need the ref obj loader to get the flux field
487 self.makeSubtask("refObjLoader", butler=butler)
489 # Check to make sure that the fgcmBuildStars config exists, to retrieve
490 # the visit and ccd dataset tags
491 if not butler.datasetExists('fgcmBuildStarsTable_config') and \
492 not butler.datasetExists('fgcmBuildStars_config'):
493 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
494 "which is prereq for fgcmOutputProducts")
496 if butler.datasetExists('fgcmBuildStarsTable_config'):
497 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
498 else:
499 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
500 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
501 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
502 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
504 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
505 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
506 "in fgcmBuildStarsTask.")
508 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
509 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
511 # And make sure that the atmosphere was output properly
512 if (self.config.doAtmosphereOutput
513 and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
514 raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
516 if not butler.datasetExists('fgcmStandardStars',
517 fgcmcycle=self.config.cycleNumber):
518 raise RuntimeError("Standard stars are missing for cycle %d." %
519 (self.config.cycleNumber))
521 if (self.config.doZeropointOutput
522 and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
523 raise RuntimeError("Zeropoints are missing for cycle %d." %
524 (self.config.cycleNumber))
526 dataRefDict = {}
527 # This is the _actual_ camera
528 dataRefDict['camera'] = butler.get('camera')
529 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
530 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
531 dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars',
532 fgcmcycle=self.config.cycleNumber)
534 if self.config.doZeropointOutput:
535 dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints',
536 fgcmcycle=self.config.cycleNumber)
537 if self.config.doAtmosphereOutput:
538 dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters',
539 fgcmcycle=self.config.cycleNumber)
541 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False)
543 if struct.photoCalibs is not None:
544 self.log.info("Outputting photoCalib files.")
546 for visit, detector, physicalFilter, photoCalib in struct.photoCalibs:
547 butler.put(photoCalib, 'fgcm_photoCalib',
548 dataId={visitDataRefName: visit,
549 ccdDataRefName: detector,
550 'filter': physicalFilter})
552 self.log.info("Done outputting photoCalib files.")
554 if struct.atmospheres is not None:
555 self.log.info("Outputting atmosphere transmission files.")
556 for visit, atm in struct.atmospheres:
557 butler.put(atm, "transmission_atmosphere_fgcm",
558 dataId={visitDataRefName: visit})
559 self.log.info("Done outputting atmosphere transmissions.")
561 return pipeBase.Struct(offsets=struct.offsets)
563 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
564 """Run the output products task.
566 Parameters
567 ----------
568 dataRefDict : `dict`
569 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
570 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
571 dataRef dictionary with keys:
573 ``"camera"``
574 Camera object (`lsst.afw.cameraGeom.Camera`)
575 ``"fgcmLookUpTable"``
576 dataRef for the FGCM look-up table.
577 ``"fgcmVisitCatalog"``
578 dataRef for visit summary catalog.
579 ``"fgcmStandardStars"``
580 dataRef for the output standard star catalog.
581 ``"fgcmZeropoints"``
582 dataRef for the zeropoint data catalog.
583 ``"fgcmAtmosphereParameters"``
584 dataRef for the atmosphere parameter catalog.
585 ``"fgcmBuildStarsTableConfig"``
586 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
587 physicalFilterMap : `dict`
588 Dictionary of mappings from physical filter to FGCM band.
589 returnCatalogs : `bool`, optional
590 Return photoCalibs as per-visit exposure catalogs.
591 butler : `lsst.daf.persistence.Butler`, optional
592 Gen2 butler used for reference star outputs
594 Returns
595 -------
596 retStruct : `lsst.pipe.base.Struct`
597 Output structure with keys:
599 offsets : `np.ndarray`
600 Final reference offsets, per band.
601 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
602 Generator that returns (visit, transmissionCurve) tuples.
603 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
604 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
605 (returned if returnCatalogs is False).
606 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
607 Generator that returns (visit, exposureCatalog) tuples.
608 (returned if returnCatalogs is True).
609 """
610 stdCat = dataRefDict['fgcmStandardStars'].get()
611 md = stdCat.getMetadata()
612 bands = md.getArray('BANDS')
614 if self.config.doReferenceCalibration:
615 lutCat = dataRefDict['fgcmLookUpTable'].get()
616 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands)
617 else:
618 offsets = np.zeros(len(bands))
620 # This is Gen2 only, and requires the butler.
621 if self.config.doRefcatOutput and butler is not None:
622 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
624 del stdCat
626 if self.config.doZeropointOutput:
627 zptCat = dataRefDict['fgcmZeropoints'].get()
628 visitCat = dataRefDict['fgcmVisitCatalog'].get()
630 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
631 physicalFilterMap, returnCatalogs=returnCatalogs)
632 else:
633 pcgen = None
635 if self.config.doAtmosphereOutput:
636 atmCat = dataRefDict['fgcmAtmosphereParameters'].get()
637 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
638 else:
639 atmgen = None
641 retStruct = pipeBase.Struct(offsets=offsets,
642 atmospheres=atmgen)
643 if returnCatalogs:
644 retStruct.photoCalibCatalogs = pcgen
645 else:
646 retStruct.photoCalibs = pcgen
648 return retStruct
650 def generateTractOutputProducts(self, dataRefDict, tract,
651 visitCat, zptCat, atmCat, stdCat,
652 fgcmBuildStarsConfig,
653 returnCatalogs=True,
654 butler=None):
655 """
656 Generate the output products for a given tract, as specified in the config.
658 This method is here to have an alternate entry-point for
659 FgcmCalibrateTract.
661 Parameters
662 ----------
663 dataRefDict : `dict`
664 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
665 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
666 dataRef dictionary with keys:
668 ``"camera"``
669 Camera object (`lsst.afw.cameraGeom.Camera`)
670 ``"fgcmLookUpTable"``
671 dataRef for the FGCM look-up table.
672 tract : `int`
673 Tract number
674 visitCat : `lsst.afw.table.BaseCatalog`
675 FGCM visitCat from `FgcmBuildStarsTask`
676 zptCat : `lsst.afw.table.BaseCatalog`
677 FGCM zeropoint catalog from `FgcmFitCycleTask`
678 atmCat : `lsst.afw.table.BaseCatalog`
679 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
680 stdCat : `lsst.afw.table.SimpleCatalog`
681 FGCM standard star catalog from `FgcmFitCycleTask`
682 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
683 Configuration object from `FgcmBuildStarsTask`
684 returnCatalogs : `bool`, optional
685 Return photoCalibs as per-visit exposure catalogs.
686 butler: `lsst.daf.persistence.Butler`, optional
687 Gen2 butler used for reference star outputs
689 Returns
690 -------
691 retStruct : `lsst.pipe.base.Struct`
692 Output structure with keys:
694 offsets : `np.ndarray`
695 Final reference offsets, per band.
696 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
697 Generator that returns (visit, transmissionCurve) tuples.
698 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
699 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
700 (returned if returnCatalogs is False).
701 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
702 Generator that returns (visit, exposureCatalog) tuples.
703 (returned if returnCatalogs is True).
704 """
705 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
707 md = stdCat.getMetadata()
708 bands = md.getArray('BANDS')
710 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
711 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
712 "in fgcmBuildStarsTask.")
714 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
715 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
717 if self.config.doReferenceCalibration:
718 lutCat = dataRefDict['fgcmLookUpTable'].get()
719 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap)
720 else:
721 offsets = np.zeros(len(bands))
723 if self.config.doRefcatOutput and butler is not None:
724 # Create a special config that has the tract number in it
725 datasetConfig = copy.copy(self.config.datasetConfig)
726 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
727 tract)
728 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
730 if self.config.doZeropointOutput:
731 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
732 physicalFilterMap, returnCatalogs=returnCatalogs)
733 else:
734 pcgen = None
736 if self.config.doAtmosphereOutput:
737 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
738 else:
739 atmgen = None
741 retStruct = pipeBase.Struct(offsets=offsets,
742 atmospheres=atmgen)
743 if returnCatalogs:
744 retStruct.photoCalibCatalogs = pcgen
745 else:
746 retStruct.photoCalibs = pcgen
748 return retStruct
750 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
751 """
752 Compute offsets relative to a reference catalog.
754 This method splits the star catalog into healpix pixels
755 and computes the calibration transfer for a sample of
756 these pixels to approximate the 'absolute' calibration
757 values (on for each band) to apply to transfer the
758 absolute scale.
760 Parameters
761 ----------
762 stdCat : `lsst.afw.table.SimpleCatalog`
763 FGCM standard stars
764 lutCat : `lsst.afw.table.SimpleCatalog`
765 FGCM Look-up table
766 physicalFilterMap : `dict`
767 Dictionary of mappings from physical filter to FGCM band.
768 bands : `list` [`str`]
769 List of band names from FGCM output
770 Returns
771 -------
772 offsets : `numpy.array` of floats
773 Per band zeropoint offsets
774 """
776 # Only use stars that are observed in all the bands that were actually used
777 # This will ensure that we use the same healpix pixels for the absolute
778 # calibration of each band.
779 minObs = stdCat['ngood'].min(axis=1)
781 goodStars = (minObs >= 1)
782 stdCat = stdCat[goodStars]
784 self.log.info("Found %d stars with at least 1 good observation in each band" %
785 (len(stdCat)))
787 # Associate each band with the appropriate physicalFilter and make
788 # filterLabels
789 filterLabels = []
791 lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',')
792 lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',')
793 physicalFilterMapBands = list(physicalFilterMap.values())
794 physicalFilterMapFilters = list(physicalFilterMap.keys())
795 for band in bands:
796 # Find a physical filter associated from the band by doing
797 # a reverse lookup on the physicalFilterMap dict
798 physicalFilterMapIndex = physicalFilterMapBands.index(band)
799 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
800 # Find the appropriate fgcm standard physicalFilter
801 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
802 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
803 filterLabels.append(afwImage.FilterLabel(band=band,
804 physical=stdPhysicalFilter))
806 # We have to make a table for each pixel with flux/fluxErr
807 # This is a temporary table generated for input to the photoCal task.
808 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
809 # have had chromatic corrections applied to get to the standard system
810 # specified by the atmosphere/instrumental parameters), nor are they
811 # in Jansky (since they don't have a proper absolute calibration: the overall
812 # zeropoint is estimated from the telescope size, etc.)
813 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
814 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
815 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
816 doc="instrumental flux (counts)")
817 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
818 doc="instrumental flux error (counts)")
819 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
820 type='Flag',
821 doc="bad flag")
823 # Split up the stars
824 # Note that there is an assumption here that the ra/dec coords stored
825 # on-disk are in radians, and therefore that starObs['coord_ra'] /
826 # starObs['coord_dec'] return radians when used as an array of numpy float64s.
827 theta = np.pi/2. - stdCat['coord_dec']
828 phi = stdCat['coord_ra']
830 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
831 h, rev = esutil.stat.histogram(ipring, rev=True)
833 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
835 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
836 (gdpix.size,
837 self.config.referencePixelizationNside,
838 self.config.referencePixelizationMinStars))
840 if gdpix.size < self.config.referencePixelizationNPixels:
841 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
842 (gdpix.size, self.config.referencePixelizationNPixels))
843 else:
844 # Sample out the pixels we want to use
845 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
847 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
848 ('nstar', 'i4', len(bands)),
849 ('nmatch', 'i4', len(bands)),
850 ('zp', 'f4', len(bands)),
851 ('zpErr', 'f4', len(bands))])
852 results['hpix'] = ipring[rev[rev[gdpix]]]
854 # We need a boolean index to deal with catalogs...
855 selected = np.zeros(len(stdCat), dtype=bool)
857 refFluxFields = [None]*len(bands)
859 for p_index, pix in enumerate(gdpix):
860 i1a = rev[rev[pix]: rev[pix + 1]]
862 # the stdCat afwTable can only be indexed with boolean arrays,
863 # and not numpy index arrays (see DM-16497). This little trick
864 # converts the index array into a boolean array
865 selected[:] = False
866 selected[i1a] = True
868 for b_index, filterLabel in enumerate(filterLabels):
869 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index,
870 filterLabel, stdCat,
871 selected, refFluxFields)
872 results['nstar'][p_index, b_index] = len(i1a)
873 results['nmatch'][p_index, b_index] = len(struct.arrays.refMag)
874 results['zp'][p_index, b_index] = struct.zp
875 results['zpErr'][p_index, b_index] = struct.sigma
877 # And compute the summary statistics
878 offsets = np.zeros(len(bands))
880 for b_index, band in enumerate(bands):
881 # make configurable
882 ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch)
883 offsets[b_index] = np.median(results['zp'][ok, b_index])
884 # use median absolute deviation to estimate Normal sigma
885 # see https://en.wikipedia.org/wiki/Median_absolute_deviation
886 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index]))
887 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f",
888 band, offsets[b_index], madSigma)
890 return offsets
892 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
893 b_index, filterLabel, stdCat, selected, refFluxFields):
894 """
895 Compute the zeropoint offset between the fgcm stdCat and the reference
896 stars for one pixel in one band
898 Parameters
899 ----------
900 sourceMapper : `lsst.afw.table.SchemaMapper`
901 Mapper to go from stdCat to calibratable catalog
902 badStarKey : `lsst.afw.table.Key`
903 Key for the field with bad stars
904 b_index : `int`
905 Index of the band in the star catalog
906 filterLabel : `lsst.afw.image.FilterLabel`
907 filterLabel with band and physical filter
908 stdCat : `lsst.afw.table.SimpleCatalog`
909 FGCM standard stars
910 selected : `numpy.array(dtype=bool)`
911 Boolean array of which stars are in the pixel
912 refFluxFields : `list`
913 List of names of flux fields for reference catalog
914 """
916 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
917 sourceCat.reserve(selected.sum())
918 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
919 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5))
920 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index]
921 * sourceCat['instFlux'])
922 # Make sure we only use stars that have valid measurements
923 # (This is perhaps redundant with requirements above that the
924 # stars be observed in all bands, but it can't hurt)
925 badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0)
926 for rec in sourceCat[badStar]:
927 rec.set(badStarKey, True)
929 exposure = afwImage.ExposureF()
930 exposure.setFilterLabel(filterLabel)
932 if refFluxFields[b_index] is None:
933 # Need to find the flux field in the reference catalog
934 # to work around limitations of DirectMatch in PhotoCal
935 ctr = stdCat[0].getCoord()
936 rad = 0.05*lsst.geom.degrees
937 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel)
938 refFluxFields[b_index] = refDataTest.fluxField
940 # Make a copy of the config so that we can modify it
941 calConfig = copy.copy(self.config.photoCal.value)
942 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index]
943 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err'
944 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
945 config=calConfig,
946 schema=sourceCat.getSchema())
948 struct = calTask.run(exposure, sourceCat)
950 return struct
952 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
953 """
954 Output standard stars in indexed reference catalog format.
955 This is not currently supported in Gen3.
957 Parameters
958 ----------
959 butler : `lsst.daf.persistence.Butler`
960 stdCat : `lsst.afw.table.SimpleCatalog`
961 FGCM standard star catalog from fgcmFitCycleTask
962 offsets : `numpy.array` of floats
963 Per band zeropoint offsets
964 bands : `list` [`str`]
965 List of band names from FGCM output
966 datasetConfig : `lsst.meas.algorithms.DatasetConfig`
967 Config for reference dataset
968 """
970 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
972 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
973 self.config.datasetConfig.indexer.active)
975 # We determine the conversion from the native units (typically radians) to
976 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
977 # numpy arrays rather than Angles, which would we approximately 600x slower.
978 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
979 # (as Angles) for input
980 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
981 indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv,
982 stdCat['coord_dec']*conv))
984 formattedCat = self._formatCatalog(stdCat, offsets, bands)
986 # Write the master schema
987 dataId = indexer.makeDataId('master_schema',
988 datasetConfig.ref_dataset_name)
989 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
990 addRefCatMetadata(masterCat)
991 butler.put(masterCat, 'ref_cat', dataId=dataId)
993 # Break up the pixels using a histogram
994 h, rev = esutil.stat.histogram(indices, rev=True)
995 gd, = np.where(h > 0)
996 selected = np.zeros(len(formattedCat), dtype=bool)
997 for i in gd:
998 i1a = rev[rev[i]: rev[i + 1]]
1000 # the formattedCat afwTable can only be indexed with boolean arrays,
1001 # and not numpy index arrays (see DM-16497). This little trick
1002 # converts the index array into a boolean array
1003 selected[:] = False
1004 selected[i1a] = True
1006 # Write the individual pixel
1007 dataId = indexer.makeDataId(indices[i1a[0]],
1008 datasetConfig.ref_dataset_name)
1009 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
1011 # And save the dataset configuration
1012 dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name)
1013 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
1015 self.log.info("Done outputting standard stars.")
1017 def _formatCatalog(self, fgcmStarCat, offsets, bands):
1018 """
1019 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
1021 Parameters
1022 ----------
1023 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
1024 SimpleCatalog as output by fgcmcal
1025 offsets : `list` with len(self.bands) entries
1026 Zeropoint offsets to apply
1027 bands : `list` [`str`]
1028 List of band names from FGCM output
1030 Returns
1031 -------
1032 formattedCat: `lsst.afw.table.SimpleCatalog`
1033 SimpleCatalog suitable for using as a reference catalog
1034 """
1036 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1037 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1038 addCentroid=False,
1039 addIsResolved=True,
1040 coordErrDim=0)
1041 sourceMapper.addMinimalSchema(minSchema)
1042 for band in bands:
1043 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
1044 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
1045 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
1047 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1048 formattedCat.reserve(len(fgcmStarCat))
1049 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1051 # Note that we don't have to set `resolved` because the default is False
1053 for b, band in enumerate(bands):
1054 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1055 # We want fluxes in nJy from calibrated AB magnitudes
1056 # (after applying offset). Updated after RFC-549 and RFC-575.
1057 flux = (mag*units.ABmag).to_value(units.nJy)
1058 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
1060 formattedCat['%s_flux' % (band)][:] = flux
1061 formattedCat['%s_fluxErr' % (band)][:] = fluxErr
1062 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
1063 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
1064 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
1066 addRefCatMetadata(formattedCat)
1068 return formattedCat
1070 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1071 physicalFilterMap, returnCatalogs=True,
1072 tract=None):
1073 """Output the zeropoints in fgcm_photoCalib format.
1075 Parameters
1076 ----------
1077 camera : `lsst.afw.cameraGeom.Camera`
1078 Camera from the butler.
1079 zptCat : `lsst.afw.table.BaseCatalog`
1080 FGCM zeropoint catalog from `FgcmFitCycleTask`.
1081 visitCat : `lsst.afw.table.BaseCatalog`
1082 FGCM visitCat from `FgcmBuildStarsTask`.
1083 offsets : `numpy.array`
1084 Float array of absolute calibration offsets, one for each filter.
1085 bands : `list` [`str`]
1086 List of band names from FGCM output.
1087 physicalFilterMap : `dict`
1088 Dictionary of mappings from physical filter to FGCM band.
1089 returnCatalogs : `bool`, optional
1090 Return photoCalibs as per-visit exposure catalogs.
1091 tract: `int`, optional
1092 Tract number to output. Default is None (global calibration)
1094 Returns
1095 -------
1096 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1097 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1098 (returned if returnCatalogs is False).
1099 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1100 Generator that returns (visit, exposureCatalog) tuples.
1101 (returned if returnCatalogs is True).
1102 """
1103 # Select visit/ccds where we have a calibration
1104 # This includes ccds where we were able to interpolate from neighboring
1105 # ccds.
1106 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
1107 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
1108 & (zptCat['fgcmZptVar'] > 0.0))
1110 # Log warnings for any visit which has no valid zeropoints
1111 badVisits = np.unique(zptCat['visit'][~selected])
1112 goodVisits = np.unique(zptCat['visit'][selected])
1113 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1114 for allBadVisit in allBadVisits:
1115 self.log.warn(f'No suitable photoCalib for visit {allBadVisit}')
1117 # Get a mapping from filtername to the offsets
1118 offsetMapping = {}
1119 for f in physicalFilterMap:
1120 # Not every filter in the map will necesarily have a band.
1121 if physicalFilterMap[f] in bands:
1122 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1124 # Get a mapping from "ccd" to the ccd index used for the scaling
1125 ccdMapping = {}
1126 for ccdIndex, detector in enumerate(camera):
1127 ccdMapping[detector.getId()] = ccdIndex
1129 # And a mapping to get the flat-field scaling values
1130 scalingMapping = {}
1131 for rec in visitCat:
1132 scalingMapping[rec['visit']] = rec['scaling']
1134 if self.config.doComposeWcsJacobian:
1135 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
1137 # The zptCat is sorted by visit, which is useful
1138 lastVisit = -1
1139 zptVisitCatalog = None
1141 metadata = dafBase.PropertyList()
1142 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
1143 metadata.add("COMMENT", "Only detectors with data have entries.")
1145 for rec in zptCat[selected]:
1146 # Retrieve overall scaling
1147 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
1149 # The postCalibrationOffset describe any zeropoint offsets
1150 # to apply after the fgcm calibration. The first part comes
1151 # from the reference catalog match (used in testing). The
1152 # second part comes from the mean chromatic correction
1153 # (if configured).
1154 postCalibrationOffset = offsetMapping[rec['filtername']]
1155 if self.config.doApplyMeanChromaticCorrection:
1156 postCalibrationOffset += rec['fgcmDeltaChrom']
1158 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
1159 rec['fgcmfZptChebXyMax'])
1160 # Convert from FGCM AB to nJy
1161 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
1162 rec['fgcmfZptChebXyMax'],
1163 offset=postCalibrationOffset,
1164 scaling=scaling)
1166 if self.config.doComposeWcsJacobian:
1168 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']],
1169 fgcmSuperStarField,
1170 fgcmZptField])
1171 else:
1172 # The photoCalib is just the product of the fgcmSuperStarField and the
1173 # fgcmZptField
1174 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1176 # The "mean" calibration will be set to the center of the ccd for reference
1177 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1178 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
1179 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1180 calibrationErr=calibErr,
1181 calibration=fgcmField,
1182 isConstant=False)
1184 if not returnCatalogs:
1185 # Return individual photoCalibs
1186 yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib)
1187 else:
1188 # Return full per-visit exposure catalogs
1189 if rec['visit'] != lastVisit:
1190 # This is a new visit. If the last visit was not -1, yield
1191 # the ExposureCatalog
1192 if lastVisit > -1:
1193 # ensure that the detectors are in sorted order, for fast lookups
1194 zptVisitCatalog.sort()
1195 yield (int(lastVisit), zptVisitCatalog)
1196 else:
1197 # We need to create a new schema
1198 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1199 zptExpCatSchema.addField('visit', type='I', doc='Visit number')
1201 # And start a new one
1202 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1203 zptVisitCatalog.setMetadata(metadata)
1205 lastVisit = int(rec['visit'])
1207 catRecord = zptVisitCatalog.addNew()
1208 catRecord['id'] = int(rec['detector'])
1209 catRecord['visit'] = rec['visit']
1210 catRecord.setPhotoCalib(photoCalib)
1212 # Final output of last exposure catalog
1213 if returnCatalogs:
1214 # ensure that the detectors are in sorted order, for fast lookups
1215 zptVisitCatalog.sort()
1216 yield (int(lastVisit), zptVisitCatalog)
1218 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1219 """
1220 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1221 and scaling.
1223 Parameters
1224 ----------
1225 coefficients: `numpy.array`
1226 Flattened array of chebyshev coefficients
1227 xyMax: `list` of length 2
1228 Maximum x and y of the chebyshev bounding box
1229 offset: `float`, optional
1230 Absolute calibration offset. Default is 0.0
1231 scaling: `float`, optional
1232 Flat scaling value from fgcmBuildStars. Default is 1.0
1234 Returns
1235 -------
1236 boundedField: `lsst.afw.math.ChebyshevBoundedField`
1237 """
1239 orderPlus1 = int(np.sqrt(coefficients.size))
1240 pars = np.zeros((orderPlus1, orderPlus1))
1242 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1243 lsst.geom.Point2I(*xyMax))
1245 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1246 * (10.**(offset/-2.5))*scaling)
1248 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1250 return boundedField
1252 def _outputAtmospheres(self, dataRefDict, atmCat):
1253 """
1254 Output the atmospheres.
1256 Parameters
1257 ----------
1258 dataRefDict : `dict`
1259 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1260 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1261 dataRef dictionary with keys:
1263 ``"fgcmLookUpTable"``
1264 dataRef for the FGCM look-up table.
1265 atmCat : `lsst.afw.table.BaseCatalog`
1266 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1268 Returns
1269 -------
1270 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1271 Generator that returns (visit, transmissionCurve) tuples.
1272 """
1273 # First, we need to grab the look-up table and key info
1274 lutCat = dataRefDict['fgcmLookUpTable'].get()
1276 atmosphereTableName = lutCat[0]['tablename']
1277 elevation = lutCat[0]['elevation']
1278 atmLambda = lutCat[0]['atmLambda']
1279 lutCat = None
1281 # Make the atmosphere table if possible
1282 try:
1283 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1284 atmTable.loadTable()
1285 except IOError:
1286 atmTable = None
1288 if atmTable is None:
1289 # Try to use MODTRAN instead
1290 try:
1291 modGen = fgcm.ModtranGenerator(elevation)
1292 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1293 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1294 except (ValueError, IOError) as e:
1295 raise RuntimeError("FGCM look-up-table generated with modtran, "
1296 "but modtran not configured to run.") from e
1298 zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
1300 for i, visit in enumerate(atmCat['visit']):
1301 if atmTable is not None:
1302 # Interpolate the atmosphere table
1303 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
1304 pwv=atmCat[i]['pwv'],
1305 o3=atmCat[i]['o3'],
1306 tau=atmCat[i]['tau'],
1307 alpha=atmCat[i]['alpha'],
1308 zenith=zenith[i],
1309 ctranslamstd=[atmCat[i]['cTrans'],
1310 atmCat[i]['lamStd']])
1311 else:
1312 # Run modtran
1313 modAtm = modGen(pmb=atmCat[i]['pmb'],
1314 pwv=atmCat[i]['pwv'],
1315 o3=atmCat[i]['o3'],
1316 tau=atmCat[i]['tau'],
1317 alpha=atmCat[i]['alpha'],
1318 zenith=zenith[i],
1319 lambdaRange=lambdaRange,
1320 lambdaStep=lambdaStep,
1321 ctranslamstd=[atmCat[i]['cTrans'],
1322 atmCat[i]['lamStd']])
1323 atmVals = modAtm['COMBINED']
1325 # Now need to create something to persist...
1326 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1327 wavelengths=atmLambda,
1328 throughputAtMin=atmVals[0],
1329 throughputAtMax=atmVals[-1])
1331 yield (int(visit), curve)