Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Make the final fgcmcal output products.
25This task takes the final output from fgcmFitCycle and produces the following
26outputs for use in the DM stack: the FGCM standard stars in a reference
27catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29task can transfer the 'absolute' calibration from a reference catalog
30to put the fgcm standard stars in units of Jansky. This is accomplished
31by matching stars in a sample of healpix pixels, and applying the median
32offset per band.
33"""
34import sys
35import traceback
36import copy
38import numpy as np
39import healpy as hp
40import esutil
41from astropy import units
43import lsst.daf.base as dafBase
44import lsst.pex.config as pexConfig
45import lsst.pipe.base as pipeBase
46from lsst.pipe.base import connectionTypes
47from lsst.afw.image import TransmissionCurve
48from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
49from lsst.meas.algorithms import ReferenceObjectLoader
50from lsst.pipe.tasks.photoCal import PhotoCalTask
51import lsst.geom
52import lsst.afw.image as afwImage
53import lsst.afw.math as afwMath
54import lsst.afw.table as afwTable
55from lsst.meas.algorithms import IndexerRegistry
56from lsst.meas.algorithms import DatasetConfig
57from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
59from .utilities import computeApproxPixelAreaFields
60from .utilities import lookupStaticCalibrations
62import fgcm
64__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
67class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
68 dimensions=("instrument",),
69 defaultTemplates={"cycleNumber": "0"}):
70 camera = connectionTypes.PrerequisiteInput(
71 doc="Camera instrument",
72 name="camera",
73 storageClass="Camera",
74 dimensions=("instrument",),
75 lookupFunction=lookupStaticCalibrations,
76 isCalibration=True,
77 )
79 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
80 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
81 "chromatic corrections."),
82 name="fgcmLookUpTable",
83 storageClass="Catalog",
84 dimensions=("instrument",),
85 deferLoad=True,
86 )
88 fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
89 doc="Catalog of visit information for fgcm",
90 name="fgcmVisitCatalog",
91 storageClass="Catalog",
92 dimensions=("instrument",),
93 deferLoad=True,
94 )
96 fgcmStandardStars = connectionTypes.PrerequisiteInput(
97 doc="Catalog of standard star data from fgcm fit",
98 name="fgcmStandardStars{cycleNumber}",
99 storageClass="SimpleCatalog",
100 dimensions=("instrument",),
101 deferLoad=True,
102 )
104 fgcmZeropoints = connectionTypes.PrerequisiteInput(
105 doc="Catalog of zeropoints from fgcm fit",
106 name="fgcmZeropoints{cycleNumber}",
107 storageClass="Catalog",
108 dimensions=("instrument",),
109 deferLoad=True,
110 )
112 fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
113 doc="Catalog of atmosphere parameters from fgcm fit",
114 name="fgcmAtmosphereParameters{cycleNumber}",
115 storageClass="Catalog",
116 dimensions=("instrument",),
117 deferLoad=True,
118 )
120 refCat = connectionTypes.PrerequisiteInput(
121 doc="Reference catalog to use for photometric calibration",
122 name="cal_ref_cat",
123 storageClass="SimpleCatalog",
124 dimensions=("skypix",),
125 deferLoad=True,
126 multiple=True,
127 )
129 fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
130 doc="Config used to build FGCM input stars",
131 name="fgcmBuildStarsTable_config",
132 storageClass="Config",
133 )
135 fgcmPhotoCalib = connectionTypes.Output(
136 doc=("Per-visit photometric calibrations derived from fgcm calibration. "
137 "These catalogs use detector id for the id and are sorted for "
138 "fast lookups of a detector."),
139 name="fgcmPhotoCalibCatalog",
140 storageClass="ExposureCatalog",
141 dimensions=("instrument", "visit",),
142 multiple=True,
143 )
145 fgcmTransmissionAtmosphere = connectionTypes.Output(
146 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
147 name="transmission_atmosphere_fgcm",
148 storageClass="TransmissionCurve",
149 dimensions=("instrument",
150 "visit",),
151 multiple=True,
152 )
154 fgcmOffsets = connectionTypes.Output(
155 doc="Per-band offsets computed from doReferenceCalibration",
156 name="fgcmReferenceCalibrationOffsets",
157 storageClass="Catalog",
158 dimensions=("instrument",),
159 multiple=False,
160 )
162 def __init__(self, *, config=None):
163 super().__init__(config=config)
165 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
166 raise ValueError("cycleNumber must be of integer format")
167 if config.connections.refCat != config.refObjLoader.ref_dataset_name:
168 raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name")
170 if config.doRefcatOutput:
171 raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
173 if not config.doReferenceCalibration:
174 self.prerequisiteInputs.remove("refCat")
175 if not config.doAtmosphereOutput:
176 self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
177 if not config.doZeropointOutput:
178 self.prerequisiteInputs.remove("fgcmZeropoints")
179 if not config.doReferenceCalibration:
180 self.outputs.remove("fgcmOffsets")
183class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
184 pipelineConnections=FgcmOutputProductsConnections):
185 """Config for FgcmOutputProductsTask"""
187 cycleNumber = pexConfig.Field(
188 doc="Final fit cycle from FGCM fit",
189 dtype=int,
190 default=None,
191 )
193 # The following fields refer to calibrating from a reference
194 # catalog, but in the future this might need to be expanded
195 doReferenceCalibration = pexConfig.Field(
196 doc=("Transfer 'absolute' calibration from reference catalog? "
197 "This afterburner step is unnecessary if reference stars "
198 "were used in the full fit in FgcmFitCycleTask."),
199 dtype=bool,
200 default=False,
201 )
202 doRefcatOutput = pexConfig.Field(
203 doc="Output standard stars in reference catalog format",
204 dtype=bool,
205 default=True,
206 )
207 doAtmosphereOutput = pexConfig.Field(
208 doc="Output atmospheres in transmission_atmosphere_fgcm format",
209 dtype=bool,
210 default=True,
211 )
212 doZeropointOutput = pexConfig.Field(
213 doc="Output zeropoints in fgcm_photoCalib format",
214 dtype=bool,
215 default=True,
216 )
217 doComposeWcsJacobian = pexConfig.Field(
218 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
219 dtype=bool,
220 default=True,
221 )
222 doApplyMeanChromaticCorrection = pexConfig.Field(
223 doc="Apply the mean chromatic correction to the zeropoints?",
224 dtype=bool,
225 default=True,
226 )
227 refObjLoader = pexConfig.ConfigurableField(
228 target=LoadIndexedReferenceObjectsTask,
229 doc="reference object loader for 'absolute' photometric calibration",
230 )
231 photoCal = pexConfig.ConfigurableField(
232 target=PhotoCalTask,
233 doc="task to perform 'absolute' calibration",
234 )
235 referencePixelizationNside = pexConfig.Field(
236 doc="Healpix nside to pixelize catalog to compare to reference catalog",
237 dtype=int,
238 default=64,
239 )
240 referencePixelizationMinStars = pexConfig.Field(
241 doc=("Minimum number of stars per healpix pixel to select for comparison"
242 "to the specified reference catalog"),
243 dtype=int,
244 default=200,
245 )
246 referenceMinMatch = pexConfig.Field(
247 doc="Minimum number of stars matched to reference catalog to be used in statistics",
248 dtype=int,
249 default=50,
250 )
251 referencePixelizationNPixels = pexConfig.Field(
252 doc=("Number of healpix pixels to sample to do comparison. "
253 "Doing too many will take a long time and not yield any more "
254 "precise results because the final number is the median offset "
255 "(per band) from the set of pixels."),
256 dtype=int,
257 default=100,
258 )
259 datasetConfig = pexConfig.ConfigField(
260 dtype=DatasetConfig,
261 doc="Configuration for writing/reading ingested catalog",
262 )
264 def setDefaults(self):
265 pexConfig.Config.setDefaults(self)
267 # In order to transfer the "absolute" calibration from a reference
268 # catalog to the relatively calibrated FGCM standard stars (one number
269 # per band), we use the PhotoCalTask to match stars in a sample of healpix
270 # pixels. These basic settings ensure that only well-measured, good stars
271 # from the source and reference catalogs are used for the matching.
273 # applyColorTerms needs to be False if doReferenceCalibration is False,
274 # as is the new default after DM-16702
275 self.photoCal.applyColorTerms = False
276 self.photoCal.fluxField = 'instFlux'
277 self.photoCal.magErrFloor = 0.003
278 self.photoCal.match.referenceSelection.doSignalToNoise = True
279 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
280 self.photoCal.match.sourceSelection.doSignalToNoise = True
281 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
282 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
283 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
284 self.photoCal.match.sourceSelection.doFlags = True
285 self.photoCal.match.sourceSelection.flags.good = []
286 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
287 self.photoCal.match.sourceSelection.doUnresolved = False
288 self.datasetConfig.ref_dataset_name = 'fgcm_stars'
289 self.datasetConfig.format_version = 1
291 def validate(self):
292 super().validate()
294 # Force the connections to conform with cycleNumber
295 self.connections.cycleNumber = str(self.cycleNumber)
298class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
299 """Subclass of TaskRunner for fgcmOutputProductsTask
301 fgcmOutputProductsTask.run() takes one argument, the butler, and
302 does not run on any data in the repository.
303 This runner does not use any parallelization.
304 """
306 @staticmethod
307 def getTargetList(parsedCmd):
308 """
309 Return a list with one element, the butler.
310 """
311 return [parsedCmd.butler]
313 def __call__(self, butler):
314 """
315 Parameters
316 ----------
317 butler: `lsst.daf.persistence.Butler`
319 Returns
320 -------
321 exitStatus: `list` with `pipeBase.Struct`
322 exitStatus (0: success; 1: failure)
323 if self.doReturnResults also
324 results (`np.array` with absolute zeropoint offsets)
325 """
326 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
328 exitStatus = 0
329 if self.doRaise:
330 results = task.runDataRef(butler)
331 else:
332 try:
333 results = task.runDataRef(butler)
334 except Exception as e:
335 exitStatus = 1
336 task.log.fatal("Failed: %s" % e)
337 if not isinstance(e, pipeBase.TaskError):
338 traceback.print_exc(file=sys.stderr)
340 task.writeMetadata(butler)
342 if self.doReturnResults:
343 # The results here are the zeropoint offsets for each band
344 return [pipeBase.Struct(exitStatus=exitStatus,
345 results=results)]
346 else:
347 return [pipeBase.Struct(exitStatus=exitStatus)]
349 def run(self, parsedCmd):
350 """
351 Run the task, with no multiprocessing
353 Parameters
354 ----------
355 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
356 """
358 resultList = []
360 if self.precall(parsedCmd):
361 targetList = self.getTargetList(parsedCmd)
362 # make sure that we only get 1
363 resultList = self(targetList[0])
365 return resultList
368class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
369 """
370 Output products from FGCM global calibration.
371 """
373 ConfigClass = FgcmOutputProductsConfig
374 RunnerClass = FgcmOutputProductsRunner
375 _DefaultName = "fgcmOutputProducts"
377 def __init__(self, butler=None, **kwargs):
378 super().__init__(**kwargs)
380 # no saving of metadata for now
381 def _getMetadataName(self):
382 return None
384 def runQuantum(self, butlerQC, inputRefs, outputRefs):
385 dataRefDict = {}
386 dataRefDict['camera'] = butlerQC.get(inputRefs.camera)
387 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
388 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
389 dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
391 if self.config.doZeropointOutput:
392 dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
393 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
394 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
396 if self.config.doAtmosphereOutput:
397 dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
398 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
399 atmRef in outputRefs.fgcmTransmissionAtmosphere}
401 if self.config.doReferenceCalibration:
402 refConfig = self.config.refObjLoader
403 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
404 for ref in inputRefs.refCat],
405 refCats=butlerQC.get(inputRefs.refCat),
406 config=refConfig,
407 log=self.log)
408 else:
409 self.refObjLoader = None
411 dataRefDict['fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
413 fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
414 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
416 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
417 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
418 "in fgcmBuildStarsTask.")
419 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
420 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
422 struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=True)
424 # Output the photoCalib exposure catalogs
425 if struct.photoCalibCatalogs is not None:
426 self.log.info("Outputting photoCalib catalogs.")
427 for visit, expCatalog in struct.photoCalibCatalogs:
428 butlerQC.put(expCatalog, photoCalibRefDict[visit])
429 self.log.info("Done outputting photoCalib catalogs.")
431 # Output the atmospheres
432 if struct.atmospheres is not None:
433 self.log.info("Outputting atmosphere transmission files.")
434 for visit, atm in struct.atmospheres:
435 butlerQC.put(atm, atmRefDict[visit])
436 self.log.info("Done outputting atmosphere files.")
438 if self.config.doReferenceCalibration:
439 # Turn offset into simple catalog for persistence if necessary
440 schema = afwTable.Schema()
441 schema.addField('offset', type=np.float64,
442 doc="Post-process calibration offset (mag)")
443 offsetCat = afwTable.BaseCatalog(schema)
444 offsetCat.resize(len(struct.offsets))
445 offsetCat['offset'][:] = struct.offsets
447 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
449 return
451 @pipeBase.timeMethod
452 def runDataRef(self, butler):
453 """
454 Make FGCM output products for use in the stack
456 Parameters
457 ----------
458 butler: `lsst.daf.persistence.Butler`
459 cycleNumber: `int`
460 Final fit cycle number, override config.
462 Returns
463 -------
464 offsets: `lsst.pipe.base.Struct`
465 A structure with array of zeropoint offsets
467 Raises
468 ------
469 RuntimeError:
470 Raised if any one of the following is true:
472 - butler cannot find "fgcmBuildStars_config" or
473 "fgcmBuildStarsTable_config".
474 - butler cannot find "fgcmFitCycle_config".
475 - "fgcmFitCycle_config" does not refer to
476 `self.config.cycleNumber`.
477 - butler cannot find "fgcmAtmosphereParameters" and
478 `self.config.doAtmosphereOutput` is `True`.
479 - butler cannot find "fgcmStandardStars" and
480 `self.config.doReferenceCalibration` is `True` or
481 `self.config.doRefcatOutput` is `True`.
482 - butler cannot find "fgcmZeropoints" and
483 `self.config.doZeropointOutput` is `True`.
484 """
485 if self.config.doReferenceCalibration:
486 # We need the ref obj loader to get the flux field
487 self.makeSubtask("refObjLoader", butler=butler)
489 # Check to make sure that the fgcmBuildStars config exists, to retrieve
490 # the visit and ccd dataset tags
491 if not butler.datasetExists('fgcmBuildStarsTable_config') and \
492 not butler.datasetExists('fgcmBuildStars_config'):
493 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
494 "which is prereq for fgcmOutputProducts")
496 if butler.datasetExists('fgcmBuildStarsTable_config'):
497 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
498 else:
499 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
500 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
501 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
502 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
504 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
505 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
506 "in fgcmBuildStarsTask.")
508 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
509 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
511 # And make sure that the atmosphere was output properly
512 if (self.config.doAtmosphereOutput
513 and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
514 raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
516 if not butler.datasetExists('fgcmStandardStars',
517 fgcmcycle=self.config.cycleNumber):
518 raise RuntimeError("Standard stars are missing for cycle %d." %
519 (self.config.cycleNumber))
521 if (self.config.doZeropointOutput
522 and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
523 raise RuntimeError("Zeropoints are missing for cycle %d." %
524 (self.config.cycleNumber))
526 dataRefDict = {}
527 # This is the _actual_ camera
528 dataRefDict['camera'] = butler.get('camera')
529 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
530 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
531 dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars',
532 fgcmcycle=self.config.cycleNumber)
534 if self.config.doZeropointOutput:
535 dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints',
536 fgcmcycle=self.config.cycleNumber)
537 if self.config.doAtmosphereOutput:
538 dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters',
539 fgcmcycle=self.config.cycleNumber)
541 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False)
543 if struct.photoCalibs is not None:
544 self.log.info("Outputting photoCalib files.")
546 for visit, detector, physicalFilter, photoCalib in struct.photoCalibs:
547 butler.put(photoCalib, 'fgcm_photoCalib',
548 dataId={visitDataRefName: visit,
549 ccdDataRefName: detector,
550 'filter': physicalFilter})
552 self.log.info("Done outputting photoCalib files.")
554 if struct.atmospheres is not None:
555 self.log.info("Outputting atmosphere transmission files.")
556 for visit, atm in struct.atmospheres:
557 butler.put(atm, "transmission_atmosphere_fgcm",
558 dataId={visitDataRefName: visit})
559 self.log.info("Done outputting atmosphere transmissions.")
561 return pipeBase.Struct(offsets=struct.offsets)
563 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
564 """Run the output products task.
566 Parameters
567 ----------
568 dataRefDict : `dict`
569 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
570 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
571 dataRef dictionary with keys:
573 ``"camera"``
574 Camera object (`lsst.afw.cameraGeom.Camera`)
575 ``"fgcmLookUpTable"``
576 dataRef for the FGCM look-up table.
577 ``"fgcmVisitCatalog"``
578 dataRef for visit summary catalog.
579 ``"fgcmStandardStars"``
580 dataRef for the output standard star catalog.
581 ``"fgcmZeropoints"``
582 dataRef for the zeropoint data catalog.
583 ``"fgcmAtmosphereParameters"``
584 dataRef for the atmosphere parameter catalog.
585 ``"fgcmBuildStarsTableConfig"``
586 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
587 physicalFilterMap : `dict`
588 Dictionary of mappings from physical filter to FGCM band.
589 returnCatalogs : `bool`, optional
590 Return photoCalibs as per-visit exposure catalogs.
591 butler : `lsst.daf.persistence.Butler`, optional
592 Gen2 butler used for reference star outputs
594 Returns
595 -------
596 retStruct : `lsst.pipe.base.Struct`
597 Output structure with keys:
599 offsets : `np.ndarray`
600 Final reference offsets, per band.
601 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
602 Generator that returns (visit, transmissionCurve) tuples.
603 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
604 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
605 (returned if returnCatalogs is False).
606 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
607 Generator that returns (visit, exposureCatalog) tuples.
608 (returned if returnCatalogs is True).
609 """
610 stdCat = dataRefDict['fgcmStandardStars'].get()
611 md = stdCat.getMetadata()
612 bands = md.getArray('BANDS')
614 if self.config.doReferenceCalibration:
615 offsets = self._computeReferenceOffsets(stdCat, bands)
616 else:
617 offsets = np.zeros(len(bands))
619 # This is Gen2 only, and requires the butler.
620 if self.config.doRefcatOutput and butler is not None:
621 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
623 del stdCat
625 if self.config.doZeropointOutput:
626 zptCat = dataRefDict['fgcmZeropoints'].get()
627 visitCat = dataRefDict['fgcmVisitCatalog'].get()
629 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
630 physicalFilterMap, returnCatalogs=returnCatalogs)
631 else:
632 pcgen = None
634 if self.config.doAtmosphereOutput:
635 atmCat = dataRefDict['fgcmAtmosphereParameters'].get()
636 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
637 else:
638 atmgen = None
640 retStruct = pipeBase.Struct(offsets=offsets,
641 atmospheres=atmgen)
642 if returnCatalogs:
643 retStruct.photoCalibCatalogs = pcgen
644 else:
645 retStruct.photoCalibs = pcgen
647 return retStruct
649 def generateTractOutputProducts(self, dataRefDict, tract,
650 visitCat, zptCat, atmCat, stdCat,
651 fgcmBuildStarsConfig,
652 returnCatalogs=True,
653 butler=None):
654 """
655 Generate the output products for a given tract, as specified in the config.
657 This method is here to have an alternate entry-point for
658 FgcmCalibrateTract.
660 Parameters
661 ----------
662 dataRefDict : `dict`
663 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
664 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
665 dataRef dictionary with keys:
667 ``"camera"``
668 Camera object (`lsst.afw.cameraGeom.Camera`)
669 ``"fgcmLookUpTable"``
670 dataRef for the FGCM look-up table.
671 tract : `int`
672 Tract number
673 visitCat : `lsst.afw.table.BaseCatalog`
674 FGCM visitCat from `FgcmBuildStarsTask`
675 zptCat : `lsst.afw.table.BaseCatalog`
676 FGCM zeropoint catalog from `FgcmFitCycleTask`
677 atmCat : `lsst.afw.table.BaseCatalog`
678 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
679 stdCat : `lsst.afw.table.SimpleCatalog`
680 FGCM standard star catalog from `FgcmFitCycleTask`
681 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
682 Configuration object from `FgcmBuildStarsTask`
683 returnCatalogs : `bool`, optional
684 Return photoCalibs as per-visit exposure catalogs.
685 butler: `lsst.daf.persistence.Butler`, optional
686 Gen2 butler used for reference star outputs
688 Returns
689 -------
690 retStruct : `lsst.pipe.base.Struct`
691 Output structure with keys:
693 offsets : `np.ndarray`
694 Final reference offsets, per band.
695 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
696 Generator that returns (visit, transmissionCurve) tuples.
697 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
698 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
699 (returned if returnCatalogs is False).
700 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
701 Generator that returns (visit, exposureCatalog) tuples.
702 (returned if returnCatalogs is True).
703 """
704 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
706 md = stdCat.getMetadata()
707 bands = md.getArray('BANDS')
709 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
710 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
711 "in fgcmBuildStarsTask.")
713 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
714 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
716 if self.config.doReferenceCalibration:
717 offsets = self._computeReferenceOffsets(stdCat, bands)
718 else:
719 offsets = np.zeros(len(bands))
721 if self.config.doRefcatOutput and butler is not None:
722 # Create a special config that has the tract number in it
723 datasetConfig = copy.copy(self.config.datasetConfig)
724 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
725 tract)
726 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
728 if self.config.doZeropointOutput:
729 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
730 physicalFilterMap, returnCatalogs=returnCatalogs)
731 else:
732 pcgen = None
734 if self.config.doAtmosphereOutput:
735 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
736 else:
737 atmgen = None
739 retStruct = pipeBase.Struct(offsets=offsets,
740 atmospheres=atmgen)
741 if returnCatalogs:
742 retStruct.photoCalibCatalogs = pcgen
743 else:
744 retStruct.photoCalibs = pcgen
746 return retStruct
748 def _computeReferenceOffsets(self, stdCat, bands):
749 """
750 Compute offsets relative to a reference catalog.
752 This method splits the star catalog into healpix pixels
753 and computes the calibration transfer for a sample of
754 these pixels to approximate the 'absolute' calibration
755 values (on for each band) to apply to transfer the
756 absolute scale.
758 Parameters
759 ----------
760 stdCat : `lsst.afw.table.SimpleCatalog`
761 FGCM standard stars
762 bands : `list` [`str`]
763 List of band names from FGCM output
764 Returns
765 -------
766 offsets : `numpy.array` of floats
767 Per band zeropoint offsets
768 """
770 # Only use stars that are observed in all the bands that were actually used
771 # This will ensure that we use the same healpix pixels for the absolute
772 # calibration of each band.
773 minObs = stdCat['ngood'].min(axis=1)
775 goodStars = (minObs >= 1)
776 stdCat = stdCat[goodStars]
778 self.log.info("Found %d stars with at least 1 good observation in each band" %
779 (len(stdCat)))
781 # We have to make a table for each pixel with flux/fluxErr
782 # This is a temporary table generated for input to the photoCal task.
783 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
784 # have had chromatic corrections applied to get to the standard system
785 # specified by the atmosphere/instrumental parameters), nor are they
786 # in Jansky (since they don't have a proper absolute calibration: the overall
787 # zeropoint is estimated from the telescope size, etc.)
788 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
789 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
790 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
791 doc="instrumental flux (counts)")
792 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
793 doc="instrumental flux error (counts)")
794 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
795 type='Flag',
796 doc="bad flag")
798 # Split up the stars
799 # Note that there is an assumption here that the ra/dec coords stored
800 # on-disk are in radians, and therefore that starObs['coord_ra'] /
801 # starObs['coord_dec'] return radians when used as an array of numpy float64s.
802 theta = np.pi/2. - stdCat['coord_dec']
803 phi = stdCat['coord_ra']
805 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
806 h, rev = esutil.stat.histogram(ipring, rev=True)
808 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
810 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
811 (gdpix.size,
812 self.config.referencePixelizationNside,
813 self.config.referencePixelizationMinStars))
815 if gdpix.size < self.config.referencePixelizationNPixels:
816 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
817 (gdpix.size, self.config.referencePixelizationNPixels))
818 else:
819 # Sample out the pixels we want to use
820 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
822 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
823 ('nstar', 'i4', len(bands)),
824 ('nmatch', 'i4', len(bands)),
825 ('zp', 'f4', len(bands)),
826 ('zpErr', 'f4', len(bands))])
827 results['hpix'] = ipring[rev[rev[gdpix]]]
829 # We need a boolean index to deal with catalogs...
830 selected = np.zeros(len(stdCat), dtype=bool)
832 refFluxFields = [None]*len(bands)
834 for p, pix in enumerate(gdpix):
835 i1a = rev[rev[pix]: rev[pix + 1]]
837 # the stdCat afwTable can only be indexed with boolean arrays,
838 # and not numpy index arrays (see DM-16497). This little trick
839 # converts the index array into a boolean array
840 selected[:] = False
841 selected[i1a] = True
843 for b, band in enumerate(bands):
845 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
846 selected, refFluxFields)
847 results['nstar'][p, b] = len(i1a)
848 results['nmatch'][p, b] = len(struct.arrays.refMag)
849 results['zp'][p, b] = struct.zp
850 results['zpErr'][p, b] = struct.sigma
852 # And compute the summary statistics
853 offsets = np.zeros(len(bands))
855 for b, band in enumerate(bands):
856 # make configurable
857 ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)
858 offsets[b] = np.median(results['zp'][ok, b])
859 # use median absolute deviation to estimate Normal sigma
860 # see https://en.wikipedia.org/wiki/Median_absolute_deviation
861 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b]))
862 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" %
863 (band, offsets[b], madSigma))
865 return offsets
867 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
868 b, band, stdCat, selected, refFluxFields):
869 """
870 Compute the zeropoint offset between the fgcm stdCat and the reference
871 stars for one pixel in one band
873 Parameters
874 ----------
875 sourceMapper: `lsst.afw.table.SchemaMapper`
876 Mapper to go from stdCat to calibratable catalog
877 badStarKey: `lsst.afw.table.Key`
878 Key for the field with bad stars
879 b: `int`
880 Index of the band in the star catalog
881 band: `str`
882 Name of band for reference catalog
883 stdCat: `lsst.afw.table.SimpleCatalog`
884 FGCM standard stars
885 selected: `numpy.array(dtype=bool)`
886 Boolean array of which stars are in the pixel
887 refFluxFields: `list`
888 List of names of flux fields for reference catalog
889 """
891 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
892 sourceCat.reserve(selected.sum())
893 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
894 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5))
895 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b]
896 * sourceCat['instFlux'])
897 # Make sure we only use stars that have valid measurements
898 # (This is perhaps redundant with requirements above that the
899 # stars be observed in all bands, but it can't hurt)
900 badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0)
901 for rec in sourceCat[badStar]:
902 rec.set(badStarKey, True)
904 exposure = afwImage.ExposureF()
905 exposure.setFilterLabel(afwImage.FilterLabel(band=band))
907 if refFluxFields[b] is None:
908 # Need to find the flux field in the reference catalog
909 # to work around limitations of DirectMatch in PhotoCal
910 ctr = stdCat[0].getCoord()
911 rad = 0.05*lsst.geom.degrees
912 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
913 refFluxFields[b] = refDataTest.fluxField
915 # Make a copy of the config so that we can modify it
916 calConfig = copy.copy(self.config.photoCal.value)
917 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
918 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'
919 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
920 config=calConfig,
921 schema=sourceCat.getSchema())
923 struct = calTask.run(exposure, sourceCat)
925 return struct
927 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
928 """
929 Output standard stars in indexed reference catalog format.
930 This is not currently supported in Gen3.
932 Parameters
933 ----------
934 butler : `lsst.daf.persistence.Butler`
935 stdCat : `lsst.afw.table.SimpleCatalog`
936 FGCM standard star catalog from fgcmFitCycleTask
937 offsets : `numpy.array` of floats
938 Per band zeropoint offsets
939 bands : `list` [`str`]
940 List of band names from FGCM output
941 datasetConfig : `lsst.meas.algorithms.DatasetConfig`
942 Config for reference dataset
943 """
945 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
947 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
948 self.config.datasetConfig.indexer.active)
950 # We determine the conversion from the native units (typically radians) to
951 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
952 # numpy arrays rather than Angles, which would we approximately 600x slower.
953 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
954 # (as Angles) for input
955 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
956 indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv,
957 stdCat['coord_dec']*conv))
959 formattedCat = self._formatCatalog(stdCat, offsets, bands)
961 # Write the master schema
962 dataId = indexer.makeDataId('master_schema',
963 datasetConfig.ref_dataset_name)
964 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
965 addRefCatMetadata(masterCat)
966 butler.put(masterCat, 'ref_cat', dataId=dataId)
968 # Break up the pixels using a histogram
969 h, rev = esutil.stat.histogram(indices, rev=True)
970 gd, = np.where(h > 0)
971 selected = np.zeros(len(formattedCat), dtype=bool)
972 for i in gd:
973 i1a = rev[rev[i]: rev[i + 1]]
975 # the formattedCat afwTable can only be indexed with boolean arrays,
976 # and not numpy index arrays (see DM-16497). This little trick
977 # converts the index array into a boolean array
978 selected[:] = False
979 selected[i1a] = True
981 # Write the individual pixel
982 dataId = indexer.makeDataId(indices[i1a[0]],
983 datasetConfig.ref_dataset_name)
984 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
986 # And save the dataset configuration
987 dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name)
988 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
990 self.log.info("Done outputting standard stars.")
992 def _formatCatalog(self, fgcmStarCat, offsets, bands):
993 """
994 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
996 Parameters
997 ----------
998 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
999 SimpleCatalog as output by fgcmcal
1000 offsets : `list` with len(self.bands) entries
1001 Zeropoint offsets to apply
1002 bands : `list` [`str`]
1003 List of band names from FGCM output
1005 Returns
1006 -------
1007 formattedCat: `lsst.afw.table.SimpleCatalog`
1008 SimpleCatalog suitable for using as a reference catalog
1009 """
1011 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1012 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1013 addCentroid=False,
1014 addIsResolved=True,
1015 coordErrDim=0)
1016 sourceMapper.addMinimalSchema(minSchema)
1017 for band in bands:
1018 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
1019 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
1020 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
1022 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1023 formattedCat.reserve(len(fgcmStarCat))
1024 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1026 # Note that we don't have to set `resolved` because the default is False
1028 for b, band in enumerate(bands):
1029 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1030 # We want fluxes in nJy from calibrated AB magnitudes
1031 # (after applying offset). Updated after RFC-549 and RFC-575.
1032 flux = (mag*units.ABmag).to_value(units.nJy)
1033 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
1035 formattedCat['%s_flux' % (band)][:] = flux
1036 formattedCat['%s_fluxErr' % (band)][:] = fluxErr
1037 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
1038 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
1039 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
1041 addRefCatMetadata(formattedCat)
1043 return formattedCat
1045 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1046 physicalFilterMap, returnCatalogs=True,
1047 tract=None):
1048 """Output the zeropoints in fgcm_photoCalib format.
1050 Parameters
1051 ----------
1052 camera : `lsst.afw.cameraGeom.Camera`
1053 Camera from the butler.
1054 zptCat : `lsst.afw.table.BaseCatalog`
1055 FGCM zeropoint catalog from `FgcmFitCycleTask`.
1056 visitCat : `lsst.afw.table.BaseCatalog`
1057 FGCM visitCat from `FgcmBuildStarsTask`.
1058 offsets : `numpy.array`
1059 Float array of absolute calibration offsets, one for each filter.
1060 bands : `list` [`str`]
1061 List of band names from FGCM output.
1062 physicalFilterMap : `dict`
1063 Dictionary of mappings from physical filter to FGCM band.
1064 returnCatalogs : `bool`, optional
1065 Return photoCalibs as per-visit exposure catalogs.
1066 tract: `int`, optional
1067 Tract number to output. Default is None (global calibration)
1069 Returns
1070 -------
1071 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1072 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1073 (returned if returnCatalogs is False).
1074 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1075 Generator that returns (visit, exposureCatalog) tuples.
1076 (returned if returnCatalogs is True).
1077 """
1078 # Select visit/ccds where we have a calibration
1079 # This includes ccds where we were able to interpolate from neighboring
1080 # ccds.
1081 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
1082 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
1083 & (zptCat['fgcmZptVar'] > 0.0))
1085 # Log warnings for any visit which has no valid zeropoints
1086 badVisits = np.unique(zptCat['visit'][~selected])
1087 goodVisits = np.unique(zptCat['visit'][selected])
1088 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1089 for allBadVisit in allBadVisits:
1090 self.log.warn(f'No suitable photoCalib for visit {allBadVisit}')
1092 # Get a mapping from filtername to the offsets
1093 offsetMapping = {}
1094 for f in physicalFilterMap:
1095 # Not every filter in the map will necesarily have a band.
1096 if physicalFilterMap[f] in bands:
1097 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1099 # Get a mapping from "ccd" to the ccd index used for the scaling
1100 ccdMapping = {}
1101 for ccdIndex, detector in enumerate(camera):
1102 ccdMapping[detector.getId()] = ccdIndex
1104 # And a mapping to get the flat-field scaling values
1105 scalingMapping = {}
1106 for rec in visitCat:
1107 scalingMapping[rec['visit']] = rec['scaling']
1109 if self.config.doComposeWcsJacobian:
1110 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
1112 # The zptCat is sorted by visit, which is useful
1113 lastVisit = -1
1114 zptVisitCatalog = None
1116 metadata = dafBase.PropertyList()
1117 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
1118 metadata.add("COMMENT", "Only detectors with data have entries.")
1120 for rec in zptCat[selected]:
1121 # Retrieve overall scaling
1122 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
1124 # The postCalibrationOffset describe any zeropoint offsets
1125 # to apply after the fgcm calibration. The first part comes
1126 # from the reference catalog match (used in testing). The
1127 # second part comes from the mean chromatic correction
1128 # (if configured).
1129 postCalibrationOffset = offsetMapping[rec['filtername']]
1130 if self.config.doApplyMeanChromaticCorrection:
1131 postCalibrationOffset += rec['fgcmDeltaChrom']
1133 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
1134 rec['fgcmfZptChebXyMax'])
1135 # Convert from FGCM AB to nJy
1136 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
1137 rec['fgcmfZptChebXyMax'],
1138 offset=postCalibrationOffset,
1139 scaling=scaling)
1141 if self.config.doComposeWcsJacobian:
1143 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']],
1144 fgcmSuperStarField,
1145 fgcmZptField])
1146 else:
1147 # The photoCalib is just the product of the fgcmSuperStarField and the
1148 # fgcmZptField
1149 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1151 # The "mean" calibration will be set to the center of the ccd for reference
1152 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1153 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
1154 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1155 calibrationErr=calibErr,
1156 calibration=fgcmField,
1157 isConstant=False)
1159 if not returnCatalogs:
1160 # Return individual photoCalibs
1161 yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib)
1162 else:
1163 # Return full per-visit exposure catalogs
1164 if rec['visit'] != lastVisit:
1165 # This is a new visit. If the last visit was not -1, yield
1166 # the ExposureCatalog
1167 if lastVisit > -1:
1168 # ensure that the detectors are in sorted order, for fast lookups
1169 zptVisitCatalog.sort()
1170 yield (int(lastVisit), zptVisitCatalog)
1171 else:
1172 # We need to create a new schema
1173 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1174 zptExpCatSchema.addField('visit', type='I', doc='Visit number')
1176 # And start a new one
1177 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1178 zptVisitCatalog.setMetadata(metadata)
1180 lastVisit = int(rec['visit'])
1182 catRecord = zptVisitCatalog.addNew()
1183 catRecord['id'] = int(rec['detector'])
1184 catRecord['visit'] = rec['visit']
1185 catRecord.setPhotoCalib(photoCalib)
1187 # Final output of last exposure catalog
1188 if returnCatalogs:
1189 # ensure that the detectors are in sorted order, for fast lookups
1190 zptVisitCatalog.sort()
1191 yield (int(lastVisit), zptVisitCatalog)
1193 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1194 """
1195 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1196 and scaling.
1198 Parameters
1199 ----------
1200 coefficients: `numpy.array`
1201 Flattened array of chebyshev coefficients
1202 xyMax: `list` of length 2
1203 Maximum x and y of the chebyshev bounding box
1204 offset: `float`, optional
1205 Absolute calibration offset. Default is 0.0
1206 scaling: `float`, optional
1207 Flat scaling value from fgcmBuildStars. Default is 1.0
1209 Returns
1210 -------
1211 boundedField: `lsst.afw.math.ChebyshevBoundedField`
1212 """
1214 orderPlus1 = int(np.sqrt(coefficients.size))
1215 pars = np.zeros((orderPlus1, orderPlus1))
1217 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1218 lsst.geom.Point2I(*xyMax))
1220 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1221 * (10.**(offset/-2.5))*scaling)
1223 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1225 return boundedField
1227 def _outputAtmospheres(self, dataRefDict, atmCat):
1228 """
1229 Output the atmospheres.
1231 Parameters
1232 ----------
1233 dataRefDict : `dict`
1234 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1235 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1236 dataRef dictionary with keys:
1238 ``"fgcmLookUpTable"``
1239 dataRef for the FGCM look-up table.
1240 atmCat : `lsst.afw.table.BaseCatalog`
1241 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1243 Returns
1244 -------
1245 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1246 Generator that returns (visit, transmissionCurve) tuples.
1247 """
1248 # First, we need to grab the look-up table and key info
1249 lutCat = dataRefDict['fgcmLookUpTable'].get()
1251 atmosphereTableName = lutCat[0]['tablename']
1252 elevation = lutCat[0]['elevation']
1253 atmLambda = lutCat[0]['atmLambda']
1254 lutCat = None
1256 # Make the atmosphere table if possible
1257 try:
1258 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1259 atmTable.loadTable()
1260 except IOError:
1261 atmTable = None
1263 if atmTable is None:
1264 # Try to use MODTRAN instead
1265 try:
1266 modGen = fgcm.ModtranGenerator(elevation)
1267 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1268 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1269 except (ValueError, IOError) as e:
1270 raise RuntimeError("FGCM look-up-table generated with modtran, "
1271 "but modtran not configured to run.") from e
1273 zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
1275 for i, visit in enumerate(atmCat['visit']):
1276 if atmTable is not None:
1277 # Interpolate the atmosphere table
1278 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
1279 pwv=atmCat[i]['pwv'],
1280 o3=atmCat[i]['o3'],
1281 tau=atmCat[i]['tau'],
1282 alpha=atmCat[i]['alpha'],
1283 zenith=zenith[i],
1284 ctranslamstd=[atmCat[i]['cTrans'],
1285 atmCat[i]['lamStd']])
1286 else:
1287 # Run modtran
1288 modAtm = modGen(pmb=atmCat[i]['pmb'],
1289 pwv=atmCat[i]['pwv'],
1290 o3=atmCat[i]['o3'],
1291 tau=atmCat[i]['tau'],
1292 alpha=atmCat[i]['alpha'],
1293 zenith=zenith[i],
1294 lambdaRange=lambdaRange,
1295 lambdaStep=lambdaStep,
1296 ctranslamstd=[atmCat[i]['cTrans'],
1297 atmCat[i]['lamStd']])
1298 atmVals = modAtm['COMBINED']
1300 # Now need to create something to persist...
1301 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1302 wavelengths=atmLambda,
1303 throughputAtMin=atmVals[0],
1304 throughputAtMax=atmVals[-1])
1306 yield (int(visit), curve)