Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Make the final fgcmcal output products.
25This task takes the final output from fgcmFitCycle and produces the following
26outputs for use in the DM stack: the FGCM standard stars in a reference
27catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29task can transfer the 'absolute' calibration from a reference catalog
30to put the fgcm standard stars in units of Jansky. This is accomplished
31by matching stars in a sample of healpix pixels, and applying the median
32offset per band.
33"""
34import sys
35import traceback
36import copy
38import numpy as np
39import healpy as hp
40import esutil
41from astropy import units
43import lsst.pex.config as pexConfig
44import lsst.pipe.base as pipeBase
45from lsst.pipe.base import connectionTypes
46from lsst.afw.image import TransmissionCurve
47from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
48from lsst.meas.algorithms import ReferenceObjectLoader
49from lsst.pipe.tasks.photoCal import PhotoCalTask
50import lsst.geom
51import lsst.afw.image as afwImage
52import lsst.afw.math as afwMath
53import lsst.afw.table as afwTable
54from lsst.meas.algorithms import IndexerRegistry
55from lsst.meas.algorithms import DatasetConfig
56from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
58from .utilities import computeApproxPixelAreaFields
59from .utilities import lookupStaticCalibrations
61import fgcm
63__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
66class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
67 dimensions=("instrument",),
68 defaultTemplates={"cycleNumber": "0"}):
69 camera = connectionTypes.PrerequisiteInput(
70 doc="Camera instrument",
71 name="camera",
72 storageClass="Camera",
73 dimensions=("instrument",),
74 lookupFunction=lookupStaticCalibrations,
75 isCalibration=True,
76 )
78 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
79 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
80 "chromatic corrections."),
81 name="fgcmLookUpTable",
82 storageClass="Catalog",
83 dimensions=("instrument",),
84 deferLoad=True,
85 )
87 fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
88 doc="Catalog of visit information for fgcm",
89 name="fgcmVisitCatalog",
90 storageClass="Catalog",
91 dimensions=("instrument",),
92 deferLoad=True,
93 )
95 fgcmStandardStars = connectionTypes.PrerequisiteInput(
96 doc="Catalog of standard star data from fgcm fit",
97 name="fgcmStandardStars{cycleNumber}",
98 storageClass="SimpleCatalog",
99 dimensions=("instrument",),
100 deferLoad=True,
101 )
103 fgcmZeropoints = connectionTypes.PrerequisiteInput(
104 doc="Catalog of zeropoints from fgcm fit",
105 name="fgcmZeropoints{cycleNumber}",
106 storageClass="Catalog",
107 dimensions=("instrument",),
108 deferLoad=True,
109 )
111 fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
112 doc="Catalog of atmosphere parameters from fgcm fit",
113 name="fgcmAtmosphereParameters{cycleNumber}",
114 storageClass="Catalog",
115 dimensions=("instrument",),
116 deferLoad=True,
117 )
119 refCat = connectionTypes.PrerequisiteInput(
120 doc="Reference catalog to use for photometric calibration",
121 name="cal_ref_cat",
122 storageClass="SimpleCatalog",
123 dimensions=("skypix",),
124 deferLoad=True,
125 multiple=True,
126 )
128 fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
129 doc="Config used to build FGCM input stars",
130 name="fgcmBuildStarsTable_config",
131 storageClass="Config",
132 )
134 fgcmPhotoCalib = connectionTypes.Output(
135 doc="Per-visit photoCalib exposure catalogs produced from fgcm calibration",
136 name="fgcmPhotoCalibCatalog",
137 storageClass="ExposureCatalog",
138 dimensions=("instrument", "visit",),
139 multiple=True,
140 )
142 fgcmTransmissionAtmosphere = connectionTypes.Output(
143 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
144 name="transmission_atmosphere_fgcm",
145 storageClass="TransmissionCurve",
146 dimensions=("instrument",
147 "visit",),
148 multiple=True,
149 )
151 fgcmOffsets = connectionTypes.Output(
152 doc="Per-band offsets computed from doReferenceCalibration",
153 name="fgcmReferenceCalibrationOffsets",
154 storageClass="Catalog",
155 dimensions=("instrument",),
156 multiple=False,
157 )
159 def __init__(self, *, config=None):
160 super().__init__(config=config)
162 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
163 raise ValueError("cycleNumber must be of integer format")
164 if config.connections.refCat != config.refObjLoader.ref_dataset_name:
165 raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name")
167 if config.doRefcatOutput:
168 raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
170 if not config.doReferenceCalibration:
171 self.prerequisiteInputs.remove("refCat")
172 if not config.doAtmosphereOutput:
173 self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
174 if not config.doZeropointOutput:
175 self.prerequisiteInputs.remove("fgcmZeropoints")
176 if not config.doReferenceCalibration:
177 self.outputs.remove("fgcmOffsets")
180class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
181 pipelineConnections=FgcmOutputProductsConnections):
182 """Config for FgcmOutputProductsTask"""
184 cycleNumber = pexConfig.Field(
185 doc="Final fit cycle from FGCM fit",
186 dtype=int,
187 default=None,
188 )
190 # The following fields refer to calibrating from a reference
191 # catalog, but in the future this might need to be expanded
192 doReferenceCalibration = pexConfig.Field(
193 doc=("Transfer 'absolute' calibration from reference catalog? "
194 "This afterburner step is unnecessary if reference stars "
195 "were used in the full fit in FgcmFitCycleTask."),
196 dtype=bool,
197 default=False,
198 )
199 doRefcatOutput = pexConfig.Field(
200 doc="Output standard stars in reference catalog format",
201 dtype=bool,
202 default=True,
203 )
204 doAtmosphereOutput = pexConfig.Field(
205 doc="Output atmospheres in transmission_atmosphere_fgcm format",
206 dtype=bool,
207 default=True,
208 )
209 doZeropointOutput = pexConfig.Field(
210 doc="Output zeropoints in fgcm_photoCalib format",
211 dtype=bool,
212 default=True,
213 )
214 doComposeWcsJacobian = pexConfig.Field(
215 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
216 dtype=bool,
217 default=True,
218 )
219 doApplyMeanChromaticCorrection = pexConfig.Field(
220 doc="Apply the mean chromatic correction to the zeropoints?",
221 dtype=bool,
222 default=True,
223 )
224 refObjLoader = pexConfig.ConfigurableField(
225 target=LoadIndexedReferenceObjectsTask,
226 doc="reference object loader for 'absolute' photometric calibration",
227 )
228 photoCal = pexConfig.ConfigurableField(
229 target=PhotoCalTask,
230 doc="task to perform 'absolute' calibration",
231 )
232 referencePixelizationNside = pexConfig.Field(
233 doc="Healpix nside to pixelize catalog to compare to reference catalog",
234 dtype=int,
235 default=64,
236 )
237 referencePixelizationMinStars = pexConfig.Field(
238 doc=("Minimum number of stars per healpix pixel to select for comparison"
239 "to the specified reference catalog"),
240 dtype=int,
241 default=200,
242 )
243 referenceMinMatch = pexConfig.Field(
244 doc="Minimum number of stars matched to reference catalog to be used in statistics",
245 dtype=int,
246 default=50,
247 )
248 referencePixelizationNPixels = pexConfig.Field(
249 doc=("Number of healpix pixels to sample to do comparison. "
250 "Doing too many will take a long time and not yield any more "
251 "precise results because the final number is the median offset "
252 "(per band) from the set of pixels."),
253 dtype=int,
254 default=100,
255 )
256 datasetConfig = pexConfig.ConfigField(
257 dtype=DatasetConfig,
258 doc="Configuration for writing/reading ingested catalog",
259 )
261 def setDefaults(self):
262 pexConfig.Config.setDefaults(self)
264 # In order to transfer the "absolute" calibration from a reference
265 # catalog to the relatively calibrated FGCM standard stars (one number
266 # per band), we use the PhotoCalTask to match stars in a sample of healpix
267 # pixels. These basic settings ensure that only well-measured, good stars
268 # from the source and reference catalogs are used for the matching.
270 # applyColorTerms needs to be False if doReferenceCalibration is False,
271 # as is the new default after DM-16702
272 self.photoCal.applyColorTerms = False
273 self.photoCal.fluxField = 'instFlux'
274 self.photoCal.magErrFloor = 0.003
275 self.photoCal.match.referenceSelection.doSignalToNoise = True
276 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
277 self.photoCal.match.sourceSelection.doSignalToNoise = True
278 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
279 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
280 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
281 self.photoCal.match.sourceSelection.doFlags = True
282 self.photoCal.match.sourceSelection.flags.good = []
283 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
284 self.photoCal.match.sourceSelection.doUnresolved = False
285 self.datasetConfig.ref_dataset_name = 'fgcm_stars'
286 self.datasetConfig.format_version = 1
288 def validate(self):
289 super().validate()
291 # Force the connections to conform with cycleNumber
292 self.connections.cycleNumber = str(self.cycleNumber)
295class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
296 """Subclass of TaskRunner for fgcmOutputProductsTask
298 fgcmOutputProductsTask.run() takes one argument, the butler, and
299 does not run on any data in the repository.
300 This runner does not use any parallelization.
301 """
303 @staticmethod
304 def getTargetList(parsedCmd):
305 """
306 Return a list with one element, the butler.
307 """
308 return [parsedCmd.butler]
310 def __call__(self, butler):
311 """
312 Parameters
313 ----------
314 butler: `lsst.daf.persistence.Butler`
316 Returns
317 -------
318 exitStatus: `list` with `pipeBase.Struct`
319 exitStatus (0: success; 1: failure)
320 if self.doReturnResults also
321 results (`np.array` with absolute zeropoint offsets)
322 """
323 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
325 exitStatus = 0
326 if self.doRaise:
327 results = task.runDataRef(butler)
328 else:
329 try:
330 results = task.runDataRef(butler)
331 except Exception as e:
332 exitStatus = 1
333 task.log.fatal("Failed: %s" % e)
334 if not isinstance(e, pipeBase.TaskError):
335 traceback.print_exc(file=sys.stderr)
337 task.writeMetadata(butler)
339 if self.doReturnResults:
340 # The results here are the zeropoint offsets for each band
341 return [pipeBase.Struct(exitStatus=exitStatus,
342 results=results)]
343 else:
344 return [pipeBase.Struct(exitStatus=exitStatus)]
346 def run(self, parsedCmd):
347 """
348 Run the task, with no multiprocessing
350 Parameters
351 ----------
352 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
353 """
355 resultList = []
357 if self.precall(parsedCmd):
358 targetList = self.getTargetList(parsedCmd)
359 # make sure that we only get 1
360 resultList = self(targetList[0])
362 return resultList
365class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
366 """
367 Output products from FGCM global calibration.
368 """
370 ConfigClass = FgcmOutputProductsConfig
371 RunnerClass = FgcmOutputProductsRunner
372 _DefaultName = "fgcmOutputProducts"
374 def __init__(self, butler=None, **kwargs):
375 super().__init__(**kwargs)
377 # no saving of metadata for now
378 def _getMetadataName(self):
379 return None
381 def runQuantum(self, butlerQC, inputRefs, outputRefs):
382 dataRefDict = {}
383 dataRefDict['camera'] = butlerQC.get(inputRefs.camera)
384 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
385 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
386 dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
388 if self.config.doZeropointOutput:
389 dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
390 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
391 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
393 if self.config.doAtmosphereOutput:
394 dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
395 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
396 atmRef in outputRefs.fgcmTransmissionAtmosphere}
398 if self.config.doReferenceCalibration:
399 refConfig = self.config.refObjLoader
400 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
401 for ref in inputRefs.refCat],
402 refCats=butlerQC.get(inputRefs.refCat),
403 config=refConfig,
404 log=self.log)
405 else:
406 self.refObjLoader = None
408 dataRefDict['fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
410 fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
411 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
413 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
414 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
415 "in fgcmBuildStarsTask.")
416 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
417 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
419 struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=True)
421 # Output the photoCalib exposure catalogs
422 if struct.photoCalibCatalogs is not None:
423 self.log.info("Outputting photoCalib catalogs.")
424 for visit, expCatalog in struct.photoCalibCatalogs:
425 butlerQC.put(expCatalog, photoCalibRefDict[visit])
426 self.log.info("Done outputting photoCalib catalogs.")
428 # Output the atmospheres
429 if struct.atmospheres is not None:
430 self.log.info("Outputting atmosphere transmission files.")
431 for visit, atm in struct.atmospheres:
432 butlerQC.put(atm, atmRefDict[visit])
433 self.log.info("Done outputting atmosphere files.")
435 if self.config.doReferenceCalibration:
436 # Turn offset into simple catalog for persistence if necessary
437 schema = afwTable.Schema()
438 schema.addField('offset', type=np.float64,
439 doc="Post-process calibration offset (mag)")
440 offsetCat = afwTable.BaseCatalog(schema)
441 offsetCat.resize(len(struct.offsets))
442 offsetCat['offset'][:] = struct.offsets
444 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
446 return
448 @pipeBase.timeMethod
449 def runDataRef(self, butler):
450 """
451 Make FGCM output products for use in the stack
453 Parameters
454 ----------
455 butler: `lsst.daf.persistence.Butler`
456 cycleNumber: `int`
457 Final fit cycle number, override config.
459 Returns
460 -------
461 offsets: `lsst.pipe.base.Struct`
462 A structure with array of zeropoint offsets
464 Raises
465 ------
466 RuntimeError:
467 Raised if any one of the following is true:
469 - butler cannot find "fgcmBuildStars_config" or
470 "fgcmBuildStarsTable_config".
471 - butler cannot find "fgcmFitCycle_config".
472 - "fgcmFitCycle_config" does not refer to
473 `self.config.cycleNumber`.
474 - butler cannot find "fgcmAtmosphereParameters" and
475 `self.config.doAtmosphereOutput` is `True`.
476 - butler cannot find "fgcmStandardStars" and
477 `self.config.doReferenceCalibration` is `True` or
478 `self.config.doRefcatOutput` is `True`.
479 - butler cannot find "fgcmZeropoints" and
480 `self.config.doZeropointOutput` is `True`.
481 """
482 if self.config.doReferenceCalibration:
483 # We need the ref obj loader to get the flux field
484 self.makeSubtask("refObjLoader", butler=butler)
486 # Check to make sure that the fgcmBuildStars config exists, to retrieve
487 # the visit and ccd dataset tags
488 if not butler.datasetExists('fgcmBuildStarsTable_config') and \
489 not butler.datasetExists('fgcmBuildStars_config'):
490 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
491 "which is prereq for fgcmOutputProducts")
493 if butler.datasetExists('fgcmBuildStarsTable_config'):
494 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
495 else:
496 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
497 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
498 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
499 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
501 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
502 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
503 "in fgcmBuildStarsTask.")
505 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
506 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
508 # And make sure that the atmosphere was output properly
509 if (self.config.doAtmosphereOutput
510 and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
511 raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
513 if not butler.datasetExists('fgcmStandardStars',
514 fgcmcycle=self.config.cycleNumber):
515 raise RuntimeError("Standard stars are missing for cycle %d." %
516 (self.config.cycleNumber))
518 if (self.config.doZeropointOutput
519 and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
520 raise RuntimeError("Zeropoints are missing for cycle %d." %
521 (self.config.cycleNumber))
523 dataRefDict = {}
524 # This is the _actual_ camera
525 dataRefDict['camera'] = butler.get('camera')
526 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
527 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
528 dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars',
529 fgcmcycle=self.config.cycleNumber)
531 if self.config.doZeropointOutput:
532 dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints',
533 fgcmcycle=self.config.cycleNumber)
534 if self.config.doAtmosphereOutput:
535 dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters',
536 fgcmcycle=self.config.cycleNumber)
538 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False)
540 if struct.photoCalibs is not None:
541 self.log.info("Outputting photoCalib files.")
543 for visit, detector, physicalFilter, photoCalib in struct.photoCalibs:
544 butler.put(photoCalib, 'fgcm_photoCalib',
545 dataId={visitDataRefName: visit,
546 ccdDataRefName: detector,
547 'filter': physicalFilter})
549 self.log.info("Done outputting photoCalib files.")
551 if struct.atmospheres is not None:
552 self.log.info("Outputting atmosphere transmission files.")
553 for visit, atm in struct.atmospheres:
554 butler.put(atm, "transmission_atmosphere_fgcm",
555 dataId={visitDataRefName: visit})
556 self.log.info("Done outputting atmosphere transmissions.")
558 return pipeBase.Struct(offsets=struct.offsets)
560 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
561 """Run the output products task.
563 Parameters
564 ----------
565 dataRefDict : `dict`
566 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
567 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
568 dataRef dictionary with keys:
570 ``"camera"``
571 Camera object (`lsst.afw.cameraGeom.Camera`)
572 ``"fgcmLookUpTable"``
573 dataRef for the FGCM look-up table.
574 ``"fgcmVisitCatalog"``
575 dataRef for visit summary catalog.
576 ``"fgcmStandardStars"``
577 dataRef for the output standard star catalog.
578 ``"fgcmZeropoints"``
579 dataRef for the zeropoint data catalog.
580 ``"fgcmAtmosphereParameters"``
581 dataRef for the atmosphere parameter catalog.
582 ``"fgcmBuildStarsTableConfig"``
583 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
584 physicalFilterMap : `dict`
585 Dictionary of mappings from physical filter to FGCM band.
586 returnCatalogs : `bool`, optional
587 Return photoCalibs as per-visit exposure catalogs.
588 butler : `lsst.daf.persistence.Butler`, optional
589 Gen2 butler used for reference star outputs
591 Returns
592 -------
593 retStruct : `lsst.pipe.base.Struct`
594 Output structure with keys:
596 offsets : `np.ndarray`
597 Final reference offsets, per band.
598 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
599 Generator that returns (visit, transmissionCurve) tuples.
600 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
601 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
602 (returned if returnCatalogs is False).
603 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
604 Generator that returns (visit, exposureCatalog) tuples.
605 (returned if returnCatalogs is True).
606 """
607 stdCat = dataRefDict['fgcmStandardStars'].get()
608 md = stdCat.getMetadata()
609 bands = md.getArray('BANDS')
611 if self.config.doReferenceCalibration:
612 offsets = self._computeReferenceOffsets(stdCat, bands)
613 else:
614 offsets = np.zeros(len(bands))
616 # This is Gen2 only, and requires the butler.
617 if self.config.doRefcatOutput and butler is not None:
618 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
620 del stdCat
622 if self.config.doZeropointOutput:
623 zptCat = dataRefDict['fgcmZeropoints'].get()
624 visitCat = dataRefDict['fgcmVisitCatalog'].get()
626 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
627 physicalFilterMap, returnCatalogs=returnCatalogs)
628 else:
629 pcgen = None
631 if self.config.doAtmosphereOutput:
632 atmCat = dataRefDict['fgcmAtmosphereParameters'].get()
633 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
634 else:
635 atmgen = None
637 retStruct = pipeBase.Struct(offsets=offsets,
638 atmospheres=atmgen)
639 if returnCatalogs:
640 retStruct.photoCalibCatalogs = pcgen
641 else:
642 retStruct.photoCalibs = pcgen
644 return retStruct
646 def generateTractOutputProducts(self, dataRefDict, tract,
647 visitCat, zptCat, atmCat, stdCat,
648 fgcmBuildStarsConfig,
649 returnCatalogs=True,
650 butler=None):
651 """
652 Generate the output products for a given tract, as specified in the config.
654 This method is here to have an alternate entry-point for
655 FgcmCalibrateTract.
657 Parameters
658 ----------
659 dataRefDict : `dict`
660 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
661 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
662 dataRef dictionary with keys:
664 ``"camera"``
665 Camera object (`lsst.afw.cameraGeom.Camera`)
666 ``"fgcmLookUpTable"``
667 dataRef for the FGCM look-up table.
668 tract : `int`
669 Tract number
670 visitCat : `lsst.afw.table.BaseCatalog`
671 FGCM visitCat from `FgcmBuildStarsTask`
672 zptCat : `lsst.afw.table.BaseCatalog`
673 FGCM zeropoint catalog from `FgcmFitCycleTask`
674 atmCat : `lsst.afw.table.BaseCatalog`
675 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
676 stdCat : `lsst.afw.table.SimpleCatalog`
677 FGCM standard star catalog from `FgcmFitCycleTask`
678 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
679 Configuration object from `FgcmBuildStarsTask`
680 returnCatalogs : `bool`, optional
681 Return photoCalibs as per-visit exposure catalogs.
682 butler: `lsst.daf.persistence.Butler`, optional
683 Gen2 butler used for reference star outputs
685 Returns
686 -------
687 retStruct : `lsst.pipe.base.Struct`
688 Output structure with keys:
690 offsets : `np.ndarray`
691 Final reference offsets, per band.
692 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
693 Generator that returns (visit, transmissionCurve) tuples.
694 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
695 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
696 (returned if returnCatalogs is False).
697 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
698 Generator that returns (visit, exposureCatalog) tuples.
699 (returned if returnCatalogs is True).
700 """
701 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
703 md = stdCat.getMetadata()
704 bands = md.getArray('BANDS')
706 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
707 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
708 "in fgcmBuildStarsTask.")
710 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
711 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
713 if self.config.doReferenceCalibration:
714 offsets = self._computeReferenceOffsets(stdCat, bands)
715 else:
716 offsets = np.zeros(len(bands))
718 if self.config.doRefcatOutput and butler is not None:
719 # Create a special config that has the tract number in it
720 datasetConfig = copy.copy(self.config.datasetConfig)
721 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
722 tract)
723 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
725 if self.config.doZeropointOutput:
726 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
727 physicalFilterMap, returnCatalogs=returnCatalogs)
728 else:
729 pcgen = None
731 if self.config.doAtmosphereOutput:
732 atmgen = self._outputAtmospheres(dataRefDict, atmCat)
733 else:
734 atmgen = None
736 retStruct = pipeBase.Struct(offsets=offsets,
737 atmospheres=atmgen)
738 if returnCatalogs:
739 retStruct.photoCalibCatalogs = pcgen
740 else:
741 retStruct.photoCalibs = pcgen
743 return retStruct
745 def _computeReferenceOffsets(self, stdCat, bands):
746 """
747 Compute offsets relative to a reference catalog.
749 This method splits the star catalog into healpix pixels
750 and computes the calibration transfer for a sample of
751 these pixels to approximate the 'absolute' calibration
752 values (on for each band) to apply to transfer the
753 absolute scale.
755 Parameters
756 ----------
757 stdCat : `lsst.afw.table.SimpleCatalog`
758 FGCM standard stars
759 bands : `list` [`str`]
760 List of band names from FGCM output
761 Returns
762 -------
763 offsets : `numpy.array` of floats
764 Per band zeropoint offsets
765 """
767 # Only use stars that are observed in all the bands that were actually used
768 # This will ensure that we use the same healpix pixels for the absolute
769 # calibration of each band.
770 minObs = stdCat['ngood'].min(axis=1)
772 goodStars = (minObs >= 1)
773 stdCat = stdCat[goodStars]
775 self.log.info("Found %d stars with at least 1 good observation in each band" %
776 (len(stdCat)))
778 # We have to make a table for each pixel with flux/fluxErr
779 # This is a temporary table generated for input to the photoCal task.
780 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
781 # have had chromatic corrections applied to get to the standard system
782 # specified by the atmosphere/instrumental parameters), nor are they
783 # in Jansky (since they don't have a proper absolute calibration: the overall
784 # zeropoint is estimated from the telescope size, etc.)
785 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
786 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
787 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
788 doc="instrumental flux (counts)")
789 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
790 doc="instrumental flux error (counts)")
791 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
792 type='Flag',
793 doc="bad flag")
795 # Split up the stars
796 # Note that there is an assumption here that the ra/dec coords stored
797 # on-disk are in radians, and therefore that starObs['coord_ra'] /
798 # starObs['coord_dec'] return radians when used as an array of numpy float64s.
799 theta = np.pi/2. - stdCat['coord_dec']
800 phi = stdCat['coord_ra']
802 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
803 h, rev = esutil.stat.histogram(ipring, rev=True)
805 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
807 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
808 (gdpix.size,
809 self.config.referencePixelizationNside,
810 self.config.referencePixelizationMinStars))
812 if gdpix.size < self.config.referencePixelizationNPixels:
813 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
814 (gdpix.size, self.config.referencePixelizationNPixels))
815 else:
816 # Sample out the pixels we want to use
817 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
819 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
820 ('nstar', 'i4', len(bands)),
821 ('nmatch', 'i4', len(bands)),
822 ('zp', 'f4', len(bands)),
823 ('zpErr', 'f4', len(bands))])
824 results['hpix'] = ipring[rev[rev[gdpix]]]
826 # We need a boolean index to deal with catalogs...
827 selected = np.zeros(len(stdCat), dtype=np.bool)
829 refFluxFields = [None]*len(bands)
831 for p, pix in enumerate(gdpix):
832 i1a = rev[rev[pix]: rev[pix + 1]]
834 # the stdCat afwTable can only be indexed with boolean arrays,
835 # and not numpy index arrays (see DM-16497). This little trick
836 # converts the index array into a boolean array
837 selected[:] = False
838 selected[i1a] = True
840 for b, band in enumerate(bands):
842 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
843 selected, refFluxFields)
844 results['nstar'][p, b] = len(i1a)
845 results['nmatch'][p, b] = len(struct.arrays.refMag)
846 results['zp'][p, b] = struct.zp
847 results['zpErr'][p, b] = struct.sigma
849 # And compute the summary statistics
850 offsets = np.zeros(len(bands))
852 for b, band in enumerate(bands):
853 # make configurable
854 ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)
855 offsets[b] = np.median(results['zp'][ok, b])
856 # use median absolute deviation to estimate Normal sigma
857 # see https://en.wikipedia.org/wiki/Median_absolute_deviation
858 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b]))
859 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" %
860 (band, offsets[b], madSigma))
862 return offsets
864 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
865 b, band, stdCat, selected, refFluxFields):
866 """
867 Compute the zeropoint offset between the fgcm stdCat and the reference
868 stars for one pixel in one band
870 Parameters
871 ----------
872 sourceMapper: `lsst.afw.table.SchemaMapper`
873 Mapper to go from stdCat to calibratable catalog
874 badStarKey: `lsst.afw.table.Key`
875 Key for the field with bad stars
876 b: `int`
877 Index of the band in the star catalog
878 band: `str`
879 Name of band for reference catalog
880 stdCat: `lsst.afw.table.SimpleCatalog`
881 FGCM standard stars
882 selected: `numpy.array(dtype=np.bool)`
883 Boolean array of which stars are in the pixel
884 refFluxFields: `list`
885 List of names of flux fields for reference catalog
886 """
888 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
889 sourceCat.reserve(selected.sum())
890 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
891 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5))
892 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b]
893 * sourceCat['instFlux'])
894 # Make sure we only use stars that have valid measurements
895 # (This is perhaps redundant with requirements above that the
896 # stars be observed in all bands, but it can't hurt)
897 badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0)
898 for rec in sourceCat[badStar]:
899 rec.set(badStarKey, True)
901 exposure = afwImage.ExposureF()
902 exposure.setFilter(afwImage.Filter(band))
904 if refFluxFields[b] is None:
905 # Need to find the flux field in the reference catalog
906 # to work around limitations of DirectMatch in PhotoCal
907 ctr = stdCat[0].getCoord()
908 rad = 0.05*lsst.geom.degrees
909 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
910 refFluxFields[b] = refDataTest.fluxField
912 # Make a copy of the config so that we can modify it
913 calConfig = copy.copy(self.config.photoCal.value)
914 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
915 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'
916 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
917 config=calConfig,
918 schema=sourceCat.getSchema())
920 struct = calTask.run(exposure, sourceCat)
922 return struct
924 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
925 """
926 Output standard stars in indexed reference catalog format.
927 This is not currently supported in Gen3.
929 Parameters
930 ----------
931 butler : `lsst.daf.persistence.Butler`
932 stdCat : `lsst.afw.table.SimpleCatalog`
933 FGCM standard star catalog from fgcmFitCycleTask
934 offsets : `numpy.array` of floats
935 Per band zeropoint offsets
936 bands : `list` [`str`]
937 List of band names from FGCM output
938 datasetConfig : `lsst.meas.algorithms.DatasetConfig`
939 Config for reference dataset
940 """
942 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
944 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
945 self.config.datasetConfig.indexer.active)
947 # We determine the conversion from the native units (typically radians) to
948 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
949 # numpy arrays rather than Angles, which would we approximately 600x slower.
950 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
951 # (as Angles) for input
952 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
953 indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv,
954 stdCat['coord_dec']*conv))
956 formattedCat = self._formatCatalog(stdCat, offsets, bands)
958 # Write the master schema
959 dataId = indexer.makeDataId('master_schema',
960 datasetConfig.ref_dataset_name)
961 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
962 addRefCatMetadata(masterCat)
963 butler.put(masterCat, 'ref_cat', dataId=dataId)
965 # Break up the pixels using a histogram
966 h, rev = esutil.stat.histogram(indices, rev=True)
967 gd, = np.where(h > 0)
968 selected = np.zeros(len(formattedCat), dtype=np.bool)
969 for i in gd:
970 i1a = rev[rev[i]: rev[i + 1]]
972 # the formattedCat afwTable can only be indexed with boolean arrays,
973 # and not numpy index arrays (see DM-16497). This little trick
974 # converts the index array into a boolean array
975 selected[:] = False
976 selected[i1a] = True
978 # Write the individual pixel
979 dataId = indexer.makeDataId(indices[i1a[0]],
980 datasetConfig.ref_dataset_name)
981 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
983 # And save the dataset configuration
984 dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name)
985 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
987 self.log.info("Done outputting standard stars.")
989 def _formatCatalog(self, fgcmStarCat, offsets, bands):
990 """
991 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
993 Parameters
994 ----------
995 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
996 SimpleCatalog as output by fgcmcal
997 offsets : `list` with len(self.bands) entries
998 Zeropoint offsets to apply
999 bands : `list` [`str`]
1000 List of band names from FGCM output
1002 Returns
1003 -------
1004 formattedCat: `lsst.afw.table.SimpleCatalog`
1005 SimpleCatalog suitable for using as a reference catalog
1006 """
1008 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1009 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1010 addCentroid=False,
1011 addIsResolved=True,
1012 coordErrDim=0)
1013 sourceMapper.addMinimalSchema(minSchema)
1014 for band in bands:
1015 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
1016 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
1017 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
1019 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1020 formattedCat.reserve(len(fgcmStarCat))
1021 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1023 # Note that we don't have to set `resolved` because the default is False
1025 for b, band in enumerate(bands):
1026 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1027 # We want fluxes in nJy from calibrated AB magnitudes
1028 # (after applying offset). Updated after RFC-549 and RFC-575.
1029 flux = (mag*units.ABmag).to_value(units.nJy)
1030 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
1032 formattedCat['%s_flux' % (band)][:] = flux
1033 formattedCat['%s_fluxErr' % (band)][:] = fluxErr
1034 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
1035 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
1036 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
1038 addRefCatMetadata(formattedCat)
1040 return formattedCat
1042 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1043 physicalFilterMap, returnCatalogs=True,
1044 tract=None):
1045 """Output the zeropoints in fgcm_photoCalib format.
1047 Parameters
1048 ----------
1049 camera : `lsst.afw.cameraGeom.Camera`
1050 Camera from the butler.
1051 zptCat : `lsst.afw.table.BaseCatalog`
1052 FGCM zeropoint catalog from `FgcmFitCycleTask`.
1053 visitCat : `lsst.afw.table.BaseCatalog`
1054 FGCM visitCat from `FgcmBuildStarsTask`.
1055 offsets : `numpy.array`
1056 Float array of absolute calibration offsets, one for each filter.
1057 bands : `list` [`str`]
1058 List of band names from FGCM output.
1059 physicalFilterMap : `dict`
1060 Dictionary of mappings from physical filter to FGCM band.
1061 returnCatalogs : `bool`, optional
1062 Return photoCalibs as per-visit exposure catalogs.
1063 tract: `int`, optional
1064 Tract number to output. Default is None (global calibration)
1066 Returns
1067 -------
1068 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1069 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1070 (returned if returnCatalogs is False).
1071 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1072 Generator that returns (visit, exposureCatalog) tuples.
1073 (returned if returnCatalogs is True).
1074 """
1075 # Select visit/ccds where we have a calibration
1076 # This includes ccds where we were able to interpolate from neighboring
1077 # ccds.
1078 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
1079 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
1080 & (zptCat['fgcmZptVar'] > 0.0))
1082 # Log warnings for any visit which has no valid zeropoints
1083 badVisits = np.unique(zptCat['visit'][~selected])
1084 goodVisits = np.unique(zptCat['visit'][selected])
1085 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1086 for allBadVisit in allBadVisits:
1087 self.log.warn(f'No suitable photoCalib for visit {allBadVisit}')
1089 # Get a mapping from filtername to the offsets
1090 offsetMapping = {}
1091 for f in physicalFilterMap:
1092 # Not every filter in the map will necesarily have a band.
1093 if physicalFilterMap[f] in bands:
1094 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1096 # Get a mapping from "ccd" to the ccd index used for the scaling
1097 ccdMapping = {}
1098 for ccdIndex, detector in enumerate(camera):
1099 ccdMapping[detector.getId()] = ccdIndex
1101 # And a mapping to get the flat-field scaling values
1102 scalingMapping = {}
1103 for rec in visitCat:
1104 scalingMapping[rec['visit']] = rec['scaling']
1106 if self.config.doComposeWcsJacobian:
1107 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
1109 # The zptCat is sorted by visit, which is useful
1110 lastVisit = -1
1111 zptCounter = 0
1112 zptVisitCatalog = None
1113 for rec in zptCat[selected]:
1115 # Retrieve overall scaling
1116 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
1118 # The postCalibrationOffset describe any zeropoint offsets
1119 # to apply after the fgcm calibration. The first part comes
1120 # from the reference catalog match (used in testing). The
1121 # second part comes from the mean chromatic correction
1122 # (if configured).
1123 postCalibrationOffset = offsetMapping[rec['filtername']]
1124 if self.config.doApplyMeanChromaticCorrection:
1125 postCalibrationOffset += rec['fgcmDeltaChrom']
1127 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
1128 rec['fgcmfZptChebXyMax'])
1129 # Convert from FGCM AB to nJy
1130 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
1131 rec['fgcmfZptChebXyMax'],
1132 offset=postCalibrationOffset,
1133 scaling=scaling)
1135 if self.config.doComposeWcsJacobian:
1137 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']],
1138 fgcmSuperStarField,
1139 fgcmZptField])
1140 else:
1141 # The photoCalib is just the product of the fgcmSuperStarField and the
1142 # fgcmZptField
1143 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1145 # The "mean" calibration will be set to the center of the ccd for reference
1146 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1147 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
1148 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1149 calibrationErr=calibErr,
1150 calibration=fgcmField,
1151 isConstant=False)
1153 if not returnCatalogs:
1154 # Return individual photoCalibs
1155 yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib)
1156 else:
1157 # Return full per-visit exposure catalogs
1158 if rec['visit'] != lastVisit:
1159 # This is a new visit. If the last visit was not -1, yield
1160 # the ExposureCatalog
1161 if lastVisit > -1:
1162 yield (int(lastVisit), zptVisitCatalog)
1163 else:
1164 # We need to create a new schema
1165 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1166 zptExpCatSchema.addField('visit', type='I', doc='Visit number')
1167 zptExpCatSchema.addField('detector_id', type='I', doc='Detector number')
1169 # And start a new one
1170 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1171 zptVisitCatalog.resize(len(camera))
1172 zptVisitCatalog['visit'] = rec['visit']
1173 # By default all records will not resolve to a valid detector.
1174 zptVisitCatalog['detector_id'] = -1
1176 # Reset the counter
1177 zptCounter = 0
1179 lastVisit = int(rec['visit'])
1181 zptVisitCatalog[zptCounter].setPhotoCalib(photoCalib)
1182 zptVisitCatalog[zptCounter]['detector_id'] = int(rec['detector'])
1184 zptCounter += 1
1186 # Final output of last exposure catalog
1187 if returnCatalogs:
1188 yield (int(lastVisit), zptVisitCatalog)
1190 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1191 """
1192 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1193 and scaling.
1195 Parameters
1196 ----------
1197 coefficients: `numpy.array`
1198 Flattened array of chebyshev coefficients
1199 xyMax: `list` of length 2
1200 Maximum x and y of the chebyshev bounding box
1201 offset: `float`, optional
1202 Absolute calibration offset. Default is 0.0
1203 scaling: `float`, optional
1204 Flat scaling value from fgcmBuildStars. Default is 1.0
1206 Returns
1207 -------
1208 boundedField: `lsst.afw.math.ChebyshevBoundedField`
1209 """
1211 orderPlus1 = int(np.sqrt(coefficients.size))
1212 pars = np.zeros((orderPlus1, orderPlus1))
1214 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1215 lsst.geom.Point2I(*xyMax))
1217 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1218 * (10.**(offset/-2.5))*scaling)
1220 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1222 return boundedField
1224 def _outputAtmospheres(self, dataRefDict, atmCat):
1225 """
1226 Output the atmospheres.
1228 Parameters
1229 ----------
1230 dataRefDict : `dict`
1231 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1232 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1233 dataRef dictionary with keys:
1235 ``"fgcmLookUpTable"``
1236 dataRef for the FGCM look-up table.
1237 atmCat : `lsst.afw.table.BaseCatalog`
1238 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1240 Returns
1241 -------
1242 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1243 Generator that returns (visit, transmissionCurve) tuples.
1244 """
1245 # First, we need to grab the look-up table and key info
1246 lutCat = dataRefDict['fgcmLookUpTable'].get()
1248 atmosphereTableName = lutCat[0]['tablename']
1249 elevation = lutCat[0]['elevation']
1250 atmLambda = lutCat[0]['atmLambda']
1251 lutCat = None
1253 # Make the atmosphere table if possible
1254 try:
1255 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1256 atmTable.loadTable()
1257 except IOError:
1258 atmTable = None
1260 if atmTable is None:
1261 # Try to use MODTRAN instead
1262 try:
1263 modGen = fgcm.ModtranGenerator(elevation)
1264 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1265 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1266 except (ValueError, IOError) as e:
1267 raise RuntimeError("FGCM look-up-table generated with modtran, "
1268 "but modtran not configured to run.") from e
1270 zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
1272 for i, visit in enumerate(atmCat['visit']):
1273 if atmTable is not None:
1274 # Interpolate the atmosphere table
1275 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
1276 pwv=atmCat[i]['pwv'],
1277 o3=atmCat[i]['o3'],
1278 tau=atmCat[i]['tau'],
1279 alpha=atmCat[i]['alpha'],
1280 zenith=zenith[i],
1281 ctranslamstd=[atmCat[i]['cTrans'],
1282 atmCat[i]['lamStd']])
1283 else:
1284 # Run modtran
1285 modAtm = modGen(pmb=atmCat[i]['pmb'],
1286 pwv=atmCat[i]['pwv'],
1287 o3=atmCat[i]['o3'],
1288 tau=atmCat[i]['tau'],
1289 alpha=atmCat[i]['alpha'],
1290 zenith=zenith[i],
1291 lambdaRange=lambdaRange,
1292 lambdaStep=lambdaStep,
1293 ctranslamstd=[atmCat[i]['cTrans'],
1294 atmCat[i]['lamStd']])
1295 atmVals = modAtm['COMBINED']
1297 # Now need to create something to persist...
1298 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1299 wavelengths=atmLambda,
1300 throughputAtMin=atmVals[0],
1301 throughputAtMax=atmVals[-1])
1303 yield (int(visit), curve)