Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py: 15%
314 statements
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-25 12:04 +0000
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-25 12:04 +0000
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Make the final fgcmcal output products.
25This task takes the final output from fgcmFitCycle and produces the following
26outputs for use in the DM stack: the FGCM standard stars in a reference
27catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29task can transfer the 'absolute' calibration from a reference catalog
30to put the fgcm standard stars in units of Jansky. This is accomplished
31by matching stars in a sample of healpix pixels, and applying the median
32offset per band.
33"""
34import copy
36import numpy as np
37import hpgeom as hpg
38import esutil
39from astropy import units
41import lsst.daf.base as dafBase
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44from lsst.pipe.base import connectionTypes
45from lsst.afw.image import TransmissionCurve
46from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
47from lsst.pipe.tasks.photoCal import PhotoCalTask
48import lsst.geom
49import lsst.afw.image as afwImage
50import lsst.afw.math as afwMath
51import lsst.afw.table as afwTable
53from .utilities import computeApproxPixelAreaFields
54from .utilities import lookupStaticCalibrations
55from .utilities import FGCM_ILLEGAL_VALUE
57import fgcm
59__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask']
62class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
63 dimensions=("instrument",),
64 defaultTemplates={"cycleNumber": "0"}):
65 camera = connectionTypes.PrerequisiteInput(
66 doc="Camera instrument",
67 name="camera",
68 storageClass="Camera",
69 dimensions=("instrument",),
70 lookupFunction=lookupStaticCalibrations,
71 isCalibration=True,
72 )
74 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
75 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
76 "chromatic corrections."),
77 name="fgcmLookUpTable",
78 storageClass="Catalog",
79 dimensions=("instrument",),
80 deferLoad=True,
81 )
83 fgcmVisitCatalog = connectionTypes.Input(
84 doc="Catalog of visit information for fgcm",
85 name="fgcmVisitCatalog",
86 storageClass="Catalog",
87 dimensions=("instrument",),
88 deferLoad=True,
89 )
91 fgcmStandardStars = connectionTypes.Input(
92 doc="Catalog of standard star data from fgcm fit",
93 name="fgcmStandardStars{cycleNumber}",
94 storageClass="SimpleCatalog",
95 dimensions=("instrument",),
96 deferLoad=True,
97 )
99 fgcmZeropoints = connectionTypes.Input(
100 doc="Catalog of zeropoints from fgcm fit",
101 name="fgcmZeropoints{cycleNumber}",
102 storageClass="Catalog",
103 dimensions=("instrument",),
104 deferLoad=True,
105 )
107 fgcmAtmosphereParameters = connectionTypes.Input(
108 doc="Catalog of atmosphere parameters from fgcm fit",
109 name="fgcmAtmosphereParameters{cycleNumber}",
110 storageClass="Catalog",
111 dimensions=("instrument",),
112 deferLoad=True,
113 )
115 refCat = connectionTypes.PrerequisiteInput(
116 doc="Reference catalog to use for photometric calibration",
117 name="cal_ref_cat",
118 storageClass="SimpleCatalog",
119 dimensions=("skypix",),
120 deferLoad=True,
121 multiple=True,
122 )
124 fgcmPhotoCalib = connectionTypes.Output(
125 doc=("Per-visit photometric calibrations derived from fgcm calibration. "
126 "These catalogs use detector id for the id and are sorted for "
127 "fast lookups of a detector."),
128 name="fgcmPhotoCalibCatalog",
129 storageClass="ExposureCatalog",
130 dimensions=("instrument", "visit",),
131 multiple=True,
132 )
134 fgcmTransmissionAtmosphere = connectionTypes.Output(
135 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
136 name="transmission_atmosphere_fgcm",
137 storageClass="TransmissionCurve",
138 dimensions=("instrument",
139 "visit",),
140 multiple=True,
141 )
143 fgcmOffsets = connectionTypes.Output(
144 doc="Per-band offsets computed from doReferenceCalibration",
145 name="fgcmReferenceCalibrationOffsets",
146 storageClass="Catalog",
147 dimensions=("instrument",),
148 multiple=False,
149 )
151 def __init__(self, *, config=None):
152 super().__init__(config=config)
154 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
155 raise ValueError("cycleNumber must be of integer format")
157 if not config.doReferenceCalibration:
158 self.prerequisiteInputs.remove("refCat")
159 if not config.doAtmosphereOutput:
160 self.inputs.remove("fgcmAtmosphereParameters")
161 if not config.doZeropointOutput:
162 self.inputs.remove("fgcmZeropoints")
163 if not config.doReferenceCalibration:
164 self.outputs.remove("fgcmOffsets")
166 def getSpatialBoundsConnections(self):
167 return ("fgcmPhotoCalib",)
170class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
171 pipelineConnections=FgcmOutputProductsConnections):
172 """Config for FgcmOutputProductsTask"""
174 physicalFilterMap = pexConfig.DictField(
175 doc="Mapping from 'physicalFilter' to band.",
176 keytype=str,
177 itemtype=str,
178 default={},
179 )
180 # The following fields refer to calibrating from a reference
181 # catalog, but in the future this might need to be expanded
182 doReferenceCalibration = pexConfig.Field(
183 doc=("Transfer 'absolute' calibration from reference catalog? "
184 "This afterburner step is unnecessary if reference stars "
185 "were used in the full fit in FgcmFitCycleTask."),
186 dtype=bool,
187 default=False,
188 )
189 doAtmosphereOutput = pexConfig.Field(
190 doc="Output atmospheres in transmission_atmosphere_fgcm format",
191 dtype=bool,
192 default=True,
193 )
194 doZeropointOutput = pexConfig.Field(
195 doc="Output zeropoints in fgcm_photoCalib format",
196 dtype=bool,
197 default=True,
198 )
199 doComposeWcsJacobian = pexConfig.Field(
200 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
201 dtype=bool,
202 default=True,
203 )
204 doApplyMeanChromaticCorrection = pexConfig.Field(
205 doc="Apply the mean chromatic correction to the zeropoints?",
206 dtype=bool,
207 default=True,
208 )
209 photoCal = pexConfig.ConfigurableField(
210 target=PhotoCalTask,
211 doc="task to perform 'absolute' calibration",
212 )
213 referencePixelizationNside = pexConfig.Field(
214 doc="Healpix nside to pixelize catalog to compare to reference catalog",
215 dtype=int,
216 default=64,
217 )
218 referencePixelizationMinStars = pexConfig.Field(
219 doc=("Minimum number of stars per healpix pixel to select for comparison"
220 "to the specified reference catalog"),
221 dtype=int,
222 default=200,
223 )
224 referenceMinMatch = pexConfig.Field(
225 doc="Minimum number of stars matched to reference catalog to be used in statistics",
226 dtype=int,
227 default=50,
228 )
229 referencePixelizationNPixels = pexConfig.Field(
230 doc=("Number of healpix pixels to sample to do comparison. "
231 "Doing too many will take a long time and not yield any more "
232 "precise results because the final number is the median offset "
233 "(per band) from the set of pixels."),
234 dtype=int,
235 default=100,
236 )
238 def setDefaults(self):
239 pexConfig.Config.setDefaults(self)
241 # In order to transfer the "absolute" calibration from a reference
242 # catalog to the relatively calibrated FGCM standard stars (one number
243 # per band), we use the PhotoCalTask to match stars in a sample of healpix
244 # pixels. These basic settings ensure that only well-measured, good stars
245 # from the source and reference catalogs are used for the matching.
247 # applyColorTerms needs to be False if doReferenceCalibration is False,
248 # as is the new default after DM-16702
249 self.photoCal.applyColorTerms = False
250 self.photoCal.fluxField = 'instFlux'
251 self.photoCal.magErrFloor = 0.003
252 self.photoCal.match.referenceSelection.doSignalToNoise = True
253 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
254 self.photoCal.match.sourceSelection.doSignalToNoise = True
255 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
256 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
257 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
258 self.photoCal.match.sourceSelection.doFlags = True
259 self.photoCal.match.sourceSelection.flags.good = []
260 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
261 self.photoCal.match.sourceSelection.doUnresolved = False
262 self.photoCal.match.sourceSelection.doRequirePrimary = False
265class FgcmOutputProductsTask(pipeBase.PipelineTask):
266 """
267 Output products from FGCM global calibration.
268 """
270 ConfigClass = FgcmOutputProductsConfig
271 _DefaultName = "fgcmOutputProducts"
273 def __init__(self, **kwargs):
274 super().__init__(**kwargs)
276 def runQuantum(self, butlerQC, inputRefs, outputRefs):
277 handleDict = {}
278 handleDict['camera'] = butlerQC.get(inputRefs.camera)
279 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
280 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
281 handleDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
283 if self.config.doZeropointOutput:
284 handleDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
285 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
286 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
288 if self.config.doAtmosphereOutput:
289 handleDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
290 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
291 atmRef in outputRefs.fgcmTransmissionAtmosphere}
293 if self.config.doReferenceCalibration:
294 refConfig = LoadReferenceObjectsConfig()
295 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
296 for ref in inputRefs.refCat],
297 refCats=butlerQC.get(inputRefs.refCat),
298 name=self.config.connections.refCat,
299 log=self.log,
300 config=refConfig)
301 else:
302 self.refObjLoader = None
304 struct = self.run(handleDict, self.config.physicalFilterMap)
306 # Output the photoCalib exposure catalogs
307 if struct.photoCalibCatalogs is not None:
308 self.log.info("Outputting photoCalib catalogs.")
309 for visit, expCatalog in struct.photoCalibCatalogs:
310 butlerQC.put(expCatalog, photoCalibRefDict[visit])
311 self.log.info("Done outputting photoCalib catalogs.")
313 # Output the atmospheres
314 if struct.atmospheres is not None:
315 self.log.info("Outputting atmosphere transmission files.")
316 for visit, atm in struct.atmospheres:
317 butlerQC.put(atm, atmRefDict[visit])
318 self.log.info("Done outputting atmosphere files.")
320 if self.config.doReferenceCalibration:
321 # Turn offset into simple catalog for persistence if necessary
322 schema = afwTable.Schema()
323 schema.addField('offset', type=np.float64,
324 doc="Post-process calibration offset (mag)")
325 offsetCat = afwTable.BaseCatalog(schema)
326 offsetCat.resize(len(struct.offsets))
327 offsetCat['offset'][:] = struct.offsets
329 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
331 return
333 def run(self, handleDict, physicalFilterMap):
334 """Run the output products task.
336 Parameters
337 ----------
338 handleDict : `dict`
339 All handles are `lsst.daf.butler.DeferredDatasetHandle`
340 handle dictionary with keys:
342 ``"camera"``
343 Camera object (`lsst.afw.cameraGeom.Camera`)
344 ``"fgcmLookUpTable"``
345 handle for the FGCM look-up table.
346 ``"fgcmVisitCatalog"``
347 handle for visit summary catalog.
348 ``"fgcmStandardStars"``
349 handle for the output standard star catalog.
350 ``"fgcmZeropoints"``
351 handle for the zeropoint data catalog.
352 ``"fgcmAtmosphereParameters"``
353 handle for the atmosphere parameter catalog.
354 ``"fgcmBuildStarsTableConfig"``
355 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
356 physicalFilterMap : `dict`
357 Dictionary of mappings from physical filter to FGCM band.
359 Returns
360 -------
361 retStruct : `lsst.pipe.base.Struct`
362 Output structure with keys:
364 offsets : `np.ndarray`
365 Final reference offsets, per band.
366 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
367 Generator that returns (visit, transmissionCurve) tuples.
368 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
369 Generator that returns (visit, exposureCatalog) tuples.
370 """
371 stdCat = handleDict['fgcmStandardStars'].get()
372 md = stdCat.getMetadata()
373 bands = md.getArray('BANDS')
375 if self.config.doReferenceCalibration:
376 lutCat = handleDict['fgcmLookUpTable'].get()
377 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands)
378 else:
379 offsets = np.zeros(len(bands))
381 del stdCat
383 if self.config.doZeropointOutput:
384 zptCat = handleDict['fgcmZeropoints'].get()
385 visitCat = handleDict['fgcmVisitCatalog'].get()
387 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands,
388 physicalFilterMap)
389 else:
390 pcgen = None
392 if self.config.doAtmosphereOutput:
393 atmCat = handleDict['fgcmAtmosphereParameters'].get()
394 atmgen = self._outputAtmospheres(handleDict, atmCat)
395 else:
396 atmgen = None
398 retStruct = pipeBase.Struct(offsets=offsets,
399 atmospheres=atmgen)
400 retStruct.photoCalibCatalogs = pcgen
402 return retStruct
404 def generateTractOutputProducts(self, handleDict, tract,
405 visitCat, zptCat, atmCat, stdCat,
406 fgcmBuildStarsConfig):
407 """
408 Generate the output products for a given tract, as specified in the config.
410 This method is here to have an alternate entry-point for
411 FgcmCalibrateTract.
413 Parameters
414 ----------
415 handleDict : `dict`
416 All handles are `lsst.daf.butler.DeferredDatasetHandle`
417 handle dictionary with keys:
419 ``"camera"``
420 Camera object (`lsst.afw.cameraGeom.Camera`)
421 ``"fgcmLookUpTable"``
422 handle for the FGCM look-up table.
423 tract : `int`
424 Tract number
425 visitCat : `lsst.afw.table.BaseCatalog`
426 FGCM visitCat from `FgcmBuildStarsTask`
427 zptCat : `lsst.afw.table.BaseCatalog`
428 FGCM zeropoint catalog from `FgcmFitCycleTask`
429 atmCat : `lsst.afw.table.BaseCatalog`
430 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
431 stdCat : `lsst.afw.table.SimpleCatalog`
432 FGCM standard star catalog from `FgcmFitCycleTask`
433 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
434 Configuration object from `FgcmBuildStarsTask`
436 Returns
437 -------
438 retStruct : `lsst.pipe.base.Struct`
439 Output structure with keys:
441 offsets : `np.ndarray`
442 Final reference offsets, per band.
443 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
444 Generator that returns (visit, transmissionCurve) tuples.
445 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
446 Generator that returns (visit, exposureCatalog) tuples.
447 """
448 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
450 md = stdCat.getMetadata()
451 bands = md.getArray('BANDS')
453 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
454 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
455 "in fgcmBuildStarsTask.")
457 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
458 self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
460 if self.config.doReferenceCalibration:
461 lutCat = handleDict['fgcmLookUpTable'].get()
462 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap)
463 else:
464 offsets = np.zeros(len(bands))
466 if self.config.doZeropointOutput:
467 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands,
468 physicalFilterMap)
469 else:
470 pcgen = None
472 if self.config.doAtmosphereOutput:
473 atmgen = self._outputAtmospheres(handleDict, atmCat)
474 else:
475 atmgen = None
477 retStruct = pipeBase.Struct(offsets=offsets,
478 atmospheres=atmgen)
479 retStruct.photoCalibCatalogs = pcgen
481 return retStruct
483 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
484 """
485 Compute offsets relative to a reference catalog.
487 This method splits the star catalog into healpix pixels
488 and computes the calibration transfer for a sample of
489 these pixels to approximate the 'absolute' calibration
490 values (on for each band) to apply to transfer the
491 absolute scale.
493 Parameters
494 ----------
495 stdCat : `lsst.afw.table.SimpleCatalog`
496 FGCM standard stars
497 lutCat : `lsst.afw.table.SimpleCatalog`
498 FGCM Look-up table
499 physicalFilterMap : `dict`
500 Dictionary of mappings from physical filter to FGCM band.
501 bands : `list` [`str`]
502 List of band names from FGCM output
503 Returns
504 -------
505 offsets : `numpy.array` of floats
506 Per band zeropoint offsets
507 """
509 # Only use stars that are observed in all the bands that were actually used
510 # This will ensure that we use the same healpix pixels for the absolute
511 # calibration of each band.
512 minObs = stdCat['ngood'].min(axis=1)
514 goodStars = (minObs >= 1)
515 stdCat = stdCat[goodStars]
517 self.log.info("Found %d stars with at least 1 good observation in each band" %
518 (len(stdCat)))
520 # Associate each band with the appropriate physicalFilter and make
521 # filterLabels
522 filterLabels = []
524 lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',')
525 lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',')
526 physicalFilterMapBands = list(physicalFilterMap.values())
527 physicalFilterMapFilters = list(physicalFilterMap.keys())
528 for band in bands:
529 # Find a physical filter associated from the band by doing
530 # a reverse lookup on the physicalFilterMap dict
531 physicalFilterMapIndex = physicalFilterMapBands.index(band)
532 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
533 # Find the appropriate fgcm standard physicalFilter
534 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
535 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
536 filterLabels.append(afwImage.FilterLabel(band=band,
537 physical=stdPhysicalFilter))
539 # We have to make a table for each pixel with flux/fluxErr
540 # This is a temporary table generated for input to the photoCal task.
541 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
542 # have had chromatic corrections applied to get to the standard system
543 # specified by the atmosphere/instrumental parameters), nor are they
544 # in Jansky (since they don't have a proper absolute calibration: the overall
545 # zeropoint is estimated from the telescope size, etc.)
546 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
547 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
548 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
549 doc="instrumental flux (counts)")
550 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
551 doc="instrumental flux error (counts)")
552 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
553 type='Flag',
554 doc="bad flag")
556 # Split up the stars
557 # Note that there is an assumption here that the ra/dec coords stored
558 # on-disk are in radians, and therefore that starObs['coord_ra'] /
559 # starObs['coord_dec'] return radians when used as an array of numpy float64s.
560 ipring = hpg.angle_to_pixel(
561 self.config.referencePixelizationNside,
562 stdCat['coord_ra'],
563 stdCat['coord_dec'],
564 degrees=False,
565 )
566 h, rev = esutil.stat.histogram(ipring, rev=True)
568 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
570 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
571 (gdpix.size,
572 self.config.referencePixelizationNside,
573 self.config.referencePixelizationMinStars))
575 if gdpix.size < self.config.referencePixelizationNPixels:
576 self.log.warning("Found fewer good pixels (%d) than preferred in configuration (%d)" %
577 (gdpix.size, self.config.referencePixelizationNPixels))
578 else:
579 # Sample out the pixels we want to use
580 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
582 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
583 ('nstar', 'i4', len(bands)),
584 ('nmatch', 'i4', len(bands)),
585 ('zp', 'f4', len(bands)),
586 ('zpErr', 'f4', len(bands))])
587 results['hpix'] = ipring[rev[rev[gdpix]]]
589 # We need a boolean index to deal with catalogs...
590 selected = np.zeros(len(stdCat), dtype=bool)
592 refFluxFields = [None]*len(bands)
594 for p_index, pix in enumerate(gdpix):
595 i1a = rev[rev[pix]: rev[pix + 1]]
597 # the stdCat afwTable can only be indexed with boolean arrays,
598 # and not numpy index arrays (see DM-16497). This little trick
599 # converts the index array into a boolean array
600 selected[:] = False
601 selected[i1a] = True
603 for b_index, filterLabel in enumerate(filterLabels):
604 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index,
605 filterLabel, stdCat,
606 selected, refFluxFields)
607 results['nstar'][p_index, b_index] = len(i1a)
608 results['nmatch'][p_index, b_index] = len(struct.arrays.refMag)
609 results['zp'][p_index, b_index] = struct.zp
610 results['zpErr'][p_index, b_index] = struct.sigma
612 # And compute the summary statistics
613 offsets = np.zeros(len(bands))
615 for b_index, band in enumerate(bands):
616 # make configurable
617 ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch)
618 offsets[b_index] = np.median(results['zp'][ok, b_index])
619 # use median absolute deviation to estimate Normal sigma
620 # see https://en.wikipedia.org/wiki/Median_absolute_deviation
621 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index]))
622 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f",
623 band, offsets[b_index], madSigma)
625 return offsets
627 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
628 b_index, filterLabel, stdCat, selected, refFluxFields):
629 """
630 Compute the zeropoint offset between the fgcm stdCat and the reference
631 stars for one pixel in one band
633 Parameters
634 ----------
635 sourceMapper : `lsst.afw.table.SchemaMapper`
636 Mapper to go from stdCat to calibratable catalog
637 badStarKey : `lsst.afw.table.Key`
638 Key for the field with bad stars
639 b_index : `int`
640 Index of the band in the star catalog
641 filterLabel : `lsst.afw.image.FilterLabel`
642 filterLabel with band and physical filter
643 stdCat : `lsst.afw.table.SimpleCatalog`
644 FGCM standard stars
645 selected : `numpy.array(dtype=bool)`
646 Boolean array of which stars are in the pixel
647 refFluxFields : `list`
648 List of names of flux fields for reference catalog
649 """
651 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
652 sourceCat.reserve(selected.sum())
653 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
654 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5))
655 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index]
656 * sourceCat['instFlux'])
657 # Make sure we only use stars that have valid measurements
658 # (This is perhaps redundant with requirements above that the
659 # stars be observed in all bands, but it can't hurt)
660 badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0)
661 for rec in sourceCat[badStar]:
662 rec.set(badStarKey, True)
664 exposure = afwImage.ExposureF()
665 exposure.setFilter(filterLabel)
667 if refFluxFields[b_index] is None:
668 # Need to find the flux field in the reference catalog
669 # to work around limitations of DirectMatch in PhotoCal
670 ctr = stdCat[0].getCoord()
671 rad = 0.05*lsst.geom.degrees
672 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel)
673 refFluxFields[b_index] = refDataTest.fluxField
675 # Make a copy of the config so that we can modify it
676 calConfig = copy.copy(self.config.photoCal.value)
677 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index]
678 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err'
679 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
680 config=calConfig,
681 schema=sourceCat.getSchema())
683 struct = calTask.run(exposure, sourceCat)
685 return struct
687 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
688 physicalFilterMap, tract=None):
689 """Output the zeropoints in fgcm_photoCalib format.
691 Parameters
692 ----------
693 camera : `lsst.afw.cameraGeom.Camera`
694 Camera from the butler.
695 zptCat : `lsst.afw.table.BaseCatalog`
696 FGCM zeropoint catalog from `FgcmFitCycleTask`.
697 visitCat : `lsst.afw.table.BaseCatalog`
698 FGCM visitCat from `FgcmBuildStarsTask`.
699 offsets : `numpy.array`
700 Float array of absolute calibration offsets, one for each filter.
701 bands : `list` [`str`]
702 List of band names from FGCM output.
703 physicalFilterMap : `dict`
704 Dictionary of mappings from physical filter to FGCM band.
705 tract: `int`, optional
706 Tract number to output. Default is None (global calibration)
708 Returns
709 -------
710 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
711 Generator that returns (visit, exposureCatalog) tuples.
712 """
713 # Select visit/ccds where we have a calibration
714 # This includes ccds where we were able to interpolate from neighboring
715 # ccds.
716 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
717 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
718 & (zptCat['fgcmZptVar'] > 0.0)
719 & (zptCat['fgcmZpt'] > FGCM_ILLEGAL_VALUE))
721 # Log warnings for any visit which has no valid zeropoints
722 badVisits = np.unique(zptCat['visit'][~selected])
723 goodVisits = np.unique(zptCat['visit'][selected])
724 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
725 for allBadVisit in allBadVisits:
726 self.log.warning(f'No suitable photoCalib for visit {allBadVisit}')
728 # Get a mapping from filtername to the offsets
729 offsetMapping = {}
730 for f in physicalFilterMap:
731 # Not every filter in the map will necesarily have a band.
732 if physicalFilterMap[f] in bands:
733 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
735 # Get a mapping from "ccd" to the ccd index used for the scaling
736 ccdMapping = {}
737 for ccdIndex, detector in enumerate(camera):
738 ccdMapping[detector.getId()] = ccdIndex
740 # And a mapping to get the flat-field scaling values
741 scalingMapping = {}
742 for rec in visitCat:
743 scalingMapping[rec['visit']] = rec['scaling']
745 if self.config.doComposeWcsJacobian:
746 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
748 # The zptCat is sorted by visit, which is useful
749 lastVisit = -1
750 zptVisitCatalog = None
752 metadata = dafBase.PropertyList()
753 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
754 metadata.add("COMMENT", "Only detectors with data have entries.")
756 for rec in zptCat[selected]:
757 # Retrieve overall scaling
758 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
760 # The postCalibrationOffset describe any zeropoint offsets
761 # to apply after the fgcm calibration. The first part comes
762 # from the reference catalog match (used in testing). The
763 # second part comes from the mean chromatic correction
764 # (if configured).
765 postCalibrationOffset = offsetMapping[rec['filtername']]
766 if self.config.doApplyMeanChromaticCorrection:
767 postCalibrationOffset += rec['fgcmDeltaChrom']
769 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
770 rec['fgcmfZptChebXyMax'])
771 # Convert from FGCM AB to nJy
772 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
773 rec['fgcmfZptChebXyMax'],
774 offset=postCalibrationOffset,
775 scaling=scaling)
777 if self.config.doComposeWcsJacobian:
779 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']],
780 fgcmSuperStarField,
781 fgcmZptField])
782 else:
783 # The photoCalib is just the product of the fgcmSuperStarField and the
784 # fgcmZptField
785 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
787 # The "mean" calibration will be set to the center of the ccd for reference
788 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
789 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
790 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
791 calibrationErr=calibErr,
792 calibration=fgcmField,
793 isConstant=False)
795 # Return full per-visit exposure catalogs
796 if rec['visit'] != lastVisit:
797 # This is a new visit. If the last visit was not -1, yield
798 # the ExposureCatalog
799 if lastVisit > -1:
800 # ensure that the detectors are in sorted order, for fast lookups
801 zptVisitCatalog.sort()
802 yield (int(lastVisit), zptVisitCatalog)
803 else:
804 # We need to create a new schema
805 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
806 zptExpCatSchema.addField('visit', type='L', doc='Visit number')
808 # And start a new one
809 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
810 zptVisitCatalog.setMetadata(metadata)
812 lastVisit = int(rec['visit'])
814 catRecord = zptVisitCatalog.addNew()
815 catRecord['id'] = int(rec['detector'])
816 catRecord['visit'] = rec['visit']
817 catRecord.setPhotoCalib(photoCalib)
819 # Final output of last exposure catalog
820 # ensure that the detectors are in sorted order, for fast lookups
821 zptVisitCatalog.sort()
822 yield (int(lastVisit), zptVisitCatalog)
824 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
825 """
826 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
827 and scaling.
829 Parameters
830 ----------
831 coefficients: `numpy.array`
832 Flattened array of chebyshev coefficients
833 xyMax: `list` of length 2
834 Maximum x and y of the chebyshev bounding box
835 offset: `float`, optional
836 Absolute calibration offset. Default is 0.0
837 scaling: `float`, optional
838 Flat scaling value from fgcmBuildStars. Default is 1.0
840 Returns
841 -------
842 boundedField: `lsst.afw.math.ChebyshevBoundedField`
843 """
845 orderPlus1 = int(np.sqrt(coefficients.size))
846 pars = np.zeros((orderPlus1, orderPlus1))
848 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
849 lsst.geom.Point2I(*xyMax))
851 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
852 * (10.**(offset/-2.5))*scaling)
854 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
856 return boundedField
858 def _outputAtmospheres(self, handleDict, atmCat):
859 """
860 Output the atmospheres.
862 Parameters
863 ----------
864 handleDict : `dict`
865 All data handles are `lsst.daf.butler.DeferredDatasetHandle`
866 The handleDict has the follownig keys:
868 ``"fgcmLookUpTable"``
869 handle for the FGCM look-up table.
870 atmCat : `lsst.afw.table.BaseCatalog`
871 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
873 Returns
874 -------
875 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
876 Generator that returns (visit, transmissionCurve) tuples.
877 """
878 # First, we need to grab the look-up table and key info
879 lutCat = handleDict['fgcmLookUpTable'].get()
881 atmosphereTableName = lutCat[0]['tablename']
882 elevation = lutCat[0]['elevation']
883 atmLambda = lutCat[0]['atmLambda']
884 lutCat = None
886 # Make the atmosphere table if possible
887 try:
888 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
889 atmTable.loadTable()
890 except IOError:
891 atmTable = None
893 if atmTable is None:
894 # Try to use MODTRAN instead
895 try:
896 modGen = fgcm.ModtranGenerator(elevation)
897 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
898 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
899 except (ValueError, IOError) as e:
900 raise RuntimeError("FGCM look-up-table generated with modtran, "
901 "but modtran not configured to run.") from e
903 zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
905 for i, visit in enumerate(atmCat['visit']):
906 if atmTable is not None:
907 # Interpolate the atmosphere table
908 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
909 pwv=atmCat[i]['pwv'],
910 o3=atmCat[i]['o3'],
911 tau=atmCat[i]['tau'],
912 alpha=atmCat[i]['alpha'],
913 zenith=zenith[i],
914 ctranslamstd=[atmCat[i]['cTrans'],
915 atmCat[i]['lamStd']])
916 else:
917 # Run modtran
918 modAtm = modGen(pmb=atmCat[i]['pmb'],
919 pwv=atmCat[i]['pwv'],
920 o3=atmCat[i]['o3'],
921 tau=atmCat[i]['tau'],
922 alpha=atmCat[i]['alpha'],
923 zenith=zenith[i],
924 lambdaRange=lambdaRange,
925 lambdaStep=lambdaStep,
926 ctranslamstd=[atmCat[i]['cTrans'],
927 atmCat[i]['lamStd']])
928 atmVals = modAtm['COMBINED']
930 # Now need to create something to persist...
931 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
932 wavelengths=atmLambda,
933 throughputAtMin=atmVals[0],
934 throughputAtMax=atmVals[-1])
936 yield (int(visit), curve)