Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Make the final fgcmcal output products.
25This task takes the final output from fgcmFitCycle and produces the following
26outputs for use in the DM stack: the FGCM standard stars in a reference
27catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29task can transfer the 'absolute' calibration from a reference catalog
30to put the fgcm standard stars in units of Jansky. This is accomplished
31by matching stars in a sample of healpix pixels, and applying the median
32offset per band.
33"""
35import sys
36import traceback
37import copy
39import numpy as np
40import healpy as hp
41import esutil
42from astropy import units
44import lsst.pex.config as pexConfig
45import lsst.pipe.base as pipeBase
46from lsst.afw.image import TransmissionCurve
47from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
48from lsst.pipe.tasks.photoCal import PhotoCalTask
49import lsst.geom
50import lsst.afw.image as afwImage
51import lsst.afw.math as afwMath
52import lsst.afw.table as afwTable
53from lsst.meas.algorithms import IndexerRegistry
54from lsst.meas.algorithms import DatasetConfig
55from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
57from .utilities import computeApproxPixelAreaFields
59import fgcm
61__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
64class FgcmOutputProductsConfig(pexConfig.Config):
65 """Config for FgcmOutputProductsTask"""
67 cycleNumber = pexConfig.Field(
68 doc="Final fit cycle from FGCM fit",
69 dtype=int,
70 default=None,
71 )
73 # The following fields refer to calibrating from a reference
74 # catalog, but in the future this might need to be expanded
75 doReferenceCalibration = pexConfig.Field(
76 doc=("Transfer 'absolute' calibration from reference catalog? "
77 "This afterburner step is unnecessary if reference stars "
78 "were used in the full fit in FgcmFitCycleTask."),
79 dtype=bool,
80 default=False,
81 )
82 doRefcatOutput = pexConfig.Field(
83 doc="Output standard stars in reference catalog format",
84 dtype=bool,
85 default=True,
86 )
87 doAtmosphereOutput = pexConfig.Field(
88 doc="Output atmospheres in transmission_atmosphere_fgcm format",
89 dtype=bool,
90 default=True,
91 )
92 doZeropointOutput = pexConfig.Field(
93 doc="Output zeropoints in fgcm_photoCalib format",
94 dtype=bool,
95 default=True,
96 )
97 doComposeWcsJacobian = pexConfig.Field(
98 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
99 dtype=bool,
100 default=True,
101 )
102 refObjLoader = pexConfig.ConfigurableField(
103 target=LoadIndexedReferenceObjectsTask,
104 doc="reference object loader for 'absolute' photometric calibration",
105 )
106 photoCal = pexConfig.ConfigurableField(
107 target=PhotoCalTask,
108 doc="task to perform 'absolute' calibration",
109 )
110 referencePixelizationNside = pexConfig.Field(
111 doc="Healpix nside to pixelize catalog to compare to reference catalog",
112 dtype=int,
113 default=64,
114 )
115 referencePixelizationMinStars = pexConfig.Field(
116 doc=("Minimum number of stars per healpix pixel to select for comparison"
117 "to the specified reference catalog"),
118 dtype=int,
119 default=200,
120 )
121 referenceMinMatch = pexConfig.Field(
122 doc="Minimum number of stars matched to reference catalog to be used in statistics",
123 dtype=int,
124 default=50,
125 )
126 referencePixelizationNPixels = pexConfig.Field(
127 doc=("Number of healpix pixels to sample to do comparison. "
128 "Doing too many will take a long time and not yield any more "
129 "precise results because the final number is the median offset "
130 "(per band) from the set of pixels."),
131 dtype=int,
132 default=100,
133 )
134 datasetConfig = pexConfig.ConfigField(
135 dtype=DatasetConfig,
136 doc="Configuration for writing/reading ingested catalog",
137 )
139 def setDefaults(self):
140 pexConfig.Config.setDefaults(self)
142 # In order to transfer the "absolute" calibration from a reference
143 # catalog to the relatively calibrated FGCM standard stars (one number
144 # per band), we use the PhotoCalTask to match stars in a sample of healpix
145 # pixels. These basic settings ensure that only well-measured, good stars
146 # from the source and reference catalogs are used for the matching.
148 # applyColorTerms needs to be False if doReferenceCalibration is False,
149 # as is the new default after DM-16702
150 self.photoCal.applyColorTerms = False
151 self.photoCal.fluxField = 'instFlux'
152 self.photoCal.magErrFloor = 0.003
153 self.photoCal.match.referenceSelection.doSignalToNoise = True
154 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
155 self.photoCal.match.sourceSelection.doSignalToNoise = True
156 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
157 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
158 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
159 self.photoCal.match.sourceSelection.doFlags = True
160 self.photoCal.match.sourceSelection.flags.good = []
161 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
162 self.photoCal.match.sourceSelection.doUnresolved = False
163 self.datasetConfig.ref_dataset_name = 'fgcm_stars'
164 self.datasetConfig.format_version = 1
167class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
168 """Subclass of TaskRunner for fgcmOutputProductsTask
170 fgcmOutputProductsTask.run() takes one argument, the butler, and
171 does not run on any data in the repository.
172 This runner does not use any parallelization.
173 """
175 @staticmethod
176 def getTargetList(parsedCmd):
177 """
178 Return a list with one element, the butler.
179 """
180 return [parsedCmd.butler]
182 def __call__(self, butler):
183 """
184 Parameters
185 ----------
186 butler: `lsst.daf.persistence.Butler`
188 Returns
189 -------
190 exitStatus: `list` with `pipeBase.Struct`
191 exitStatus (0: success; 1: failure)
192 if self.doReturnResults also
193 results (`np.array` with absolute zeropoint offsets)
194 """
195 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
197 exitStatus = 0
198 if self.doRaise:
199 results = task.runDataRef(butler)
200 else:
201 try:
202 results = task.runDataRef(butler)
203 except Exception as e:
204 exitStatus = 1
205 task.log.fatal("Failed: %s" % e)
206 if not isinstance(e, pipeBase.TaskError):
207 traceback.print_exc(file=sys.stderr)
209 task.writeMetadata(butler)
211 if self.doReturnResults:
212 # The results here are the zeropoint offsets for each band
213 return [pipeBase.Struct(exitStatus=exitStatus,
214 results=results)]
215 else:
216 return [pipeBase.Struct(exitStatus=exitStatus)]
218 def run(self, parsedCmd):
219 """
220 Run the task, with no multiprocessing
222 Parameters
223 ----------
224 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
225 """
227 resultList = []
229 if self.precall(parsedCmd):
230 targetList = self.getTargetList(parsedCmd)
231 # make sure that we only get 1
232 resultList = self(targetList[0])
234 return resultList
237class FgcmOutputProductsTask(pipeBase.CmdLineTask):
238 """
239 Output products from FGCM global calibration.
240 """
242 ConfigClass = FgcmOutputProductsConfig
243 RunnerClass = FgcmOutputProductsRunner
244 _DefaultName = "fgcmOutputProducts"
246 def __init__(self, butler=None, **kwargs):
247 """
248 Instantiate an fgcmOutputProductsTask.
250 Parameters
251 ----------
252 butler : `lsst.daf.persistence.Butler`
253 """
255 pipeBase.CmdLineTask.__init__(self, **kwargs)
257 if self.config.doReferenceCalibration:
258 # We need the ref obj loader to get the flux field
259 self.makeSubtask("refObjLoader", butler=butler)
261 if self.config.doRefcatOutput:
262 self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
263 self.config.datasetConfig.indexer.active)
265 # no saving of metadata for now
266 def _getMetadataName(self):
267 return None
269 @pipeBase.timeMethod
270 def runDataRef(self, butler):
271 """
272 Make FGCM output products for use in the stack
274 Parameters
275 ----------
276 butler: `lsst.daf.persistence.Butler`
277 cycleNumber: `int`
278 Final fit cycle number, override config.
280 Returns
281 -------
282 offsets: `lsst.pipe.base.Struct`
283 A structure with array of zeropoint offsets
285 Raises
286 ------
287 RuntimeError:
288 Raised if any one of the following is true:
290 - butler cannot find "fgcmBuildStars_config" or
291 "fgcmBuildStarsTable_config".
292 - butler cannot find "fgcmFitCycle_config".
293 - "fgcmFitCycle_config" does not refer to
294 `self.config.cycleNumber`.
295 - butler cannot find "fgcmAtmosphereParameters" and
296 `self.config.doAtmosphereOutput` is `True`.
297 - butler cannot find "fgcmStandardStars" and
298 `self.config.doReferenceCalibration` is `True` or
299 `self.config.doRefcatOutput` is `True`.
300 - butler cannot find "fgcmZeropoints" and
301 `self.config.doZeropointOutput` is `True`.
302 """
304 # Check to make sure that the fgcmBuildStars config exists, to retrieve
305 # the visit and ccd dataset tags
306 if not butler.datasetExists('fgcmBuildStarsTable_config') and \
307 not butler.datasetExists('fgcmBuildStars_config'):
308 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
309 "which is prereq for fgcmOutputProducts")
311 if butler.datasetExists('fgcmBuildStarsTable_config'):
312 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
313 else:
314 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
315 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
316 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
317 self.filterMap = fgcmBuildStarsConfig.filterMap
319 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
320 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
321 "in fgcmBuildStarsTask.")
323 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
324 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
326 # Make sure that the fit config exists, to retrieve bands and other info
327 if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber):
328 raise RuntimeError("Cannot find fgcmFitCycle_config from cycle %d " % (self.config.cycleNumber) +
329 "which is required for fgcmOutputProducts.")
331 fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber)
332 self.configBands = fitCycleConfig.bands
334 if self.config.doReferenceCalibration and fitCycleConfig.doReferenceCalibration:
335 self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
336 "fitCycleConfig.doReferenceCalibration")
338 # And make sure that the atmosphere was output properly
339 if (self.config.doAtmosphereOutput and
340 not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
341 raise RuntimeError("Atmosphere parameters are missing for cycle %d." %
342 (self.config.cycleNumber))
344 if ((self.config.doReferenceCalibration or self.config.doRefcatOutput) and
345 (not butler.datasetExists('fgcmStandardStars',
346 fgcmcycle=self.config.cycleNumber))):
347 raise RuntimeError("Standard stars are missing for cycle %d." %
348 (self.config.cycleNumber))
350 if (self.config.doZeropointOutput and
351 (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
352 raise RuntimeError("Zeropoints are missing for cycle %d." %
353 (self.config.cycleNumber))
355 # And make sure this is the last cycle
356 if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1):
357 raise RuntimeError("The task fgcmOutputProducts should only be run"
358 "on the final fit cycle products")
360 if self.config.doReferenceCalibration or self.config.doRefcatOutput:
361 stdCat = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
362 md = stdCat.getMetadata()
363 self.bands = md.getArray('BANDS')
364 else:
365 stdCat = None
366 self.bands = self.configBands
368 if self.config.doReferenceCalibration:
369 offsets = self._computeReferenceOffsets(butler, stdCat)
370 else:
371 offsets = np.zeros(len(self.bands))
373 # Output the standard stars in stack format
374 if self.config.doRefcatOutput:
375 self._outputStandardStars(butler, stdCat, offsets, self.config.datasetConfig)
377 del stdCat
379 # Output the gray zeropoints
380 if self.config.doZeropointOutput:
381 zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
382 visitCat = butler.get('fgcmVisitCatalog')
384 self._outputZeropoints(butler, zptCat, visitCat, offsets)
386 # Output the atmospheres
387 if self.config.doAtmosphereOutput:
388 atmCat = butler.get('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
389 self._outputAtmospheres(butler, atmCat)
391 # We return the zp offsets
392 return pipeBase.Struct(offsets=offsets)
394 def generateTractOutputProducts(self, butler, tract,
395 visitCat, zptCat, atmCat, stdCat,
396 fgcmBuildStarsConfig, fgcmFitCycleConfig):
397 """
398 Generate the output products for a given tract, as specified in the config.
400 This method is here to have an alternate entry-point for
401 FgcmCalibrateTract.
403 Parameters
404 ----------
405 butler: `lsst.daf.persistence.Butler`
406 tract: `int`
407 Tract number
408 visitCat: `lsst.afw.table.BaseCatalog`
409 FGCM visitCat from `FgcmBuildStarsTask`
410 zptCat: `lsst.afw.table.BaseCatalog`
411 FGCM zeropoint catalog from `FgcmFitCycleTask`
412 atmCat: `lsst.afw.table.BaseCatalog`
413 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
414 stdCat: `lsst.afw.table.SimpleCatalog`
415 FGCM standard star catalog from `FgcmFitCycleTask`
416 fgcmBuildStarsConfig: `lsst.fgcmcal.FgcmBuildStarsConfig`
417 Configuration object from `FgcmBuildStarsTask`
418 fgcmFitCycleConfig: `lsst.fgcmcal.FgcmFitCycleConfig`
419 Configuration object from `FgcmFitCycleTask`
420 """
422 self.configBands = fgcmFitCycleConfig.bands
423 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
424 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
425 self.filterMap = fgcmBuildStarsConfig.filterMap
427 if stdCat is not None:
428 md = stdCat.getMetadata()
429 self.bands = md.getArray('BANDS')
430 else:
431 self.bands = self.configBands
433 if self.config.doReferenceCalibration and fgcmFitCycleConfig.doReferenceCalibration:
434 self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
435 "fitCycleConfig.doReferenceCalibration")
437 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
438 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
439 "in fgcmBuildStarsTask.")
441 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
442 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
444 if self.config.doReferenceCalibration:
445 offsets = self._computeReferenceOffsets(butler, stdCat)
446 else:
447 offsets = np.zeros(len(self.bands))
449 if self.config.doRefcatOutput:
450 # Create a special config that has the tract number in it
451 datasetConfig = copy.copy(self.config.datasetConfig)
452 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
453 tract)
454 self._outputStandardStars(butler, stdCat, offsets, datasetConfig)
456 if self.config.doZeropointOutput:
457 self._outputZeropoints(butler, zptCat, visitCat, offsets, tract=tract)
459 if self.config.doAtmosphereOutput:
460 self._outputAtmospheres(butler, atmCat, tract=tract)
462 return pipeBase.Struct(offsets=offsets)
464 def _computeReferenceOffsets(self, butler, stdCat):
465 """
466 Compute offsets relative to a reference catalog.
468 This method splits the star catalog into healpix pixels
469 and computes the calibration transfer for a sample of
470 these pixels to approximate the 'absolute' calibration
471 values (on for each band) to apply to transfer the
472 absolute scale.
474 Parameters
475 ----------
476 butler: `lsst.daf.persistence.Butler`
477 stdCat: `lsst.afw.table.SimpleCatalog`
478 FGCM standard stars
480 Returns
481 -------
482 offsets: `numpy.array` of floats
483 Per band zeropoint offsets
484 """
486 # Only use stars that are observed in all the bands that were actually used
487 # This will ensure that we use the same healpix pixels for the absolute
488 # calibration of each band.
489 minObs = stdCat['ngood'].min(axis=1)
491 goodStars = (minObs >= 1)
492 stdCat = stdCat[goodStars]
494 self.log.info("Found %d stars with at least 1 good observation in each band" %
495 (len(stdCat)))
497 # We have to make a table for each pixel with flux/fluxErr
498 # This is a temporary table generated for input to the photoCal task.
499 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
500 # have had chromatic corrections applied to get to the standard system
501 # specified by the atmosphere/instrumental parameters), nor are they
502 # in Jansky (since they don't have a proper absolute calibration: the overall
503 # zeropoint is estimated from the telescope size, etc.)
504 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
505 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
506 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
507 doc="instrumental flux (counts)")
508 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
509 doc="instrumental flux error (counts)")
510 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
511 type='Flag',
512 doc="bad flag")
514 # Split up the stars
515 # Note that there is an assumption here that the ra/dec coords stored
516 # on-disk are in radians, and therefore that starObs['coord_ra'] /
517 # starObs['coord_dec'] return radians when used as an array of numpy float64s.
518 theta = np.pi/2. - stdCat['coord_dec']
519 phi = stdCat['coord_ra']
521 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
522 h, rev = esutil.stat.histogram(ipring, rev=True)
524 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
526 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
527 (gdpix.size,
528 self.config.referencePixelizationNside,
529 self.config.referencePixelizationMinStars))
531 if gdpix.size < self.config.referencePixelizationNPixels:
532 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
533 (gdpix.size, self.config.referencePixelizationNPixels))
534 else:
535 # Sample out the pixels we want to use
536 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
538 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
539 ('nstar', 'i4', len(self.bands)),
540 ('nmatch', 'i4', len(self.bands)),
541 ('zp', 'f4', len(self.bands)),
542 ('zpErr', 'f4', len(self.bands))])
543 results['hpix'] = ipring[rev[rev[gdpix]]]
545 # We need a boolean index to deal with catalogs...
546 selected = np.zeros(len(stdCat), dtype=np.bool)
548 refFluxFields = [None]*len(self.bands)
550 for p, pix in enumerate(gdpix):
551 i1a = rev[rev[pix]: rev[pix + 1]]
553 # the stdCat afwTable can only be indexed with boolean arrays,
554 # and not numpy index arrays (see DM-16497). This little trick
555 # converts the index array into a boolean array
556 selected[:] = False
557 selected[i1a] = True
559 for b, band in enumerate(self.bands):
561 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
562 selected, refFluxFields)
563 results['nstar'][p, b] = len(i1a)
564 results['nmatch'][p, b] = len(struct.arrays.refMag)
565 results['zp'][p, b] = struct.zp
566 results['zpErr'][p, b] = struct.sigma
568 # And compute the summary statistics
569 offsets = np.zeros(len(self.bands))
571 for b, band in enumerate(self.bands):
572 # make configurable
573 ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)
574 offsets[b] = np.median(results['zp'][ok, b])
575 # use median absolute deviation to estimate Normal sigma
576 # see https://en.wikipedia.org/wiki/Median_absolute_deviation
577 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b]))
578 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" %
579 (band, offsets[b], madSigma))
581 return offsets
583 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
584 b, band, stdCat, selected, refFluxFields):
585 """
586 Compute the zeropoint offset between the fgcm stdCat and the reference
587 stars for one pixel in one band
589 Parameters
590 ----------
591 sourceMapper: `lsst.afw.table.SchemaMapper`
592 Mapper to go from stdCat to calibratable catalog
593 badStarKey: `lsst.afw.table.Key`
594 Key for the field with bad stars
595 b: `int`
596 Index of the band in the star catalog
597 band: `str`
598 Name of band for reference catalog
599 stdCat: `lsst.afw.table.SimpleCatalog`
600 FGCM standard stars
601 selected: `numpy.array(dtype=np.bool)`
602 Boolean array of which stars are in the pixel
603 refFluxFields: `list`
604 List of names of flux fields for reference catalog
605 """
607 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
608 sourceCat.reserve(selected.sum())
609 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
610 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5))
611 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] *
612 sourceCat['instFlux'])
613 # Make sure we only use stars that have valid measurements
614 # (This is perhaps redundant with requirements above that the
615 # stars be observed in all bands, but it can't hurt)
616 badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0)
617 for rec in sourceCat[badStar]:
618 rec.set(badStarKey, True)
620 exposure = afwImage.ExposureF()
621 exposure.setFilter(afwImage.Filter(band))
623 if refFluxFields[b] is None:
624 # Need to find the flux field in the reference catalog
625 # to work around limitations of DirectMatch in PhotoCal
626 ctr = stdCat[0].getCoord()
627 rad = 0.05*lsst.geom.degrees
628 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
629 refFluxFields[b] = refDataTest.fluxField
631 # Make a copy of the config so that we can modify it
632 calConfig = copy.copy(self.config.photoCal.value)
633 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
634 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'
635 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
636 config=calConfig,
637 schema=sourceCat.getSchema())
639 struct = calTask.run(exposure, sourceCat)
641 return struct
643 def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig):
644 """
645 Output standard stars in indexed reference catalog format.
647 Parameters
648 ----------
649 butler: `lsst.daf.persistence.Butler`
650 stdCat: `lsst.afw.table.SimpleCatalog`
651 FGCM standard star catalog from fgcmFitCycleTask
652 offsets: `numpy.array` of floats
653 Per band zeropoint offsets
654 datasetConfig: `lsst.meas.algorithms.DatasetConfig`
655 Config for reference dataset
656 """
658 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
660 # We determine the conversion from the native units (typically radians) to
661 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
662 # numpy arrays rather than Angles, which would we approximately 600x slower.
663 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
664 # (as Angles) for input
665 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
666 indices = np.array(self.indexer.indexPoints(stdCat['coord_ra']*conv,
667 stdCat['coord_dec']*conv))
669 formattedCat = self._formatCatalog(stdCat, offsets)
671 # Write the master schema
672 dataId = self.indexer.makeDataId('master_schema',
673 datasetConfig.ref_dataset_name)
674 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
675 addRefCatMetadata(masterCat)
676 butler.put(masterCat, 'ref_cat', dataId=dataId)
678 # Break up the pixels using a histogram
679 h, rev = esutil.stat.histogram(indices, rev=True)
680 gd, = np.where(h > 0)
681 selected = np.zeros(len(formattedCat), dtype=np.bool)
682 for i in gd:
683 i1a = rev[rev[i]: rev[i + 1]]
685 # the formattedCat afwTable can only be indexed with boolean arrays,
686 # and not numpy index arrays (see DM-16497). This little trick
687 # converts the index array into a boolean array
688 selected[:] = False
689 selected[i1a] = True
691 # Write the individual pixel
692 dataId = self.indexer.makeDataId(indices[i1a[0]],
693 datasetConfig.ref_dataset_name)
694 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
696 # And save the dataset configuration
697 dataId = self.indexer.makeDataId(None, datasetConfig.ref_dataset_name)
698 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
700 self.log.info("Done outputting standard stars.")
702 def _formatCatalog(self, fgcmStarCat, offsets):
703 """
704 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
706 Parameters
707 ----------
708 fgcmStarCat: `lsst.afw.Table.SimpleCatalog`
709 SimpleCatalog as output by fgcmcal
710 offsets: `list` with len(self.bands) entries
711 Zeropoint offsets to apply
713 Returns
714 -------
715 formattedCat: `lsst.afw.table.SimpleCatalog`
716 SimpleCatalog suitable for using as a reference catalog
717 """
719 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
720 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(self.bands,
721 addCentroid=False,
722 addIsResolved=True,
723 coordErrDim=0)
724 sourceMapper.addMinimalSchema(minSchema)
725 for band in self.bands:
726 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
727 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
728 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
730 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
731 formattedCat.reserve(len(fgcmStarCat))
732 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
734 # Note that we don't have to set `resolved` because the default is False
736 for b, band in enumerate(self.bands):
737 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
738 # We want fluxes in nJy from calibrated AB magnitudes
739 # (after applying offset). Updated after RFC-549 and RFC-575.
740 flux = (mag*units.ABmag).to_value(units.nJy)
741 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
743 formattedCat['%s_flux' % (band)][:] = flux
744 formattedCat['%s_fluxErr' % (band)][:] = fluxErr
745 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
746 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
747 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
749 addRefCatMetadata(formattedCat)
751 return formattedCat
753 def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None):
754 """
755 Output the zeropoints in fgcm_photoCalib format.
757 Parameters
758 ----------
759 butler: `lsst.daf.persistence.Butler`
760 zptCat: `lsst.afw.table.BaseCatalog`
761 FGCM zeropoint catalog from `FgcmFitCycleTask`
762 visitCat: `lsst.afw.table.BaseCatalog`
763 FGCM visitCat from `FgcmBuildStarsTask`
764 offsets: `numpy.array`
765 Float array of absolute calibration offsets, one for each filter
766 tract: `int`, optional
767 Tract number to output. Default is None (global calibration)
768 """
770 if tract is None:
771 datasetType = 'fgcm_photoCalib'
772 else:
773 datasetType = 'fgcm_tract_photoCalib'
775 self.log.info("Outputting %s objects" % (datasetType))
777 # Select visit/ccds where we have a calibration
778 # This includes ccds where we were able to interpolate from neighboring
779 # ccds.
780 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
781 too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD']
782 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) &
783 (zptCat['fgcmZptVar'] > 0.0))
785 # We also select the "best" calibrations, avoiding interpolation. These
786 # are only used for mapping filternames
787 selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) &
788 (zptCat['fgcmZptVar'] > 0.0))
790 # Log warnings for any visit which has no valid zeropoints
791 badVisits = np.unique(zptCat['visit'][~selected])
792 goodVisits = np.unique(zptCat['visit'][selected])
793 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
794 for allBadVisit in allBadVisits:
795 self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}')
797 # Get the mapping from filtername to dataId filter name, empirically
798 filterMapping = {}
799 nFound = 0
800 for rec in zptCat[selected_best]:
801 if rec['filtername'] in filterMapping:
802 continue
803 dataId = {self.visitDataRefName: int(rec['visit']),
804 self.ccdDataRefName: int(rec['ccd'])}
805 dataRef = butler.dataRef('raw', dataId=dataId)
806 filterMapping[rec['filtername']] = dataRef.dataId['filter']
807 nFound += 1
808 if nFound == len(self.filterMap):
809 break
811 # Get a mapping from filtername to the offsets
812 offsetMapping = {}
813 for f in self.filterMap:
814 # Not every filter in the map will necesarily have a band.
815 if self.filterMap[f] in self.bands:
816 offsetMapping[f] = offsets[self.bands.index(self.filterMap[f])]
818 # Get a mapping from "ccd" to the ccd index used for the scaling
819 camera = butler.get('camera')
820 ccdMapping = {}
821 for ccdIndex, detector in enumerate(camera):
822 ccdMapping[detector.getId()] = ccdIndex
824 # And a mapping to get the flat-field scaling values
825 scalingMapping = {}
826 for rec in visitCat:
827 scalingMapping[rec['visit']] = rec['scaling']
829 if self.config.doComposeWcsJacobian:
830 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
832 for rec in zptCat[selected]:
834 # Retrieve overall scaling
835 scaling = scalingMapping[rec['visit']][ccdMapping[rec['ccd']]]
837 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
838 rec['fgcmfZptChebXyMax'])
839 # Convert from FGCM AB to nJy
840 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
841 rec['fgcmfZptChebXyMax'],
842 offset=offsetMapping[rec['filtername']],
843 scaling=scaling)
845 if self.config.doComposeWcsJacobian:
847 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['ccd']],
848 fgcmSuperStarField,
849 fgcmZptField])
850 else:
851 # The photoCalib is just the product of the fgcmSuperStarField and the
852 # fgcmZptField
853 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
855 # The "mean" calibration will be set to the center of the ccd for reference
856 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
857 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
858 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
859 calibrationErr=calibErr,
860 calibration=fgcmField,
861 isConstant=False)
863 if tract is None:
864 butler.put(photoCalib, datasetType,
865 dataId={self.visitDataRefName: int(rec['visit']),
866 self.ccdDataRefName: int(rec['ccd']),
867 'filter': filterMapping[rec['filtername']]})
868 else:
869 butler.put(photoCalib, datasetType,
870 dataId={self.visitDataRefName: int(rec['visit']),
871 self.ccdDataRefName: int(rec['ccd']),
872 'filter': filterMapping[rec['filtername']],
873 'tract': tract})
875 self.log.info("Done outputting %s objects" % (datasetType))
877 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
878 """
879 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
880 and scaling.
882 Parameters
883 ----------
884 coefficients: `numpy.array`
885 Flattened array of chebyshev coefficients
886 xyMax: `list` of length 2
887 Maximum x and y of the chebyshev bounding box
888 offset: `float`, optional
889 Absolute calibration offset. Default is 0.0
890 scaling: `float`, optional
891 Flat scaling value from fgcmBuildStars. Default is 1.0
893 Returns
894 -------
895 boundedField: `lsst.afw.math.ChebyshevBoundedField`
896 """
898 orderPlus1 = int(np.sqrt(coefficients.size))
899 pars = np.zeros((orderPlus1, orderPlus1))
901 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
902 lsst.geom.Point2I(*xyMax))
904 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) *
905 (10.**(offset/-2.5))*scaling)
907 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
909 return boundedField
911 def _outputAtmospheres(self, butler, atmCat, tract=None):
912 """
913 Output the atmospheres.
915 Parameters
916 ----------
917 butler: `lsst.daf.persistence.Butler`
918 atmCat: `lsst.afw.table.BaseCatalog`
919 FGCM atmosphere parameter catalog from fgcmFitCycleTask
920 tract: `int`, optional
921 Tract number to output. Default is None (global calibration)
922 """
924 self.log.info("Outputting atmosphere transmissions")
926 # First, we need to grab the look-up table and key info
927 lutCat = butler.get('fgcmLookUpTable')
929 atmosphereTableName = lutCat[0]['tablename']
930 elevation = lutCat[0]['elevation']
931 atmLambda = lutCat[0]['atmLambda']
932 lutCat = None
934 # Make the atmosphere table if possible
935 try:
936 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
937 atmTable.loadTable()
938 except IOError:
939 atmTable = None
941 if atmTable is None:
942 # Try to use MODTRAN instead
943 try:
944 modGen = fgcm.ModtranGenerator(elevation)
945 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
946 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
947 except (ValueError, IOError) as e:
948 raise RuntimeError("FGCM look-up-table generated with modtran, "
949 "but modtran not configured to run.") from e
951 zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
953 for i, visit in enumerate(atmCat['visit']):
954 if atmTable is not None:
955 # Interpolate the atmosphere table
956 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
957 pwv=atmCat[i]['pwv'],
958 o3=atmCat[i]['o3'],
959 tau=atmCat[i]['tau'],
960 alpha=atmCat[i]['alpha'],
961 zenith=zenith[i],
962 ctranslamstd=[atmCat[i]['cTrans'],
963 atmCat[i]['lamStd']])
964 else:
965 # Run modtran
966 modAtm = modGen(pmb=atmCat[i]['pmb'],
967 pwv=atmCat[i]['pwv'],
968 o3=atmCat[i]['o3'],
969 tau=atmCat[i]['tau'],
970 alpha=atmCat[i]['alpha'],
971 zenith=zenith[i],
972 lambdaRange=lambdaRange,
973 lambdaStep=lambdaStep,
974 ctranslamstd=[atmCat[i]['cTrans'],
975 atmCat[i]['lamStd']])
976 atmVals = modAtm['COMBINED']
978 # Now need to create something to persist...
979 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
980 wavelengths=atmLambda,
981 throughputAtMin=atmVals[0],
982 throughputAtMax=atmVals[-1])
984 if tract is None:
985 butler.put(curve, "transmission_atmosphere_fgcm",
986 dataId={self.visitDataRefName: visit})
987 else:
988 butler.put(curve, "transmission_atmosphere_fgcm_tract",
989 dataId={self.visitDataRefName: visit,
990 'tract': tract})
992 self.log.info("Done outputting atmosphere transmissions")