Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Make the final fgcmcal output products.
25This task takes the final output from fgcmFitCycle and produces the following
26outputs for use in the DM stack: the FGCM standard stars in a reference
27catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29task can transfer the 'absolute' calibration from a reference catalog
30to put the fgcm standard stars in units of Jansky. This is accomplished
31by matching stars in a sample of healpix pixels, and applying the median
32offset per band.
33"""
35import sys
36import traceback
37import copy
39import numpy as np
40import healpy as hp
41import esutil
42from astropy import units
44import lsst.pex.config as pexConfig
45import lsst.pipe.base as pipeBase
46from lsst.afw.image import TransmissionCurve
47from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
48from lsst.pipe.tasks.photoCal import PhotoCalTask
49import lsst.geom
50import lsst.afw.image as afwImage
51import lsst.afw.math as afwMath
52import lsst.afw.table as afwTable
53from lsst.meas.algorithms import IndexerRegistry
54from lsst.meas.algorithms import DatasetConfig
55from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
57from .utilities import computeApproxPixelAreaFields
59import fgcm
61__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
64class FgcmOutputProductsConfig(pexConfig.Config):
65 """Config for FgcmOutputProductsTask"""
67 cycleNumber = pexConfig.Field(
68 doc="Final fit cycle from FGCM fit",
69 dtype=int,
70 default=None,
71 )
73 # The following fields refer to calibrating from a reference
74 # catalog, but in the future this might need to be expanded
75 doReferenceCalibration = pexConfig.Field(
76 doc=("Transfer 'absolute' calibration from reference catalog? "
77 "This afterburner step is unnecessary if reference stars "
78 "were used in the full fit in FgcmFitCycleTask."),
79 dtype=bool,
80 default=False,
81 )
82 doRefcatOutput = pexConfig.Field(
83 doc="Output standard stars in reference catalog format",
84 dtype=bool,
85 default=True,
86 )
87 doAtmosphereOutput = pexConfig.Field(
88 doc="Output atmospheres in transmission_atmosphere_fgcm format",
89 dtype=bool,
90 default=True,
91 )
92 doZeropointOutput = pexConfig.Field(
93 doc="Output zeropoints in fgcm_photoCalib format",
94 dtype=bool,
95 default=True,
96 )
97 doComposeWcsJacobian = pexConfig.Field(
98 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
99 dtype=bool,
100 default=True,
101 )
102 doApplyMeanChromaticCorrection = pexConfig.Field(
103 doc="Apply the mean chromatic correction to the zeropoints?",
104 dtype=bool,
105 default=True,
106 )
107 refObjLoader = pexConfig.ConfigurableField(
108 target=LoadIndexedReferenceObjectsTask,
109 doc="reference object loader for 'absolute' photometric calibration",
110 )
111 photoCal = pexConfig.ConfigurableField(
112 target=PhotoCalTask,
113 doc="task to perform 'absolute' calibration",
114 )
115 referencePixelizationNside = pexConfig.Field(
116 doc="Healpix nside to pixelize catalog to compare to reference catalog",
117 dtype=int,
118 default=64,
119 )
120 referencePixelizationMinStars = pexConfig.Field(
121 doc=("Minimum number of stars per healpix pixel to select for comparison"
122 "to the specified reference catalog"),
123 dtype=int,
124 default=200,
125 )
126 referenceMinMatch = pexConfig.Field(
127 doc="Minimum number of stars matched to reference catalog to be used in statistics",
128 dtype=int,
129 default=50,
130 )
131 referencePixelizationNPixels = pexConfig.Field(
132 doc=("Number of healpix pixels to sample to do comparison. "
133 "Doing too many will take a long time and not yield any more "
134 "precise results because the final number is the median offset "
135 "(per band) from the set of pixels."),
136 dtype=int,
137 default=100,
138 )
139 datasetConfig = pexConfig.ConfigField(
140 dtype=DatasetConfig,
141 doc="Configuration for writing/reading ingested catalog",
142 )
144 def setDefaults(self):
145 pexConfig.Config.setDefaults(self)
147 # In order to transfer the "absolute" calibration from a reference
148 # catalog to the relatively calibrated FGCM standard stars (one number
149 # per band), we use the PhotoCalTask to match stars in a sample of healpix
150 # pixels. These basic settings ensure that only well-measured, good stars
151 # from the source and reference catalogs are used for the matching.
153 # applyColorTerms needs to be False if doReferenceCalibration is False,
154 # as is the new default after DM-16702
155 self.photoCal.applyColorTerms = False
156 self.photoCal.fluxField = 'instFlux'
157 self.photoCal.magErrFloor = 0.003
158 self.photoCal.match.referenceSelection.doSignalToNoise = True
159 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
160 self.photoCal.match.sourceSelection.doSignalToNoise = True
161 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
162 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
163 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
164 self.photoCal.match.sourceSelection.doFlags = True
165 self.photoCal.match.sourceSelection.flags.good = []
166 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
167 self.photoCal.match.sourceSelection.doUnresolved = False
168 self.datasetConfig.ref_dataset_name = 'fgcm_stars'
169 self.datasetConfig.format_version = 1
172class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
173 """Subclass of TaskRunner for fgcmOutputProductsTask
175 fgcmOutputProductsTask.run() takes one argument, the butler, and
176 does not run on any data in the repository.
177 This runner does not use any parallelization.
178 """
180 @staticmethod
181 def getTargetList(parsedCmd):
182 """
183 Return a list with one element, the butler.
184 """
185 return [parsedCmd.butler]
187 def __call__(self, butler):
188 """
189 Parameters
190 ----------
191 butler: `lsst.daf.persistence.Butler`
193 Returns
194 -------
195 exitStatus: `list` with `pipeBase.Struct`
196 exitStatus (0: success; 1: failure)
197 if self.doReturnResults also
198 results (`np.array` with absolute zeropoint offsets)
199 """
200 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
202 exitStatus = 0
203 if self.doRaise:
204 results = task.runDataRef(butler)
205 else:
206 try:
207 results = task.runDataRef(butler)
208 except Exception as e:
209 exitStatus = 1
210 task.log.fatal("Failed: %s" % e)
211 if not isinstance(e, pipeBase.TaskError):
212 traceback.print_exc(file=sys.stderr)
214 task.writeMetadata(butler)
216 if self.doReturnResults:
217 # The results here are the zeropoint offsets for each band
218 return [pipeBase.Struct(exitStatus=exitStatus,
219 results=results)]
220 else:
221 return [pipeBase.Struct(exitStatus=exitStatus)]
223 def run(self, parsedCmd):
224 """
225 Run the task, with no multiprocessing
227 Parameters
228 ----------
229 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
230 """
232 resultList = []
234 if self.precall(parsedCmd):
235 targetList = self.getTargetList(parsedCmd)
236 # make sure that we only get 1
237 resultList = self(targetList[0])
239 return resultList
242class FgcmOutputProductsTask(pipeBase.CmdLineTask):
243 """
244 Output products from FGCM global calibration.
245 """
247 ConfigClass = FgcmOutputProductsConfig
248 RunnerClass = FgcmOutputProductsRunner
249 _DefaultName = "fgcmOutputProducts"
251 def __init__(self, butler=None, **kwargs):
252 """
253 Instantiate an fgcmOutputProductsTask.
255 Parameters
256 ----------
257 butler : `lsst.daf.persistence.Butler`
258 """
260 pipeBase.CmdLineTask.__init__(self, **kwargs)
262 if self.config.doReferenceCalibration:
263 # We need the ref obj loader to get the flux field
264 self.makeSubtask("refObjLoader", butler=butler)
266 if self.config.doRefcatOutput:
267 self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
268 self.config.datasetConfig.indexer.active)
270 # no saving of metadata for now
271 def _getMetadataName(self):
272 return None
274 @pipeBase.timeMethod
275 def runDataRef(self, butler):
276 """
277 Make FGCM output products for use in the stack
279 Parameters
280 ----------
281 butler: `lsst.daf.persistence.Butler`
282 cycleNumber: `int`
283 Final fit cycle number, override config.
285 Returns
286 -------
287 offsets: `lsst.pipe.base.Struct`
288 A structure with array of zeropoint offsets
290 Raises
291 ------
292 RuntimeError:
293 Raised if any one of the following is true:
295 - butler cannot find "fgcmBuildStars_config" or
296 "fgcmBuildStarsTable_config".
297 - butler cannot find "fgcmFitCycle_config".
298 - "fgcmFitCycle_config" does not refer to
299 `self.config.cycleNumber`.
300 - butler cannot find "fgcmAtmosphereParameters" and
301 `self.config.doAtmosphereOutput` is `True`.
302 - butler cannot find "fgcmStandardStars" and
303 `self.config.doReferenceCalibration` is `True` or
304 `self.config.doRefcatOutput` is `True`.
305 - butler cannot find "fgcmZeropoints" and
306 `self.config.doZeropointOutput` is `True`.
307 """
309 # Check to make sure that the fgcmBuildStars config exists, to retrieve
310 # the visit and ccd dataset tags
311 if not butler.datasetExists('fgcmBuildStarsTable_config') and \
312 not butler.datasetExists('fgcmBuildStars_config'):
313 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
314 "which is prereq for fgcmOutputProducts")
316 if butler.datasetExists('fgcmBuildStarsTable_config'):
317 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
318 else:
319 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
320 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
321 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
322 self.filterMap = fgcmBuildStarsConfig.filterMap
324 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
325 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
326 "in fgcmBuildStarsTask.")
328 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
329 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
331 # Make sure that the fit config exists, to retrieve bands and other info
332 if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber):
333 raise RuntimeError("Cannot find fgcmFitCycle_config from cycle %d " % (self.config.cycleNumber) +
334 "which is required for fgcmOutputProducts.")
336 fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber)
337 self.configBands = fitCycleConfig.bands
339 if self.config.doReferenceCalibration and fitCycleConfig.doReferenceCalibration:
340 self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
341 "fitCycleConfig.doReferenceCalibration")
343 # And make sure that the atmosphere was output properly
344 if (self.config.doAtmosphereOutput and
345 not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
346 raise RuntimeError("Atmosphere parameters are missing for cycle %d." %
347 (self.config.cycleNumber))
349 if ((self.config.doReferenceCalibration or self.config.doRefcatOutput) and
350 (not butler.datasetExists('fgcmStandardStars',
351 fgcmcycle=self.config.cycleNumber))):
352 raise RuntimeError("Standard stars are missing for cycle %d." %
353 (self.config.cycleNumber))
355 if (self.config.doZeropointOutput and
356 (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
357 raise RuntimeError("Zeropoints are missing for cycle %d." %
358 (self.config.cycleNumber))
360 # And make sure this is the last cycle
361 if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1):
362 raise RuntimeError("The task fgcmOutputProducts should only be run"
363 "on the final fit cycle products")
365 if self.config.doReferenceCalibration or self.config.doRefcatOutput:
366 stdCat = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
367 md = stdCat.getMetadata()
368 self.bands = md.getArray('BANDS')
369 else:
370 stdCat = None
371 self.bands = self.configBands
373 if self.config.doReferenceCalibration:
374 offsets = self._computeReferenceOffsets(butler, stdCat)
375 else:
376 offsets = np.zeros(len(self.bands))
378 # Output the standard stars in stack format
379 if self.config.doRefcatOutput:
380 self._outputStandardStars(butler, stdCat, offsets, self.config.datasetConfig)
382 del stdCat
384 # Output the gray zeropoints
385 if self.config.doZeropointOutput:
386 zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
387 visitCat = butler.get('fgcmVisitCatalog')
389 self._outputZeropoints(butler, zptCat, visitCat, offsets)
391 # Output the atmospheres
392 if self.config.doAtmosphereOutput:
393 atmCat = butler.get('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
394 self._outputAtmospheres(butler, atmCat)
396 # We return the zp offsets
397 return pipeBase.Struct(offsets=offsets)
399 def generateTractOutputProducts(self, butler, tract,
400 visitCat, zptCat, atmCat, stdCat,
401 fgcmBuildStarsConfig, fgcmFitCycleConfig):
402 """
403 Generate the output products for a given tract, as specified in the config.
405 This method is here to have an alternate entry-point for
406 FgcmCalibrateTract.
408 Parameters
409 ----------
410 butler: `lsst.daf.persistence.Butler`
411 tract: `int`
412 Tract number
413 visitCat: `lsst.afw.table.BaseCatalog`
414 FGCM visitCat from `FgcmBuildStarsTask`
415 zptCat: `lsst.afw.table.BaseCatalog`
416 FGCM zeropoint catalog from `FgcmFitCycleTask`
417 atmCat: `lsst.afw.table.BaseCatalog`
418 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
419 stdCat: `lsst.afw.table.SimpleCatalog`
420 FGCM standard star catalog from `FgcmFitCycleTask`
421 fgcmBuildStarsConfig: `lsst.fgcmcal.FgcmBuildStarsConfig`
422 Configuration object from `FgcmBuildStarsTask`
423 fgcmFitCycleConfig: `lsst.fgcmcal.FgcmFitCycleConfig`
424 Configuration object from `FgcmFitCycleTask`
425 """
427 self.configBands = fgcmFitCycleConfig.bands
428 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
429 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
430 self.filterMap = fgcmBuildStarsConfig.filterMap
432 if stdCat is not None:
433 md = stdCat.getMetadata()
434 self.bands = md.getArray('BANDS')
435 else:
436 self.bands = self.configBands
438 if self.config.doReferenceCalibration and fgcmFitCycleConfig.doReferenceCalibration:
439 self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
440 "fitCycleConfig.doReferenceCalibration")
442 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
443 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
444 "in fgcmBuildStarsTask.")
446 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
447 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
449 if self.config.doReferenceCalibration:
450 offsets = self._computeReferenceOffsets(butler, stdCat)
451 else:
452 offsets = np.zeros(len(self.bands))
454 if self.config.doRefcatOutput:
455 # Create a special config that has the tract number in it
456 datasetConfig = copy.copy(self.config.datasetConfig)
457 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
458 tract)
459 self._outputStandardStars(butler, stdCat, offsets, datasetConfig)
461 if self.config.doZeropointOutput:
462 self._outputZeropoints(butler, zptCat, visitCat, offsets, tract=tract)
464 if self.config.doAtmosphereOutput:
465 self._outputAtmospheres(butler, atmCat, tract=tract)
467 return pipeBase.Struct(offsets=offsets)
469 def _computeReferenceOffsets(self, butler, stdCat):
470 """
471 Compute offsets relative to a reference catalog.
473 This method splits the star catalog into healpix pixels
474 and computes the calibration transfer for a sample of
475 these pixels to approximate the 'absolute' calibration
476 values (on for each band) to apply to transfer the
477 absolute scale.
479 Parameters
480 ----------
481 butler: `lsst.daf.persistence.Butler`
482 stdCat: `lsst.afw.table.SimpleCatalog`
483 FGCM standard stars
485 Returns
486 -------
487 offsets: `numpy.array` of floats
488 Per band zeropoint offsets
489 """
491 # Only use stars that are observed in all the bands that were actually used
492 # This will ensure that we use the same healpix pixels for the absolute
493 # calibration of each band.
494 minObs = stdCat['ngood'].min(axis=1)
496 goodStars = (minObs >= 1)
497 stdCat = stdCat[goodStars]
499 self.log.info("Found %d stars with at least 1 good observation in each band" %
500 (len(stdCat)))
502 # We have to make a table for each pixel with flux/fluxErr
503 # This is a temporary table generated for input to the photoCal task.
504 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
505 # have had chromatic corrections applied to get to the standard system
506 # specified by the atmosphere/instrumental parameters), nor are they
507 # in Jansky (since they don't have a proper absolute calibration: the overall
508 # zeropoint is estimated from the telescope size, etc.)
509 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
510 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
511 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
512 doc="instrumental flux (counts)")
513 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
514 doc="instrumental flux error (counts)")
515 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
516 type='Flag',
517 doc="bad flag")
519 # Split up the stars
520 # Note that there is an assumption here that the ra/dec coords stored
521 # on-disk are in radians, and therefore that starObs['coord_ra'] /
522 # starObs['coord_dec'] return radians when used as an array of numpy float64s.
523 theta = np.pi/2. - stdCat['coord_dec']
524 phi = stdCat['coord_ra']
526 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
527 h, rev = esutil.stat.histogram(ipring, rev=True)
529 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
531 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
532 (gdpix.size,
533 self.config.referencePixelizationNside,
534 self.config.referencePixelizationMinStars))
536 if gdpix.size < self.config.referencePixelizationNPixels:
537 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
538 (gdpix.size, self.config.referencePixelizationNPixels))
539 else:
540 # Sample out the pixels we want to use
541 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
543 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
544 ('nstar', 'i4', len(self.bands)),
545 ('nmatch', 'i4', len(self.bands)),
546 ('zp', 'f4', len(self.bands)),
547 ('zpErr', 'f4', len(self.bands))])
548 results['hpix'] = ipring[rev[rev[gdpix]]]
550 # We need a boolean index to deal with catalogs...
551 selected = np.zeros(len(stdCat), dtype=np.bool)
553 refFluxFields = [None]*len(self.bands)
555 for p, pix in enumerate(gdpix):
556 i1a = rev[rev[pix]: rev[pix + 1]]
558 # the stdCat afwTable can only be indexed with boolean arrays,
559 # and not numpy index arrays (see DM-16497). This little trick
560 # converts the index array into a boolean array
561 selected[:] = False
562 selected[i1a] = True
564 for b, band in enumerate(self.bands):
566 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
567 selected, refFluxFields)
568 results['nstar'][p, b] = len(i1a)
569 results['nmatch'][p, b] = len(struct.arrays.refMag)
570 results['zp'][p, b] = struct.zp
571 results['zpErr'][p, b] = struct.sigma
573 # And compute the summary statistics
574 offsets = np.zeros(len(self.bands))
576 for b, band in enumerate(self.bands):
577 # make configurable
578 ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)
579 offsets[b] = np.median(results['zp'][ok, b])
580 # use median absolute deviation to estimate Normal sigma
581 # see https://en.wikipedia.org/wiki/Median_absolute_deviation
582 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b]))
583 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" %
584 (band, offsets[b], madSigma))
586 return offsets
588 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
589 b, band, stdCat, selected, refFluxFields):
590 """
591 Compute the zeropoint offset between the fgcm stdCat and the reference
592 stars for one pixel in one band
594 Parameters
595 ----------
596 sourceMapper: `lsst.afw.table.SchemaMapper`
597 Mapper to go from stdCat to calibratable catalog
598 badStarKey: `lsst.afw.table.Key`
599 Key for the field with bad stars
600 b: `int`
601 Index of the band in the star catalog
602 band: `str`
603 Name of band for reference catalog
604 stdCat: `lsst.afw.table.SimpleCatalog`
605 FGCM standard stars
606 selected: `numpy.array(dtype=np.bool)`
607 Boolean array of which stars are in the pixel
608 refFluxFields: `list`
609 List of names of flux fields for reference catalog
610 """
612 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
613 sourceCat.reserve(selected.sum())
614 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
615 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5))
616 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] *
617 sourceCat['instFlux'])
618 # Make sure we only use stars that have valid measurements
619 # (This is perhaps redundant with requirements above that the
620 # stars be observed in all bands, but it can't hurt)
621 badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0)
622 for rec in sourceCat[badStar]:
623 rec.set(badStarKey, True)
625 exposure = afwImage.ExposureF()
626 exposure.setFilter(afwImage.Filter(band))
628 if refFluxFields[b] is None:
629 # Need to find the flux field in the reference catalog
630 # to work around limitations of DirectMatch in PhotoCal
631 ctr = stdCat[0].getCoord()
632 rad = 0.05*lsst.geom.degrees
633 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
634 refFluxFields[b] = refDataTest.fluxField
636 # Make a copy of the config so that we can modify it
637 calConfig = copy.copy(self.config.photoCal.value)
638 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
639 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'
640 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
641 config=calConfig,
642 schema=sourceCat.getSchema())
644 struct = calTask.run(exposure, sourceCat)
646 return struct
648 def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig):
649 """
650 Output standard stars in indexed reference catalog format.
652 Parameters
653 ----------
654 butler: `lsst.daf.persistence.Butler`
655 stdCat: `lsst.afw.table.SimpleCatalog`
656 FGCM standard star catalog from fgcmFitCycleTask
657 offsets: `numpy.array` of floats
658 Per band zeropoint offsets
659 datasetConfig: `lsst.meas.algorithms.DatasetConfig`
660 Config for reference dataset
661 """
663 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
665 # We determine the conversion from the native units (typically radians) to
666 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
667 # numpy arrays rather than Angles, which would we approximately 600x slower.
668 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
669 # (as Angles) for input
670 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
671 indices = np.array(self.indexer.indexPoints(stdCat['coord_ra']*conv,
672 stdCat['coord_dec']*conv))
674 formattedCat = self._formatCatalog(stdCat, offsets)
676 # Write the master schema
677 dataId = self.indexer.makeDataId('master_schema',
678 datasetConfig.ref_dataset_name)
679 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
680 addRefCatMetadata(masterCat)
681 butler.put(masterCat, 'ref_cat', dataId=dataId)
683 # Break up the pixels using a histogram
684 h, rev = esutil.stat.histogram(indices, rev=True)
685 gd, = np.where(h > 0)
686 selected = np.zeros(len(formattedCat), dtype=np.bool)
687 for i in gd:
688 i1a = rev[rev[i]: rev[i + 1]]
690 # the formattedCat afwTable can only be indexed with boolean arrays,
691 # and not numpy index arrays (see DM-16497). This little trick
692 # converts the index array into a boolean array
693 selected[:] = False
694 selected[i1a] = True
696 # Write the individual pixel
697 dataId = self.indexer.makeDataId(indices[i1a[0]],
698 datasetConfig.ref_dataset_name)
699 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
701 # And save the dataset configuration
702 dataId = self.indexer.makeDataId(None, datasetConfig.ref_dataset_name)
703 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
705 self.log.info("Done outputting standard stars.")
707 def _formatCatalog(self, fgcmStarCat, offsets):
708 """
709 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
711 Parameters
712 ----------
713 fgcmStarCat: `lsst.afw.Table.SimpleCatalog`
714 SimpleCatalog as output by fgcmcal
715 offsets: `list` with len(self.bands) entries
716 Zeropoint offsets to apply
718 Returns
719 -------
720 formattedCat: `lsst.afw.table.SimpleCatalog`
721 SimpleCatalog suitable for using as a reference catalog
722 """
724 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
725 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(self.bands,
726 addCentroid=False,
727 addIsResolved=True,
728 coordErrDim=0)
729 sourceMapper.addMinimalSchema(minSchema)
730 for band in self.bands:
731 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
732 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
733 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
735 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
736 formattedCat.reserve(len(fgcmStarCat))
737 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
739 # Note that we don't have to set `resolved` because the default is False
741 for b, band in enumerate(self.bands):
742 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
743 # We want fluxes in nJy from calibrated AB magnitudes
744 # (after applying offset). Updated after RFC-549 and RFC-575.
745 flux = (mag*units.ABmag).to_value(units.nJy)
746 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
748 formattedCat['%s_flux' % (band)][:] = flux
749 formattedCat['%s_fluxErr' % (band)][:] = fluxErr
750 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
751 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
752 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
754 addRefCatMetadata(formattedCat)
756 return formattedCat
758 def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None):
759 """
760 Output the zeropoints in fgcm_photoCalib format.
762 Parameters
763 ----------
764 butler: `lsst.daf.persistence.Butler`
765 zptCat: `lsst.afw.table.BaseCatalog`
766 FGCM zeropoint catalog from `FgcmFitCycleTask`
767 visitCat: `lsst.afw.table.BaseCatalog`
768 FGCM visitCat from `FgcmBuildStarsTask`
769 offsets: `numpy.array`
770 Float array of absolute calibration offsets, one for each filter
771 tract: `int`, optional
772 Tract number to output. Default is None (global calibration)
773 """
775 if tract is None:
776 datasetType = 'fgcm_photoCalib'
777 else:
778 datasetType = 'fgcm_tract_photoCalib'
780 self.log.info("Outputting %s objects" % (datasetType))
782 # Select visit/ccds where we have a calibration
783 # This includes ccds where we were able to interpolate from neighboring
784 # ccds.
785 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
786 too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD']
787 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) &
788 (zptCat['fgcmZptVar'] > 0.0))
790 # We also select the "best" calibrations, avoiding interpolation. These
791 # are only used for mapping filternames
792 selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) &
793 (zptCat['fgcmZptVar'] > 0.0))
795 # Log warnings for any visit which has no valid zeropoints
796 badVisits = np.unique(zptCat['visit'][~selected])
797 goodVisits = np.unique(zptCat['visit'][selected])
798 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
799 for allBadVisit in allBadVisits:
800 self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}')
802 # Get the mapping from filtername to dataId filter name, empirically
803 filterMapping = {}
804 nFound = 0
805 for rec in zptCat[selected_best]:
806 if rec['filtername'] in filterMapping:
807 continue
808 dataId = {self.visitDataRefName: int(rec['visit']),
809 self.ccdDataRefName: int(rec['ccd'])}
810 dataRef = butler.dataRef('raw', dataId=dataId)
811 filterMapping[rec['filtername']] = dataRef.dataId['filter']
812 nFound += 1
813 if nFound == len(self.filterMap):
814 break
816 # Get a mapping from filtername to the offsets
817 offsetMapping = {}
818 for f in self.filterMap:
819 # Not every filter in the map will necesarily have a band.
820 if self.filterMap[f] in self.bands:
821 offsetMapping[f] = offsets[self.bands.index(self.filterMap[f])]
823 # Get a mapping from "ccd" to the ccd index used for the scaling
824 camera = butler.get('camera')
825 ccdMapping = {}
826 for ccdIndex, detector in enumerate(camera):
827 ccdMapping[detector.getId()] = ccdIndex
829 # And a mapping to get the flat-field scaling values
830 scalingMapping = {}
831 for rec in visitCat:
832 scalingMapping[rec['visit']] = rec['scaling']
834 if self.config.doComposeWcsJacobian:
835 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
837 for rec in zptCat[selected]:
839 # Retrieve overall scaling
840 scaling = scalingMapping[rec['visit']][ccdMapping[rec['ccd']]]
842 # The postCalibrationOffset describe any zeropoint offsets
843 # to apply after the fgcm calibration. The first part comes
844 # from the reference catalog match (used in testing). The
845 # second part comes from the mean chromatic correction
846 # (if configured).
847 postCalibrationOffset = offsetMapping[rec['filtername']]
848 if self.config.doApplyMeanChromaticCorrection:
849 postCalibrationOffset += rec['fgcmDeltaChrom']
851 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
852 rec['fgcmfZptChebXyMax'])
853 # Convert from FGCM AB to nJy
854 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
855 rec['fgcmfZptChebXyMax'],
856 offset=postCalibrationOffset,
857 scaling=scaling)
859 if self.config.doComposeWcsJacobian:
861 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['ccd']],
862 fgcmSuperStarField,
863 fgcmZptField])
864 else:
865 # The photoCalib is just the product of the fgcmSuperStarField and the
866 # fgcmZptField
867 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
869 # The "mean" calibration will be set to the center of the ccd for reference
870 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
871 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
872 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
873 calibrationErr=calibErr,
874 calibration=fgcmField,
875 isConstant=False)
877 if tract is None:
878 butler.put(photoCalib, datasetType,
879 dataId={self.visitDataRefName: int(rec['visit']),
880 self.ccdDataRefName: int(rec['ccd']),
881 'filter': filterMapping[rec['filtername']]})
882 else:
883 butler.put(photoCalib, datasetType,
884 dataId={self.visitDataRefName: int(rec['visit']),
885 self.ccdDataRefName: int(rec['ccd']),
886 'filter': filterMapping[rec['filtername']],
887 'tract': tract})
889 self.log.info("Done outputting %s objects" % (datasetType))
891 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
892 """
893 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
894 and scaling.
896 Parameters
897 ----------
898 coefficients: `numpy.array`
899 Flattened array of chebyshev coefficients
900 xyMax: `list` of length 2
901 Maximum x and y of the chebyshev bounding box
902 offset: `float`, optional
903 Absolute calibration offset. Default is 0.0
904 scaling: `float`, optional
905 Flat scaling value from fgcmBuildStars. Default is 1.0
907 Returns
908 -------
909 boundedField: `lsst.afw.math.ChebyshevBoundedField`
910 """
912 orderPlus1 = int(np.sqrt(coefficients.size))
913 pars = np.zeros((orderPlus1, orderPlus1))
915 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
916 lsst.geom.Point2I(*xyMax))
918 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) *
919 (10.**(offset/-2.5))*scaling)
921 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
923 return boundedField
925 def _outputAtmospheres(self, butler, atmCat, tract=None):
926 """
927 Output the atmospheres.
929 Parameters
930 ----------
931 butler: `lsst.daf.persistence.Butler`
932 atmCat: `lsst.afw.table.BaseCatalog`
933 FGCM atmosphere parameter catalog from fgcmFitCycleTask
934 tract: `int`, optional
935 Tract number to output. Default is None (global calibration)
936 """
938 self.log.info("Outputting atmosphere transmissions")
940 # First, we need to grab the look-up table and key info
941 lutCat = butler.get('fgcmLookUpTable')
943 atmosphereTableName = lutCat[0]['tablename']
944 elevation = lutCat[0]['elevation']
945 atmLambda = lutCat[0]['atmLambda']
946 lutCat = None
948 # Make the atmosphere table if possible
949 try:
950 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
951 atmTable.loadTable()
952 except IOError:
953 atmTable = None
955 if atmTable is None:
956 # Try to use MODTRAN instead
957 try:
958 modGen = fgcm.ModtranGenerator(elevation)
959 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
960 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
961 except (ValueError, IOError) as e:
962 raise RuntimeError("FGCM look-up-table generated with modtran, "
963 "but modtran not configured to run.") from e
965 zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
967 for i, visit in enumerate(atmCat['visit']):
968 if atmTable is not None:
969 # Interpolate the atmosphere table
970 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
971 pwv=atmCat[i]['pwv'],
972 o3=atmCat[i]['o3'],
973 tau=atmCat[i]['tau'],
974 alpha=atmCat[i]['alpha'],
975 zenith=zenith[i],
976 ctranslamstd=[atmCat[i]['cTrans'],
977 atmCat[i]['lamStd']])
978 else:
979 # Run modtran
980 modAtm = modGen(pmb=atmCat[i]['pmb'],
981 pwv=atmCat[i]['pwv'],
982 o3=atmCat[i]['o3'],
983 tau=atmCat[i]['tau'],
984 alpha=atmCat[i]['alpha'],
985 zenith=zenith[i],
986 lambdaRange=lambdaRange,
987 lambdaStep=lambdaStep,
988 ctranslamstd=[atmCat[i]['cTrans'],
989 atmCat[i]['lamStd']])
990 atmVals = modAtm['COMBINED']
992 # Now need to create something to persist...
993 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
994 wavelengths=atmLambda,
995 throughputAtMin=atmVals[0],
996 throughputAtMax=atmVals[-1])
998 if tract is None:
999 butler.put(curve, "transmission_atmosphere_fgcm",
1000 dataId={self.visitDataRefName: visit})
1001 else:
1002 butler.put(curve, "transmission_atmosphere_fgcm_tract",
1003 dataId={self.visitDataRefName: visit,
1004 'tract': tract})
1006 self.log.info("Done outputting atmosphere transmissions")