Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Make the final fgcmcal output products.
25This task takes the final output from fgcmFitCycle and produces the following
26outputs for use in the DM stack: the FGCM standard stars in a reference
27catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29task can transfer the 'absolute' calibration from a reference catalog
30to put the fgcm standard stars in units of Jansky. This is accomplished
31by matching stars in a sample of healpix pixels, and applying the median
32offset per band.
33"""
35import sys
36import traceback
37import copy
39import numpy as np
40import healpy as hp
41import esutil
42from astropy import units
44import lsst.pex.config as pexConfig
45import lsst.pipe.base as pipeBase
46from lsst.afw.image import TransmissionCurve
47from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
48from lsst.pipe.tasks.photoCal import PhotoCalTask
49import lsst.geom
50import lsst.afw.image as afwImage
51import lsst.afw.math as afwMath
52import lsst.afw.table as afwTable
53from lsst.meas.algorithms import IndexerRegistry
54from lsst.meas.algorithms import DatasetConfig
55from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
57from .utilities import computeApproxPixelAreaFields
59import fgcm
61__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
64class FgcmOutputProductsConfig(pexConfig.Config):
65 """Config for FgcmOutputProductsTask"""
67 cycleNumber = pexConfig.Field(
68 doc="Final fit cycle from FGCM fit",
69 dtype=int,
70 default=None,
71 )
73 # The following fields refer to calibrating from a reference
74 # catalog, but in the future this might need to be expanded
75 doReferenceCalibration = pexConfig.Field(
76 doc=("Transfer 'absolute' calibration from reference catalog? "
77 "This afterburner step is unnecessary if reference stars "
78 "were used in the full fit in FgcmFitCycleTask."),
79 dtype=bool,
80 default=False,
81 )
82 doRefcatOutput = pexConfig.Field(
83 doc="Output standard stars in reference catalog format",
84 dtype=bool,
85 default=True,
86 )
87 doAtmosphereOutput = pexConfig.Field(
88 doc="Output atmospheres in transmission_atmosphere_fgcm format",
89 dtype=bool,
90 default=True,
91 )
92 doZeropointOutput = pexConfig.Field(
93 doc="Output zeropoints in fgcm_photoCalib format",
94 dtype=bool,
95 default=True,
96 )
97 doComposeWcsJacobian = pexConfig.Field(
98 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
99 dtype=bool,
100 default=True,
101 )
102 refObjLoader = pexConfig.ConfigurableField(
103 target=LoadIndexedReferenceObjectsTask,
104 doc="reference object loader for 'absolute' photometric calibration",
105 )
106 photoCal = pexConfig.ConfigurableField(
107 target=PhotoCalTask,
108 doc="task to perform 'absolute' calibration",
109 )
110 referencePixelizationNside = pexConfig.Field(
111 doc="Healpix nside to pixelize catalog to compare to reference catalog",
112 dtype=int,
113 default=64,
114 )
115 referencePixelizationMinStars = pexConfig.Field(
116 doc=("Minimum number of stars per healpix pixel to select for comparison"
117 "to the specified reference catalog"),
118 dtype=int,
119 default=200,
120 )
121 referenceMinMatch = pexConfig.Field(
122 doc="Minimum number of stars matched to reference catalog to be used in statistics",
123 dtype=int,
124 default=50,
125 )
126 referencePixelizationNPixels = pexConfig.Field(
127 doc=("Number of healpix pixels to sample to do comparison. "
128 "Doing too many will take a long time and not yield any more "
129 "precise results because the final number is the median offset "
130 "(per band) from the set of pixels."),
131 dtype=int,
132 default=100,
133 )
134 datasetConfig = pexConfig.ConfigField(
135 dtype=DatasetConfig,
136 doc="Configuration for writing/reading ingested catalog",
137 )
139 def setDefaults(self):
140 pexConfig.Config.setDefaults(self)
142 # In order to transfer the "absolute" calibration from a reference
143 # catalog to the relatively calibrated FGCM standard stars (one number
144 # per band), we use the PhotoCalTask to match stars in a sample of healpix
145 # pixels. These basic settings ensure that only well-measured, good stars
146 # from the source and reference catalogs are used for the matching.
148 # applyColorTerms needs to be False if doReferenceCalibration is False,
149 # as is the new default after DM-16702
150 self.photoCal.applyColorTerms = False
151 self.photoCal.fluxField = 'instFlux'
152 self.photoCal.magErrFloor = 0.003
153 self.photoCal.match.referenceSelection.doSignalToNoise = True
154 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
155 self.photoCal.match.sourceSelection.doSignalToNoise = True
156 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
157 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
158 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
159 self.photoCal.match.sourceSelection.doFlags = True
160 self.photoCal.match.sourceSelection.flags.good = []
161 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
162 self.photoCal.match.sourceSelection.doUnresolved = False
163 self.datasetConfig.ref_dataset_name = 'fgcm_stars'
164 self.datasetConfig.format_version = 1
167class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
168 """Subclass of TaskRunner for fgcmOutputProductsTask
170 fgcmOutputProductsTask.run() takes one argument, the butler, and
171 does not run on any data in the repository.
172 This runner does not use any parallelization.
173 """
175 @staticmethod
176 def getTargetList(parsedCmd):
177 """
178 Return a list with one element, the butler.
179 """
180 return [parsedCmd.butler]
182 def __call__(self, butler):
183 """
184 Parameters
185 ----------
186 butler: `lsst.daf.persistence.Butler`
188 Returns
189 -------
190 exitStatus: `list` with `pipeBase.Struct`
191 exitStatus (0: success; 1: failure)
192 if self.doReturnResults also
193 results (`np.array` with absolute zeropoint offsets)
194 """
195 task = self.TaskClass(butler=butler, config=self.config, log=self.log)
197 exitStatus = 0
198 if self.doRaise:
199 results = task.runDataRef(butler)
200 else:
201 try:
202 results = task.runDataRef(butler)
203 except Exception as e:
204 exitStatus = 1
205 task.log.fatal("Failed: %s" % e)
206 if not isinstance(e, pipeBase.TaskError):
207 traceback.print_exc(file=sys.stderr)
209 task.writeMetadata(butler)
211 if self.doReturnResults:
212 # The results here are the zeropoint offsets for each band
213 return [pipeBase.Struct(exitStatus=exitStatus,
214 results=results)]
215 else:
216 return [pipeBase.Struct(exitStatus=exitStatus)]
218 def run(self, parsedCmd):
219 """
220 Run the task, with no multiprocessing
222 Parameters
223 ----------
224 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
225 """
227 resultList = []
229 if self.precall(parsedCmd):
230 targetList = self.getTargetList(parsedCmd)
231 # make sure that we only get 1
232 resultList = self(targetList[0])
234 return resultList
237class FgcmOutputProductsTask(pipeBase.CmdLineTask):
238 """
239 Output products from FGCM global calibration.
240 """
242 ConfigClass = FgcmOutputProductsConfig
243 RunnerClass = FgcmOutputProductsRunner
244 _DefaultName = "fgcmOutputProducts"
246 def __init__(self, butler=None, **kwargs):
247 """
248 Instantiate an fgcmOutputProductsTask.
250 Parameters
251 ----------
252 butler : `lsst.daf.persistence.Butler`
253 """
255 pipeBase.CmdLineTask.__init__(self, **kwargs)
257 if self.config.doReferenceCalibration:
258 # We need the ref obj loader to get the flux field
259 self.makeSubtask("refObjLoader", butler=butler)
261 if self.config.doRefcatOutput:
262 self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
263 self.config.datasetConfig.indexer.active)
265 # no saving of metadata for now
266 def _getMetadataName(self):
267 return None
269 @pipeBase.timeMethod
270 def runDataRef(self, butler):
271 """
272 Make FGCM output products for use in the stack
274 Parameters
275 ----------
276 butler: `lsst.daf.persistence.Butler`
277 cycleNumber: `int`
278 Final fit cycle number, override config.
280 Returns
281 -------
282 offsets: `lsst.pipe.base.Struct`
283 A structure with array of zeropoint offsets
285 Raises
286 ------
287 RuntimeError: Raised if butler cannot find fgcmBuildStars_config, or
288 fgcmFitCycle_config, or fgcmAtmosphereParameters (and
289 `self.config.doAtmosphereOutput` is true), or fgcmStandardStars (and
290 `self.config.doReferenceCalibration or `self.config.doRefcatOutput`
291 is true), or fgcmZeropoints (and self.config.doZeropointOutput is true).
292 Also will raise if the fgcmFitCycle_config does not refer to the
293 final fit cycle.
294 """
296 # Check to make sure that the fgcmBuildStars config exists, to retrieve
297 # the visit and ccd dataset tags
298 if not butler.datasetExists('fgcmBuildStars_config'):
299 raise RuntimeError("Cannot find fgcmBuildStars_config, which is prereq for fgcmOutputProducts")
301 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
302 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
303 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
304 self.filterMap = fgcmBuildStarsConfig.filterMap
306 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
307 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
308 "in fgcmBuildStarsTask.")
310 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
311 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
313 # Make sure that the fit config exists, to retrieve bands and other info
314 if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber):
315 raise RuntimeError("Cannot find fgcmFitCycle_config from cycle %d " % (self.config.cycleNumber) +
316 "which is required for fgcmOutputProducts.")
318 fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber)
319 self.configBands = fitCycleConfig.bands
321 if self.config.doReferenceCalibration and fitCycleConfig.doReferenceCalibration:
322 self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
323 "fitCycleConfig.doReferenceCalibration")
325 # And make sure that the atmosphere was output properly
326 if (self.config.doAtmosphereOutput and
327 not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
328 raise RuntimeError("Atmosphere parameters are missing for cycle %d." %
329 (self.config.cycleNumber))
331 if ((self.config.doReferenceCalibration or self.config.doRefcatOutput) and
332 (not butler.datasetExists('fgcmStandardStars',
333 fgcmcycle=self.config.cycleNumber))):
334 raise RuntimeError("Standard stars are missing for cycle %d." %
335 (self.config.cycleNumber))
337 if (self.config.doZeropointOutput and
338 (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
339 raise RuntimeError("Zeropoints are missing for cycle %d." %
340 (self.config.cycleNumber))
342 # And make sure this is the last cycle
343 if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1):
344 raise RuntimeError("The task fgcmOutputProducts should only be run"
345 "on the final fit cycle products")
347 if self.config.doReferenceCalibration or self.config.doRefcatOutput:
348 stdCat = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
349 md = stdCat.getMetadata()
350 self.bands = md.getArray('BANDS')
351 else:
352 stdCat = None
353 self.bands = self.configBands
355 if self.config.doReferenceCalibration:
356 offsets = self._computeReferenceOffsets(butler, stdCat)
357 else:
358 offsets = np.zeros(len(self.bands))
360 # Output the standard stars in stack format
361 if self.config.doRefcatOutput:
362 self._outputStandardStars(butler, stdCat, offsets, self.config.datasetConfig)
364 del stdCat
366 # Output the gray zeropoints
367 if self.config.doZeropointOutput:
368 zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
369 visitCat = butler.get('fgcmVisitCatalog')
371 self._outputZeropoints(butler, zptCat, visitCat, offsets)
373 # Output the atmospheres
374 if self.config.doAtmosphereOutput:
375 atmCat = butler.get('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
376 self._outputAtmospheres(butler, atmCat)
378 # We return the zp offsets
379 return pipeBase.Struct(offsets=offsets)
381 def generateTractOutputProducts(self, butler, tract,
382 visitCat, zptCat, atmCat, stdCat,
383 fgcmBuildStarsConfig, fgcmFitCycleConfig):
384 """
385 Generate the output products for a given tract, as specified in the config.
387 This method is here to have an alternate entry-point for
388 FgcmCalibrateTract.
390 Parameters
391 ----------
392 butler: `lsst.daf.persistence.Butler`
393 tract: `int`
394 Tract number
395 visitCat: `lsst.afw.table.BaseCatalog`
396 FGCM visitCat from `FgcmBuildStarsTask`
397 zptCat: `lsst.afw.table.BaseCatalog`
398 FGCM zeropoint catalog from `FgcmFitCycleTask`
399 atmCat: `lsst.afw.table.BaseCatalog`
400 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
401 stdCat: `lsst.afw.table.SimpleCatalog`
402 FGCM standard star catalog from `FgcmFitCycleTask`
403 fgcmBuildStarsConfig: `lsst.fgcmcal.FgcmBuildStarsConfig`
404 Configuration object from `FgcmBuildStarsTask`
405 fgcmFitCycleConfig: `lsst.fgcmcal.FgcmFitCycleConfig`
406 Configuration object from `FgcmFitCycleTask`
407 """
409 self.configBands = fgcmFitCycleConfig.bands
410 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
411 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
412 self.filterMap = fgcmBuildStarsConfig.filterMap
414 if stdCat is not None:
415 md = stdCat.getMetadata()
416 self.bands = md.getArray('BANDS')
417 else:
418 self.bands = self.configBands
420 if self.config.doReferenceCalibration and fgcmFitCycleConfig.doReferenceCalibration:
421 self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
422 "fitCycleConfig.doReferenceCalibration")
424 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
425 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
426 "in fgcmBuildStarsTask.")
428 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
429 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
431 if self.config.doReferenceCalibration:
432 offsets = self._computeReferenceOffsets(butler, stdCat)
433 else:
434 offsets = np.zeros(len(self.bands))
436 if self.config.doRefcatOutput:
437 # Create a special config that has the tract number in it
438 datasetConfig = copy.copy(self.config.datasetConfig)
439 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
440 tract)
441 self._outputStandardStars(butler, stdCat, offsets, datasetConfig)
443 if self.config.doZeropointOutput:
444 self._outputZeropoints(butler, zptCat, visitCat, offsets, tract=tract)
446 if self.config.doAtmosphereOutput:
447 self._outputAtmospheres(butler, atmCat, tract=tract)
449 return pipeBase.Struct(offsets=offsets)
451 def _computeReferenceOffsets(self, butler, stdCat):
452 """
453 Compute offsets relative to a reference catalog.
455 This method splits the star catalog into healpix pixels
456 and computes the calibration transfer for a sample of
457 these pixels to approximate the 'absolute' calibration
458 values (on for each band) to apply to transfer the
459 absolute scale.
461 Parameters
462 ----------
463 butler: `lsst.daf.persistence.Butler`
464 stdCat: `lsst.afw.table.SimpleCatalog`
465 FGCM standard stars
467 Returns
468 -------
469 offsets: `numpy.array` of floats
470 Per band zeropoint offsets
471 """
473 # Only use stars that are observed in all the bands that were actually used
474 # This will ensure that we use the same healpix pixels for the absolute
475 # calibration of each band.
476 minObs = stdCat['ngood'].min(axis=1)
478 goodStars = (minObs >= 1)
479 stdCat = stdCat[goodStars]
481 self.log.info("Found %d stars with at least 1 good observation in each band" %
482 (len(stdCat)))
484 # We have to make a table for each pixel with flux/fluxErr
485 # This is a temporary table generated for input to the photoCal task.
486 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
487 # have had chromatic corrections applied to get to the standard system
488 # specified by the atmosphere/instrumental parameters), nor are they
489 # in Jansky (since they don't have a proper absolute calibration: the overall
490 # zeropoint is estimated from the telescope size, etc.)
491 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
492 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
493 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
494 doc="instrumental flux (counts)")
495 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
496 doc="instrumental flux error (counts)")
497 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
498 type='Flag',
499 doc="bad flag")
501 # Split up the stars
502 # Note that there is an assumption here that the ra/dec coords stored
503 # on-disk are in radians, and therefore that starObs['coord_ra'] /
504 # starObs['coord_dec'] return radians when used as an array of numpy float64s.
505 theta = np.pi/2. - stdCat['coord_dec']
506 phi = stdCat['coord_ra']
508 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
509 h, rev = esutil.stat.histogram(ipring, rev=True)
511 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
513 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
514 (gdpix.size,
515 self.config.referencePixelizationNside,
516 self.config.referencePixelizationMinStars))
518 if gdpix.size < self.config.referencePixelizationNPixels:
519 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
520 (gdpix.size, self.config.referencePixelizationNPixels))
521 else:
522 # Sample out the pixels we want to use
523 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
525 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
526 ('nstar', 'i4', len(self.bands)),
527 ('nmatch', 'i4', len(self.bands)),
528 ('zp', 'f4', len(self.bands)),
529 ('zpErr', 'f4', len(self.bands))])
530 results['hpix'] = ipring[rev[rev[gdpix]]]
532 # We need a boolean index to deal with catalogs...
533 selected = np.zeros(len(stdCat), dtype=np.bool)
535 refFluxFields = [None]*len(self.bands)
537 for p, pix in enumerate(gdpix):
538 i1a = rev[rev[pix]: rev[pix + 1]]
540 # the stdCat afwTable can only be indexed with boolean arrays,
541 # and not numpy index arrays (see DM-16497). This little trick
542 # converts the index array into a boolean array
543 selected[:] = False
544 selected[i1a] = True
546 for b, band in enumerate(self.bands):
548 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
549 selected, refFluxFields)
550 results['nstar'][p, b] = len(i1a)
551 results['nmatch'][p, b] = len(struct.arrays.refMag)
552 results['zp'][p, b] = struct.zp
553 results['zpErr'][p, b] = struct.sigma
555 # And compute the summary statistics
556 offsets = np.zeros(len(self.bands))
558 for b, band in enumerate(self.bands):
559 # make configurable
560 ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)
561 offsets[b] = np.median(results['zp'][ok, b])
562 # use median absolute deviation to estimate Normal sigma
563 # see https://en.wikipedia.org/wiki/Median_absolute_deviation
564 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b]))
565 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" %
566 (band, offsets[b], madSigma))
568 return offsets
570 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
571 b, band, stdCat, selected, refFluxFields):
572 """
573 Compute the zeropoint offset between the fgcm stdCat and the reference
574 stars for one pixel in one band
576 Parameters
577 ----------
578 sourceMapper: `lsst.afw.table.SchemaMapper`
579 Mapper to go from stdCat to calibratable catalog
580 badStarKey: `lsst.afw.table.Key`
581 Key for the field with bad stars
582 b: `int`
583 Index of the band in the star catalog
584 band: `str`
585 Name of band for reference catalog
586 stdCat: `lsst.afw.table.SimpleCatalog`
587 FGCM standard stars
588 selected: `numpy.array(dtype=np.bool)`
589 Boolean array of which stars are in the pixel
590 refFluxFields: `list`
591 List of names of flux fields for reference catalog
592 """
594 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
595 sourceCat.reserve(selected.sum())
596 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
597 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5))
598 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] *
599 sourceCat['instFlux'])
600 # Make sure we only use stars that have valid measurements
601 # (This is perhaps redundant with requirements above that the
602 # stars be observed in all bands, but it can't hurt)
603 badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0)
604 for rec in sourceCat[badStar]:
605 rec.set(badStarKey, True)
607 exposure = afwImage.ExposureF()
608 exposure.setFilter(afwImage.Filter(band))
610 if refFluxFields[b] is None:
611 # Need to find the flux field in the reference catalog
612 # to work around limitations of DirectMatch in PhotoCal
613 ctr = stdCat[0].getCoord()
614 rad = 0.05*lsst.geom.degrees
615 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
616 refFluxFields[b] = refDataTest.fluxField
618 # Make a copy of the config so that we can modify it
619 calConfig = copy.copy(self.config.photoCal.value)
620 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
621 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'
622 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
623 config=calConfig,
624 schema=sourceCat.getSchema())
626 struct = calTask.run(exposure, sourceCat)
628 return struct
630 def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig):
631 """
632 Output standard stars in indexed reference catalog format.
634 Parameters
635 ----------
636 butler: `lsst.daf.persistence.Butler`
637 stdCat: `lsst.afw.table.SimpleCatalog`
638 FGCM standard star catalog from fgcmFitCycleTask
639 offsets: `numpy.array` of floats
640 Per band zeropoint offsets
641 datasetConfig: `lsst.meas.algorithms.DatasetConfig`
642 Config for reference dataset
643 """
645 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
647 # We determine the conversion from the native units (typically radians) to
648 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
649 # numpy arrays rather than Angles, which would we approximately 600x slower.
650 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
651 # (as Angles) for input
652 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
653 indices = np.array(self.indexer.indexPoints(stdCat['coord_ra']*conv,
654 stdCat['coord_dec']*conv))
656 formattedCat = self._formatCatalog(stdCat, offsets)
658 # Write the master schema
659 dataId = self.indexer.makeDataId('master_schema',
660 datasetConfig.ref_dataset_name)
661 masterCat = afwTable.SimpleCatalog(formattedCat.schema)
662 addRefCatMetadata(masterCat)
663 butler.put(masterCat, 'ref_cat', dataId=dataId)
665 # Break up the pixels using a histogram
666 h, rev = esutil.stat.histogram(indices, rev=True)
667 gd, = np.where(h > 0)
668 selected = np.zeros(len(formattedCat), dtype=np.bool)
669 for i in gd:
670 i1a = rev[rev[i]: rev[i + 1]]
672 # the formattedCat afwTable can only be indexed with boolean arrays,
673 # and not numpy index arrays (see DM-16497). This little trick
674 # converts the index array into a boolean array
675 selected[:] = False
676 selected[i1a] = True
678 # Write the individual pixel
679 dataId = self.indexer.makeDataId(indices[i1a[0]],
680 datasetConfig.ref_dataset_name)
681 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
683 # And save the dataset configuration
684 dataId = self.indexer.makeDataId(None, datasetConfig.ref_dataset_name)
685 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
687 self.log.info("Done outputting standard stars.")
689 def _formatCatalog(self, fgcmStarCat, offsets):
690 """
691 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
693 Parameters
694 ----------
695 fgcmStarCat: `lsst.afw.Table.SimpleCatalog`
696 SimpleCatalog as output by fgcmcal
697 offsets: `list` with len(self.bands) entries
698 Zeropoint offsets to apply
700 Returns
701 -------
702 formattedCat: `lsst.afw.table.SimpleCatalog`
703 SimpleCatalog suitable for using as a reference catalog
704 """
706 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
707 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(self.bands,
708 addCentroid=False,
709 addIsResolved=True,
710 coordErrDim=0)
711 sourceMapper.addMinimalSchema(minSchema)
712 for band in self.bands:
713 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
714 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
715 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
717 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
718 formattedCat.reserve(len(fgcmStarCat))
719 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
721 # Note that we don't have to set `resolved` because the default is False
723 for b, band in enumerate(self.bands):
724 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
725 # We want fluxes in nJy from calibrated AB magnitudes
726 # (after applying offset). Updated after RFC-549 and RFC-575.
727 flux = (mag*units.ABmag).to_value(units.nJy)
728 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
730 formattedCat['%s_flux' % (band)][:] = flux
731 formattedCat['%s_fluxErr' % (band)][:] = fluxErr
732 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
733 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
734 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
736 addRefCatMetadata(formattedCat)
738 return formattedCat
740 def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None):
741 """
742 Output the zeropoints in fgcm_photoCalib format.
744 Parameters
745 ----------
746 butler: `lsst.daf.persistence.Butler`
747 zptCat: `lsst.afw.table.BaseCatalog`
748 FGCM zeropoint catalog from `FgcmFitCycleTask`
749 visitCat: `lsst.afw.table.BaseCatalog`
750 FGCM visitCat from `FgcmBuildStarsTask`
751 offsets: `numpy.array`
752 Float array of absolute calibration offsets, one for each filter
753 tract: `int`, optional
754 Tract number to output. Default is None (global calibration)
755 """
757 if tract is None:
758 datasetType = 'fgcm_photoCalib'
759 else:
760 datasetType = 'fgcm_tract_photoCalib'
762 self.log.info("Outputting %s objects" % (datasetType))
764 # Select visit/ccds where we have a calibration
765 # This includes ccds where we were able to interpolate from neighboring
766 # ccds.
767 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
768 too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD']
769 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) &
770 (zptCat['fgcmZptVar'] > 0.0))
772 # We also select the "best" calibrations, avoiding interpolation. These
773 # are only used for mapping filternames
774 selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) &
775 (zptCat['fgcmZptVar'] > 0.0))
777 # Log warnings for any visit which has no valid zeropoints
778 badVisits = np.unique(zptCat['visit'][~selected])
779 goodVisits = np.unique(zptCat['visit'][selected])
780 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
781 for allBadVisit in allBadVisits:
782 self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}')
784 # Get the mapping from filtername to dataId filter name, empirically
785 filterMapping = {}
786 nFound = 0
787 for rec in zptCat[selected_best]:
788 if rec['filtername'] in filterMapping:
789 continue
790 dataId = {self.visitDataRefName: int(rec['visit']),
791 self.ccdDataRefName: int(rec['ccd'])}
792 dataRef = butler.dataRef('raw', dataId=dataId)
793 filterMapping[rec['filtername']] = dataRef.dataId['filter']
794 nFound += 1
795 if nFound == len(self.filterMap):
796 break
798 # Get a mapping from filtername to the offsets
799 offsetMapping = {}
800 for f in self.filterMap:
801 # Not every filter in the map will necesarily have a band.
802 if self.filterMap[f] in self.bands:
803 offsetMapping[f] = offsets[self.bands.index(self.filterMap[f])]
805 # Get a mapping from "ccd" to the ccd index used for the scaling
806 camera = butler.get('camera')
807 ccdMapping = {}
808 for ccdIndex, detector in enumerate(camera):
809 ccdMapping[detector.getId()] = ccdIndex
811 # And a mapping to get the flat-field scaling values
812 scalingMapping = {}
813 for rec in visitCat:
814 scalingMapping[rec['visit']] = rec['scaling']
816 if self.config.doComposeWcsJacobian:
817 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
819 for rec in zptCat[selected]:
821 # Retrieve overall scaling
822 scaling = scalingMapping[rec['visit']][ccdMapping[rec['ccd']]]
824 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
825 rec['fgcmfZptChebXyMax'])
826 # Convert from FGCM AB to nJy
827 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
828 rec['fgcmfZptChebXyMax'],
829 offset=offsetMapping[rec['filtername']],
830 scaling=scaling)
832 if self.config.doComposeWcsJacobian:
834 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['ccd']],
835 fgcmSuperStarField,
836 fgcmZptField])
837 else:
838 # The photoCalib is just the product of the fgcmSuperStarField and the
839 # fgcmZptField
840 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
842 # The "mean" calibration will be set to the center of the ccd for reference
843 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
844 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
845 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
846 calibrationErr=calibErr,
847 calibration=fgcmField,
848 isConstant=False)
850 if tract is None:
851 butler.put(photoCalib, datasetType,
852 dataId={self.visitDataRefName: int(rec['visit']),
853 self.ccdDataRefName: int(rec['ccd']),
854 'filter': filterMapping[rec['filtername']]})
855 else:
856 butler.put(photoCalib, datasetType,
857 dataId={self.visitDataRefName: int(rec['visit']),
858 self.ccdDataRefName: int(rec['ccd']),
859 'filter': filterMapping[rec['filtername']],
860 'tract': tract})
862 self.log.info("Done outputting %s objects" % (datasetType))
864 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
865 """
866 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
867 and scaling.
869 Parameters
870 ----------
871 coefficients: `numpy.array`
872 Flattened array of chebyshev coefficients
873 xyMax: `list` of length 2
874 Maximum x and y of the chebyshev bounding box
875 offset: `float`, optional
876 Absolute calibration offset. Default is 0.0
877 scaling: `float`, optional
878 Flat scaling value from fgcmBuildStars. Default is 1.0
880 Returns
881 -------
882 boundedField: `lsst.afw.math.ChebyshevBoundedField`
883 """
885 orderPlus1 = int(np.sqrt(coefficients.size))
886 pars = np.zeros((orderPlus1, orderPlus1))
888 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
889 lsst.geom.Point2I(*xyMax))
891 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) *
892 (10.**(offset/-2.5))*scaling)
894 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
896 return boundedField
898 def _outputAtmospheres(self, butler, atmCat, tract=None):
899 """
900 Output the atmospheres.
902 Parameters
903 ----------
904 butler: `lsst.daf.persistence.Butler`
905 atmCat: `lsst.afw.table.BaseCatalog`
906 FGCM atmosphere parameter catalog from fgcmFitCycleTask
907 tract: `int`, optional
908 Tract number to output. Default is None (global calibration)
909 """
911 self.log.info("Outputting atmosphere transmissions")
913 # First, we need to grab the look-up table and key info
914 lutCat = butler.get('fgcmLookUpTable')
916 atmosphereTableName = lutCat[0]['tablename']
917 elevation = lutCat[0]['elevation']
918 atmLambda = lutCat[0]['atmLambda']
919 lutCat = None
921 # Make the atmosphere table if possible
922 try:
923 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
924 atmTable.loadTable()
925 except IOError:
926 atmTable = None
928 if atmTable is None:
929 # Try to use MODTRAN instead
930 try:
931 modGen = fgcm.ModtranGenerator(elevation)
932 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
933 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
934 except (ValueError, IOError) as e:
935 raise RuntimeError("FGCM look-up-table generated with modtran, "
936 "but modtran not configured to run.") from e
938 zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
940 for i, visit in enumerate(atmCat['visit']):
941 if atmTable is not None:
942 # Interpolate the atmosphere table
943 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
944 pwv=atmCat[i]['pwv'],
945 o3=atmCat[i]['o3'],
946 tau=atmCat[i]['tau'],
947 alpha=atmCat[i]['alpha'],
948 zenith=zenith[i],
949 ctranslamstd=[atmCat[i]['cTrans'],
950 atmCat[i]['lamStd']])
951 else:
952 # Run modtran
953 modAtm = modGen(pmb=atmCat[i]['pmb'],
954 pwv=atmCat[i]['pwv'],
955 o3=atmCat[i]['o3'],
956 tau=atmCat[i]['tau'],
957 alpha=atmCat[i]['alpha'],
958 zenith=zenith[i],
959 lambdaRange=lambdaRange,
960 lambdaStep=lambdaStep,
961 ctranslamstd=[atmCat[i]['cTrans'],
962 atmCat[i]['lamStd']])
963 atmVals = modAtm['COMBINED']
965 # Now need to create something to persist...
966 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
967 wavelengths=atmLambda,
968 throughputAtMin=atmVals[0],
969 throughputAtMax=atmVals[-1])
971 if tract is None:
972 butler.put(curve, "transmission_atmosphere_fgcm",
973 dataId={self.visitDataRefName: visit})
974 else:
975 butler.put(curve, "transmission_atmosphere_fgcm_tract",
976 dataId={self.visitDataRefName: visit,
977 'tract': tract})
979 self.log.info("Done outputting atmosphere transmissions")