lsst.fgcmcal gd8e65d7270+fd3e758935
fgcmOutputProducts.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Make the final fgcmcal output products.
24
25This task takes the final output from fgcmFitCycle and produces the following
26outputs for use in the DM stack: the FGCM standard stars in a reference
27catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29task can transfer the 'absolute' calibration from a reference catalog
30to put the fgcm standard stars in units of Jansky. This is accomplished
31by matching stars in a sample of healpix pixels, and applying the median
32offset per band.
33"""
34import copy
35
36import numpy as np
37import hpgeom as hpg
38import esutil
39from astropy import units
40
41import lsst.daf.base as dafBase
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44from lsst.pipe.base import connectionTypes
45from lsst.afw.image import TransmissionCurve
46from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
47from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
48from lsst.pipe.tasks.photoCal import PhotoCalTask
49import lsst.geom
50import lsst.afw.image as afwImage
51import lsst.afw.math as afwMath
52import lsst.afw.table as afwTable
53from lsst.meas.algorithms import DatasetConfig
54from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
55
56from .utilities import computeApproxPixelAreaFields
57from .utilities import lookupStaticCalibrations
58from .utilities import FGCM_ILLEGAL_VALUE
59
60import fgcm
61
62__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask']
63
64
65class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
66 dimensions=("instrument",),
67 defaultTemplates={"cycleNumber": "0"}):
68 camera = connectionTypes.PrerequisiteInput(
69 doc="Camera instrument",
70 name="camera",
71 storageClass="Camera",
72 dimensions=("instrument",),
73 lookupFunction=lookupStaticCalibrations,
74 isCalibration=True,
75 )
76
77 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
78 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
79 "chromatic corrections."),
80 name="fgcmLookUpTable",
81 storageClass="Catalog",
82 dimensions=("instrument",),
83 deferLoad=True,
84 )
85
86 fgcmVisitCatalog = connectionTypes.Input(
87 doc="Catalog of visit information for fgcm",
88 name="fgcmVisitCatalog",
89 storageClass="Catalog",
90 dimensions=("instrument",),
91 deferLoad=True,
92 )
93
94 fgcmStandardStars = connectionTypes.Input(
95 doc="Catalog of standard star data from fgcm fit",
96 name="fgcmStandardStars{cycleNumber}",
97 storageClass="SimpleCatalog",
98 dimensions=("instrument",),
99 deferLoad=True,
100 )
101
102 fgcmZeropoints = connectionTypes.Input(
103 doc="Catalog of zeropoints from fgcm fit",
104 name="fgcmZeropoints{cycleNumber}",
105 storageClass="Catalog",
106 dimensions=("instrument",),
107 deferLoad=True,
108 )
109
110 fgcmAtmosphereParameters = connectionTypes.Input(
111 doc="Catalog of atmosphere parameters from fgcm fit",
112 name="fgcmAtmosphereParameters{cycleNumber}",
113 storageClass="Catalog",
114 dimensions=("instrument",),
115 deferLoad=True,
116 )
117
118 refCat = connectionTypes.PrerequisiteInput(
119 doc="Reference catalog to use for photometric calibration",
120 name="cal_ref_cat",
121 storageClass="SimpleCatalog",
122 dimensions=("skypix",),
123 deferLoad=True,
124 multiple=True,
125 )
126
127 fgcmPhotoCalib = connectionTypes.Output(
128 doc=("Per-visit photometric calibrations derived from fgcm calibration. "
129 "These catalogs use detector id for the id and are sorted for "
130 "fast lookups of a detector."),
131 name="fgcmPhotoCalibCatalog",
132 storageClass="ExposureCatalog",
133 dimensions=("instrument", "visit",),
134 multiple=True,
135 )
136
137 fgcmTransmissionAtmosphere = connectionTypes.Output(
138 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
139 name="transmission_atmosphere_fgcm",
140 storageClass="TransmissionCurve",
141 dimensions=("instrument",
142 "visit",),
143 multiple=True,
144 )
145
146 fgcmOffsets = connectionTypes.Output(
147 doc="Per-band offsets computed from doReferenceCalibration",
148 name="fgcmReferenceCalibrationOffsets",
149 storageClass="Catalog",
150 dimensions=("instrument",),
151 multiple=False,
152 )
153
154 def __init__(self, *, config=None):
155 super().__init__(config=config)
156
157 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
158 raise ValueError("cycleNumber must be of integer format")
159
160 if not config.doReferenceCalibration:
161 self.prerequisiteInputs.remove("refCat")
162 if not config.doAtmosphereOutput:
163 self.inputs.remove("fgcmAtmosphereParameters")
164 if not config.doZeropointOutput:
165 self.inputs.remove("fgcmZeropoints")
166 if not config.doReferenceCalibration:
167 self.outputs.remove("fgcmOffsets")
168
169
170class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
171 pipelineConnections=FgcmOutputProductsConnections):
172 """Config for FgcmOutputProductsTask"""
173
174 cycleNumber = pexConfig.Field(
175 doc="Final fit cycle from FGCM fit",
176 dtype=int,
177 default=0,
178 deprecated=("This config is no longer used, and will be removed after v25. "
179 "Please set config.connections.cycleNumber directly instead."),
180 )
181 physicalFilterMap = pexConfig.DictField(
182 doc="Mapping from 'physicalFilter' to band.",
183 keytype=str,
184 itemtype=str,
185 default={},
186 )
187 # The following fields refer to calibrating from a reference
188 # catalog, but in the future this might need to be expanded
189 doReferenceCalibration = pexConfig.Field(
190 doc=("Transfer 'absolute' calibration from reference catalog? "
191 "This afterburner step is unnecessary if reference stars "
192 "were used in the full fit in FgcmFitCycleTask."),
193 dtype=bool,
194 default=False,
195 )
196 doRefcatOutput = pexConfig.Field(
197 doc="Output standard stars in reference catalog format",
198 dtype=bool,
199 default=False,
200 deprecated="doRefcatOutput is no longer supported; this config will be removed after v24"
201 )
202 doAtmosphereOutput = pexConfig.Field(
203 doc="Output atmospheres in transmission_atmosphere_fgcm format",
204 dtype=bool,
205 default=True,
206 )
207 doZeropointOutput = pexConfig.Field(
208 doc="Output zeropoints in fgcm_photoCalib format",
209 dtype=bool,
210 default=True,
211 )
212 doComposeWcsJacobian = pexConfig.Field(
213 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
214 dtype=bool,
215 default=True,
216 )
217 doApplyMeanChromaticCorrection = pexConfig.Field(
218 doc="Apply the mean chromatic correction to the zeropoints?",
219 dtype=bool,
220 default=True,
221 )
222 refObjLoader = pexConfig.ConfigurableField(
223 target=LoadIndexedReferenceObjectsTask,
224 doc="reference object loader for 'absolute' photometric calibration",
225 deprecated="refObjLoader is deprecated, and will be removed after v24",
226 )
227 photoCal = pexConfig.ConfigurableField(
228 target=PhotoCalTask,
229 doc="task to perform 'absolute' calibration",
230 )
231 referencePixelizationNside = pexConfig.Field(
232 doc="Healpix nside to pixelize catalog to compare to reference catalog",
233 dtype=int,
234 default=64,
235 )
236 referencePixelizationMinStars = pexConfig.Field(
237 doc=("Minimum number of stars per healpix pixel to select for comparison"
238 "to the specified reference catalog"),
239 dtype=int,
240 default=200,
241 )
242 referenceMinMatch = pexConfig.Field(
243 doc="Minimum number of stars matched to reference catalog to be used in statistics",
244 dtype=int,
245 default=50,
246 )
247 referencePixelizationNPixels = pexConfig.Field(
248 doc=("Number of healpix pixels to sample to do comparison. "
249 "Doing too many will take a long time and not yield any more "
250 "precise results because the final number is the median offset "
251 "(per band) from the set of pixels."),
252 dtype=int,
253 default=100,
254 )
255 datasetConfig = pexConfig.ConfigField(
256 dtype=DatasetConfig,
257 doc="Configuration for writing/reading ingested catalog",
258 deprecated="The datasetConfig was only used for gen2; this config will be removed after v24.",
259 )
260
261 def setDefaults(self):
262 pexConfig.Config.setDefaults(self)
263
264 # In order to transfer the "absolute" calibration from a reference
265 # catalog to the relatively calibrated FGCM standard stars (one number
266 # per band), we use the PhotoCalTask to match stars in a sample of healpix
267 # pixels. These basic settings ensure that only well-measured, good stars
268 # from the source and reference catalogs are used for the matching.
269
270 # applyColorTerms needs to be False if doReferenceCalibration is False,
271 # as is the new default after DM-16702
272 self.photoCal.applyColorTerms = False
273 self.photoCal.fluxField = 'instFlux'
274 self.photoCal.magErrFloor = 0.003
275 self.photoCal.match.referenceSelection.doSignalToNoise = True
276 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
277 self.photoCal.match.sourceSelection.doSignalToNoise = True
278 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
279 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
280 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
281 self.photoCal.match.sourceSelection.doFlags = True
282 self.photoCal.match.sourceSelection.flags.good = []
283 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
284 self.photoCal.match.sourceSelection.doUnresolved = False
285
286
287class FgcmOutputProductsTask(pipeBase.PipelineTask):
288 """
289 Output products from FGCM global calibration.
290 """
291
292 ConfigClass = FgcmOutputProductsConfig
293 _DefaultName = "fgcmOutputProducts"
294
295 def __init__(self, **kwargs):
296 super().__init__(**kwargs)
297
298 def runQuantum(self, butlerQC, inputRefs, outputRefs):
299 handleDict = {}
300 handleDict['camera'] = butlerQC.get(inputRefs.camera)
301 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
302 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
303 handleDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
304
305 if self.config.doZeropointOutput:
306 handleDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
307 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
308 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
309
310 if self.config.doAtmosphereOutput:
311 handleDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
312 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
313 atmRef in outputRefs.fgcmTransmissionAtmosphere}
314
315 if self.config.doReferenceCalibration:
316 refConfig = LoadReferenceObjectsConfig()
317 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
318 for ref in inputRefs.refCat],
319 refCats=butlerQC.get(inputRefs.refCat),
320 log=self.log,
321 config=refConfig)
322 else:
323 self.refObjLoader = None
324
325 struct = self.run(handleDict, self.config.physicalFilterMap)
326
327 # Output the photoCalib exposure catalogs
328 if struct.photoCalibCatalogs is not None:
329 self.log.info("Outputting photoCalib catalogs.")
330 for visit, expCatalog in struct.photoCalibCatalogs:
331 butlerQC.put(expCatalog, photoCalibRefDict[visit])
332 self.log.info("Done outputting photoCalib catalogs.")
333
334 # Output the atmospheres
335 if struct.atmospheres is not None:
336 self.log.info("Outputting atmosphere transmission files.")
337 for visit, atm in struct.atmospheres:
338 butlerQC.put(atm, atmRefDict[visit])
339 self.log.info("Done outputting atmosphere files.")
340
341 if self.config.doReferenceCalibration:
342 # Turn offset into simple catalog for persistence if necessary
343 schema = afwTable.Schema()
344 schema.addField('offset', type=np.float64,
345 doc="Post-process calibration offset (mag)")
346 offsetCat = afwTable.BaseCatalog(schema)
347 offsetCat.resize(len(struct.offsets))
348 offsetCat['offset'][:] = struct.offsets
349
350 butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
351
352 return
353
354 def run(self, handleDict, physicalFilterMap):
355 """Run the output products task.
356
357 Parameters
358 ----------
359 handleDict : `dict`
360 All handles are `lsst.daf.butler.DeferredDatasetHandle`
361 handle dictionary with keys:
362
363 ``"camera"``
364 Camera object (`lsst.afw.cameraGeom.Camera`)
365 ``"fgcmLookUpTable"``
366 handle for the FGCM look-up table.
367 ``"fgcmVisitCatalog"``
368 handle for visit summary catalog.
369 ``"fgcmStandardStars"``
370 handle for the output standard star catalog.
371 ``"fgcmZeropoints"``
372 handle for the zeropoint data catalog.
373 ``"fgcmAtmosphereParameters"``
374 handle for the atmosphere parameter catalog.
375 ``"fgcmBuildStarsTableConfig"``
376 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
377 physicalFilterMap : `dict`
378 Dictionary of mappings from physical filter to FGCM band.
379
380 Returns
381 -------
382 retStruct : `lsst.pipe.base.Struct`
383 Output structure with keys:
384
385 offsets : `np.ndarray`
386 Final reference offsets, per band.
387 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
388 Generator that returns (visit, transmissionCurve) tuples.
389 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
390 Generator that returns (visit, exposureCatalog) tuples.
391 """
392 stdCat = handleDict['fgcmStandardStars'].get()
393 md = stdCat.getMetadata()
394 bands = md.getArray('BANDS')
395
396 if self.config.doReferenceCalibration:
397 lutCat = handleDict['fgcmLookUpTable'].get()
398 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands)
399 else:
400 offsets = np.zeros(len(bands))
401
402 del stdCat
403
404 if self.config.doZeropointOutput:
405 zptCat = handleDict['fgcmZeropoints'].get()
406 visitCat = handleDict['fgcmVisitCatalog'].get()
407
408 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands,
409 physicalFilterMap)
410 else:
411 pcgen = None
412
413 if self.config.doAtmosphereOutput:
414 atmCat = handleDict['fgcmAtmosphereParameters'].get()
415 atmgen = self._outputAtmospheres(handleDict, atmCat)
416 else:
417 atmgen = None
418
419 retStruct = pipeBase.Struct(offsets=offsets,
420 atmospheres=atmgen)
421 retStruct.photoCalibCatalogs = pcgen
422
423 return retStruct
424
425 def generateTractOutputProducts(self, handleDict, tract,
426 visitCat, zptCat, atmCat, stdCat,
427 fgcmBuildStarsConfig):
428 """
429 Generate the output products for a given tract, as specified in the config.
430
431 This method is here to have an alternate entry-point for
432 FgcmCalibrateTract.
433
434 Parameters
435 ----------
436 handleDict : `dict`
437 All handles are `lsst.daf.butler.DeferredDatasetHandle`
438 handle dictionary with keys:
439
440 ``"camera"``
441 Camera object (`lsst.afw.cameraGeom.Camera`)
442 ``"fgcmLookUpTable"``
443 handle for the FGCM look-up table.
444 tract : `int`
445 Tract number
446 visitCat : `lsst.afw.table.BaseCatalog`
447 FGCM visitCat from `FgcmBuildStarsTask`
448 zptCat : `lsst.afw.table.BaseCatalog`
449 FGCM zeropoint catalog from `FgcmFitCycleTask`
450 atmCat : `lsst.afw.table.BaseCatalog`
451 FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
452 stdCat : `lsst.afw.table.SimpleCatalog`
453 FGCM standard star catalog from `FgcmFitCycleTask`
454 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
455 Configuration object from `FgcmBuildStarsTask`
456
457 Returns
458 -------
459 retStruct : `lsst.pipe.base.Struct`
460 Output structure with keys:
461
462 offsets : `np.ndarray`
463 Final reference offsets, per band.
464 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
465 Generator that returns (visit, transmissionCurve) tuples.
466 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
467 Generator that returns (visit, exposureCatalog) tuples.
468 """
469 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
470
471 md = stdCat.getMetadata()
472 bands = md.getArray('BANDS')
473
474 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
475 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
476 "in fgcmBuildStarsTask.")
477
478 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
479 self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
480
481 if self.config.doReferenceCalibration:
482 lutCat = handleDict['fgcmLookUpTable'].get()
483 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap)
484 else:
485 offsets = np.zeros(len(bands))
486
487 if self.config.doZeropointOutput:
488 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands,
489 physicalFilterMap)
490 else:
491 pcgen = None
492
493 if self.config.doAtmosphereOutput:
494 atmgen = self._outputAtmospheres(handleDict, atmCat)
495 else:
496 atmgen = None
497
498 retStruct = pipeBase.Struct(offsets=offsets,
499 atmospheres=atmgen)
500 retStruct.photoCalibCatalogs = pcgen
501
502 return retStruct
503
504 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
505 """
506 Compute offsets relative to a reference catalog.
507
508 This method splits the star catalog into healpix pixels
509 and computes the calibration transfer for a sample of
510 these pixels to approximate the 'absolute' calibration
511 values (on for each band) to apply to transfer the
512 absolute scale.
513
514 Parameters
515 ----------
516 stdCat : `lsst.afw.table.SimpleCatalog`
517 FGCM standard stars
518 lutCat : `lsst.afw.table.SimpleCatalog`
519 FGCM Look-up table
520 physicalFilterMap : `dict`
521 Dictionary of mappings from physical filter to FGCM band.
522 bands : `list` [`str`]
523 List of band names from FGCM output
524 Returns
525 -------
526 offsets : `numpy.array` of floats
527 Per band zeropoint offsets
528 """
529
530 # Only use stars that are observed in all the bands that were actually used
531 # This will ensure that we use the same healpix pixels for the absolute
532 # calibration of each band.
533 minObs = stdCat['ngood'].min(axis=1)
534
535 goodStars = (minObs >= 1)
536 stdCat = stdCat[goodStars]
537
538 self.log.info("Found %d stars with at least 1 good observation in each band" %
539 (len(stdCat)))
540
541 # Associate each band with the appropriate physicalFilter and make
542 # filterLabels
543 filterLabels = []
544
545 lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',')
546 lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',')
547 physicalFilterMapBands = list(physicalFilterMap.values())
548 physicalFilterMapFilters = list(physicalFilterMap.keys())
549 for band in bands:
550 # Find a physical filter associated from the band by doing
551 # a reverse lookup on the physicalFilterMap dict
552 physicalFilterMapIndex = physicalFilterMapBands.index(band)
553 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
554 # Find the appropriate fgcm standard physicalFilter
555 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
556 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
557 filterLabels.append(afwImage.FilterLabel(band=band,
558 physical=stdPhysicalFilter))
559
560 # We have to make a table for each pixel with flux/fluxErr
561 # This is a temporary table generated for input to the photoCal task.
562 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
563 # have had chromatic corrections applied to get to the standard system
564 # specified by the atmosphere/instrumental parameters), nor are they
565 # in Jansky (since they don't have a proper absolute calibration: the overall
566 # zeropoint is estimated from the telescope size, etc.)
567 sourceMapper = afwTable.SchemaMapper(stdCat.schema)
568 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
569 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
570 doc="instrumental flux (counts)")
571 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
572 doc="instrumental flux error (counts)")
573 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
574 type='Flag',
575 doc="bad flag")
576
577 # Split up the stars
578 # Note that there is an assumption here that the ra/dec coords stored
579 # on-disk are in radians, and therefore that starObs['coord_ra'] /
580 # starObs['coord_dec'] return radians when used as an array of numpy float64s.
581 ipring = hpg.angle_to_pixel(
582 self.config.referencePixelizationNside,
583 stdCat['coord_ra'],
584 stdCat['coord_dec'],
585 degrees=False,
586 )
587 h, rev = esutil.stat.histogram(ipring, rev=True)
588
589 gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
590
591 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
592 (gdpix.size,
593 self.config.referencePixelizationNside,
594 self.config.referencePixelizationMinStars))
595
596 if gdpix.size < self.config.referencePixelizationNPixels:
597 self.log.warning("Found fewer good pixels (%d) than preferred in configuration (%d)" %
598 (gdpix.size, self.config.referencePixelizationNPixels))
599 else:
600 # Sample out the pixels we want to use
601 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
602
603 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
604 ('nstar', 'i4', len(bands)),
605 ('nmatch', 'i4', len(bands)),
606 ('zp', 'f4', len(bands)),
607 ('zpErr', 'f4', len(bands))])
608 results['hpix'] = ipring[rev[rev[gdpix]]]
609
610 # We need a boolean index to deal with catalogs...
611 selected = np.zeros(len(stdCat), dtype=bool)
612
613 refFluxFields = [None]*len(bands)
614
615 for p_index, pix in enumerate(gdpix):
616 i1a = rev[rev[pix]: rev[pix + 1]]
617
618 # the stdCat afwTable can only be indexed with boolean arrays,
619 # and not numpy index arrays (see DM-16497). This little trick
620 # converts the index array into a boolean array
621 selected[:] = False
622 selected[i1a] = True
623
624 for b_index, filterLabel in enumerate(filterLabels):
625 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index,
626 filterLabel, stdCat,
627 selected, refFluxFields)
628 results['nstar'][p_index, b_index] = len(i1a)
629 results['nmatch'][p_index, b_index] = len(struct.arrays.refMag)
630 results['zp'][p_index, b_index] = struct.zp
631 results['zpErr'][p_index, b_index] = struct.sigma
632
633 # And compute the summary statistics
634 offsets = np.zeros(len(bands))
635
636 for b_index, band in enumerate(bands):
637 # make configurable
638 ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch)
639 offsets[b_index] = np.median(results['zp'][ok, b_index])
640 # use median absolute deviation to estimate Normal sigma
641 # see https://en.wikipedia.org/wiki/Median_absolute_deviation
642 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index]))
643 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f",
644 band, offsets[b_index], madSigma)
645
646 return offsets
647
648 def _computeOffsetOneBand(self, sourceMapper, badStarKey,
649 b_index, filterLabel, stdCat, selected, refFluxFields):
650 """
651 Compute the zeropoint offset between the fgcm stdCat and the reference
652 stars for one pixel in one band
653
654 Parameters
655 ----------
656 sourceMapper : `lsst.afw.table.SchemaMapper`
657 Mapper to go from stdCat to calibratable catalog
658 badStarKey : `lsst.afw.table.Key`
659 Key for the field with bad stars
660 b_index : `int`
661 Index of the band in the star catalog
662 filterLabel : `lsst.afw.image.FilterLabel`
663 filterLabel with band and physical filter
664 stdCat : `lsst.afw.table.SimpleCatalog`
665 FGCM standard stars
666 selected : `numpy.array(dtype=bool)`
667 Boolean array of which stars are in the pixel
668 refFluxFields : `list`
669 List of names of flux fields for reference catalog
670 """
671
672 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
673 sourceCat.reserve(selected.sum())
674 sourceCat.extend(stdCat[selected], mapper=sourceMapper)
675 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5))
676 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index]
677 * sourceCat['instFlux'])
678 # Make sure we only use stars that have valid measurements
679 # (This is perhaps redundant with requirements above that the
680 # stars be observed in all bands, but it can't hurt)
681 badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0)
682 for rec in sourceCat[badStar]:
683 rec.set(badStarKey, True)
684
685 exposure = afwImage.ExposureF()
686 exposure.setFilter(filterLabel)
687
688 if refFluxFields[b_index] is None:
689 # Need to find the flux field in the reference catalog
690 # to work around limitations of DirectMatch in PhotoCal
691 ctr = stdCat[0].getCoord()
692 rad = 0.05*lsst.geom.degrees
693 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel)
694 refFluxFields[b_index] = refDataTest.fluxField
695
696 # Make a copy of the config so that we can modify it
697 calConfig = copy.copy(self.config.photoCal.value)
698 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index]
699 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err'
700 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
701 config=calConfig,
702 schema=sourceCat.getSchema())
703
704 struct = calTask.run(exposure, sourceCat)
705
706 return struct
707
708 def _formatCatalog(self, fgcmStarCat, offsets, bands):
709 """
710 Turn an FGCM-formatted star catalog, applying zeropoint offsets.
711
712 Parameters
713 ----------
714 fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
715 SimpleCatalog as output by fgcmcal
716 offsets : `list` with len(self.bands) entries
717 Zeropoint offsets to apply
718 bands : `list` [`str`]
719 List of band names from FGCM output
720
721 Returns
722 -------
723 formattedCat: `lsst.afw.table.SimpleCatalog`
724 SimpleCatalog suitable for using as a reference catalog
725 """
726
727 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
728 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
729 addCentroid=False,
730 addIsResolved=True,
731 coordErrDim=0)
732 sourceMapper.addMinimalSchema(minSchema)
733 for band in bands:
734 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
735 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
736 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
737
738 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
739 formattedCat.reserve(len(fgcmStarCat))
740 formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
741
742 # Note that we don't have to set `resolved` because the default is False
743
744 for b, band in enumerate(bands):
745 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
746 # We want fluxes in nJy from calibrated AB magnitudes
747 # (after applying offset). Updated after RFC-549 and RFC-575.
748 flux = (mag*units.ABmag).to_value(units.nJy)
749 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
750
751 formattedCat['%s_flux' % (band)][:] = flux
752 formattedCat['%s_fluxErr' % (band)][:] = fluxErr
753 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
754 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
755 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
756
757 addRefCatMetadata(formattedCat)
758
759 return formattedCat
760
761 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
762 physicalFilterMap, tract=None):
763 """Output the zeropoints in fgcm_photoCalib format.
764
765 Parameters
766 ----------
767 camera : `lsst.afw.cameraGeom.Camera`
768 Camera from the butler.
769 zptCat : `lsst.afw.table.BaseCatalog`
770 FGCM zeropoint catalog from `FgcmFitCycleTask`.
771 visitCat : `lsst.afw.table.BaseCatalog`
772 FGCM visitCat from `FgcmBuildStarsTask`.
773 offsets : `numpy.array`
774 Float array of absolute calibration offsets, one for each filter.
775 bands : `list` [`str`]
776 List of band names from FGCM output.
777 physicalFilterMap : `dict`
778 Dictionary of mappings from physical filter to FGCM band.
779 tract: `int`, optional
780 Tract number to output. Default is None (global calibration)
781
782 Returns
783 -------
784 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
785 Generator that returns (visit, exposureCatalog) tuples.
786 """
787 # Select visit/ccds where we have a calibration
788 # This includes ccds where we were able to interpolate from neighboring
789 # ccds.
790 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
791 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
792 & (zptCat['fgcmZptVar'] > 0.0)
793 & (zptCat['fgcmZpt'] > FGCM_ILLEGAL_VALUE))
794
795 # Log warnings for any visit which has no valid zeropoints
796 badVisits = np.unique(zptCat['visit'][~selected])
797 goodVisits = np.unique(zptCat['visit'][selected])
798 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
799 for allBadVisit in allBadVisits:
800 self.log.warning(f'No suitable photoCalib for visit {allBadVisit}')
801
802 # Get a mapping from filtername to the offsets
803 offsetMapping = {}
804 for f in physicalFilterMap:
805 # Not every filter in the map will necesarily have a band.
806 if physicalFilterMap[f] in bands:
807 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
808
809 # Get a mapping from "ccd" to the ccd index used for the scaling
810 ccdMapping = {}
811 for ccdIndex, detector in enumerate(camera):
812 ccdMapping[detector.getId()] = ccdIndex
813
814 # And a mapping to get the flat-field scaling values
815 scalingMapping = {}
816 for rec in visitCat:
817 scalingMapping[rec['visit']] = rec['scaling']
818
819 if self.config.doComposeWcsJacobian:
820 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
821
822 # The zptCat is sorted by visit, which is useful
823 lastVisit = -1
824 zptVisitCatalog = None
825
826 metadata = dafBase.PropertyList()
827 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
828 metadata.add("COMMENT", "Only detectors with data have entries.")
829
830 for rec in zptCat[selected]:
831 # Retrieve overall scaling
832 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
833
834 # The postCalibrationOffset describe any zeropoint offsets
835 # to apply after the fgcm calibration. The first part comes
836 # from the reference catalog match (used in testing). The
837 # second part comes from the mean chromatic correction
838 # (if configured).
839 postCalibrationOffset = offsetMapping[rec['filtername']]
840 if self.config.doApplyMeanChromaticCorrection:
841 postCalibrationOffset += rec['fgcmDeltaChrom']
842
843 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
844 rec['fgcmfZptChebXyMax'])
845 # Convert from FGCM AB to nJy
846 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
847 rec['fgcmfZptChebXyMax'],
848 offset=postCalibrationOffset,
849 scaling=scaling)
850
851 if self.config.doComposeWcsJacobian:
852
853 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']],
854 fgcmSuperStarField,
855 fgcmZptField])
856 else:
857 # The photoCalib is just the product of the fgcmSuperStarField and the
858 # fgcmZptField
859 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
860
861 # The "mean" calibration will be set to the center of the ccd for reference
862 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
863 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
864 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
865 calibrationErr=calibErr,
866 calibration=fgcmField,
867 isConstant=False)
868
869 # Return full per-visit exposure catalogs
870 if rec['visit'] != lastVisit:
871 # This is a new visit. If the last visit was not -1, yield
872 # the ExposureCatalog
873 if lastVisit > -1:
874 # ensure that the detectors are in sorted order, for fast lookups
875 zptVisitCatalog.sort()
876 yield (int(lastVisit), zptVisitCatalog)
877 else:
878 # We need to create a new schema
879 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
880 zptExpCatSchema.addField('visit', type='L', doc='Visit number')
881
882 # And start a new one
883 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
884 zptVisitCatalog.setMetadata(metadata)
885
886 lastVisit = int(rec['visit'])
887
888 catRecord = zptVisitCatalog.addNew()
889 catRecord['id'] = int(rec['detector'])
890 catRecord['visit'] = rec['visit']
891 catRecord.setPhotoCalib(photoCalib)
892
893 # Final output of last exposure catalog
894 # ensure that the detectors are in sorted order, for fast lookups
895 zptVisitCatalog.sort()
896 yield (int(lastVisit), zptVisitCatalog)
897
898 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
899 """
900 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
901 and scaling.
902
903 Parameters
904 ----------
905 coefficients: `numpy.array`
906 Flattened array of chebyshev coefficients
907 xyMax: `list` of length 2
908 Maximum x and y of the chebyshev bounding box
909 offset: `float`, optional
910 Absolute calibration offset. Default is 0.0
911 scaling: `float`, optional
912 Flat scaling value from fgcmBuildStars. Default is 1.0
913
914 Returns
915 -------
916 boundedField: `lsst.afw.math.ChebyshevBoundedField`
917 """
918
919 orderPlus1 = int(np.sqrt(coefficients.size))
920 pars = np.zeros((orderPlus1, orderPlus1))
921
922 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
923 lsst.geom.Point2I(*xyMax))
924
925 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
926 * (10.**(offset/-2.5))*scaling)
927
928 boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
929
930 return boundedField
931
932 def _outputAtmospheres(self, handleDict, atmCat):
933 """
934 Output the atmospheres.
935
936 Parameters
937 ----------
938 handleDict : `dict`
939 All data handles are `lsst.daf.butler.DeferredDatasetHandle`
940 The handleDict has the follownig keys:
941
942 ``"fgcmLookUpTable"``
943 handle for the FGCM look-up table.
944 atmCat : `lsst.afw.table.BaseCatalog`
945 FGCM atmosphere parameter catalog from fgcmFitCycleTask.
946
947 Returns
948 -------
949 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
950 Generator that returns (visit, transmissionCurve) tuples.
951 """
952 # First, we need to grab the look-up table and key info
953 lutCat = handleDict['fgcmLookUpTable'].get()
954
955 atmosphereTableName = lutCat[0]['tablename']
956 elevation = lutCat[0]['elevation']
957 atmLambda = lutCat[0]['atmLambda']
958 lutCat = None
959
960 # Make the atmosphere table if possible
961 try:
962 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
963 atmTable.loadTable()
964 except IOError:
965 atmTable = None
966
967 if atmTable is None:
968 # Try to use MODTRAN instead
969 try:
970 modGen = fgcm.ModtranGenerator(elevation)
971 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
972 lambdaStep = (atmLambda[1] - atmLambda[0])/10.
973 except (ValueError, IOError) as e:
974 raise RuntimeError("FGCM look-up-table generated with modtran, "
975 "but modtran not configured to run.") from e
976
977 zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
978
979 for i, visit in enumerate(atmCat['visit']):
980 if atmTable is not None:
981 # Interpolate the atmosphere table
982 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
983 pwv=atmCat[i]['pwv'],
984 o3=atmCat[i]['o3'],
985 tau=atmCat[i]['tau'],
986 alpha=atmCat[i]['alpha'],
987 zenith=zenith[i],
988 ctranslamstd=[atmCat[i]['cTrans'],
989 atmCat[i]['lamStd']])
990 else:
991 # Run modtran
992 modAtm = modGen(pmb=atmCat[i]['pmb'],
993 pwv=atmCat[i]['pwv'],
994 o3=atmCat[i]['o3'],
995 tau=atmCat[i]['tau'],
996 alpha=atmCat[i]['alpha'],
997 zenith=zenith[i],
998 lambdaRange=lambdaRange,
999 lambdaStep=lambdaStep,
1000 ctranslamstd=[atmCat[i]['cTrans'],
1001 atmCat[i]['lamStd']])
1002 atmVals = modAtm['COMBINED']
1003
1004 # Now need to create something to persist...
1005 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1006 wavelengths=atmLambda,
1007 throughputAtMin=atmVals[0],
1008 throughputAtMax=atmVals[-1])
1009
1010 yield (int(visit), curve)
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:522