lsst.fgcmcal  22.0.1-10-g779eefa+7bfd648598
fgcmOutputProducts.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Make the final fgcmcal output products.
24 
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
32 offset per band.
33 """
34 import sys
35 import traceback
36 import copy
37 
38 import numpy as np
39 import healpy as hp
40 import esutil
41 from astropy import units
42 
43 import lsst.daf.base as dafBase
44 import lsst.pex.config as pexConfig
45 import lsst.pipe.base as pipeBase
46 from lsst.pipe.base import connectionTypes
47 from lsst.afw.image import TransmissionCurve
48 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
49 from lsst.meas.algorithms import ReferenceObjectLoader
50 from lsst.pipe.tasks.photoCal import PhotoCalTask
51 import lsst.geom
52 import lsst.afw.image as afwImage
53 import lsst.afw.math as afwMath
54 import lsst.afw.table as afwTable
55 from lsst.meas.algorithms import IndexerRegistry
56 from lsst.meas.algorithms import DatasetConfig
57 from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
58 
59 from .utilities import computeApproxPixelAreaFields
60 from .utilities import lookupStaticCalibrations
61 
62 import fgcm
63 
64 __all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
65 
66 
67 class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
68  dimensions=("instrument",),
69  defaultTemplates={"cycleNumber": "0"}):
70  camera = connectionTypes.PrerequisiteInput(
71  doc="Camera instrument",
72  name="camera",
73  storageClass="Camera",
74  dimensions=("instrument",),
75  lookupFunction=lookupStaticCalibrations,
76  isCalibration=True,
77  )
78 
79  fgcmLookUpTable = connectionTypes.PrerequisiteInput(
80  doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
81  "chromatic corrections."),
82  name="fgcmLookUpTable",
83  storageClass="Catalog",
84  dimensions=("instrument",),
85  deferLoad=True,
86  )
87 
88  fgcmVisitCatalog = connectionTypes.Input(
89  doc="Catalog of visit information for fgcm",
90  name="fgcmVisitCatalog",
91  storageClass="Catalog",
92  dimensions=("instrument",),
93  deferLoad=True,
94  )
95 
96  fgcmStandardStars = connectionTypes.Input(
97  doc="Catalog of standard star data from fgcm fit",
98  name="fgcmStandardStars{cycleNumber}",
99  storageClass="SimpleCatalog",
100  dimensions=("instrument",),
101  deferLoad=True,
102  )
103 
104  fgcmZeropoints = connectionTypes.Input(
105  doc="Catalog of zeropoints from fgcm fit",
106  name="fgcmZeropoints{cycleNumber}",
107  storageClass="Catalog",
108  dimensions=("instrument",),
109  deferLoad=True,
110  )
111 
112  fgcmAtmosphereParameters = connectionTypes.Input(
113  doc="Catalog of atmosphere parameters from fgcm fit",
114  name="fgcmAtmosphereParameters{cycleNumber}",
115  storageClass="Catalog",
116  dimensions=("instrument",),
117  deferLoad=True,
118  )
119 
120  refCat = connectionTypes.PrerequisiteInput(
121  doc="Reference catalog to use for photometric calibration",
122  name="cal_ref_cat",
123  storageClass="SimpleCatalog",
124  dimensions=("skypix",),
125  deferLoad=True,
126  multiple=True,
127  )
128 
129  fgcmPhotoCalib = connectionTypes.Output(
130  doc=("Per-visit photometric calibrations derived from fgcm calibration. "
131  "These catalogs use detector id for the id and are sorted for "
132  "fast lookups of a detector."),
133  name="fgcmPhotoCalibCatalog",
134  storageClass="ExposureCatalog",
135  dimensions=("instrument", "visit",),
136  multiple=True,
137  )
138 
139  fgcmTransmissionAtmosphere = connectionTypes.Output(
140  doc="Per-visit atmosphere transmission files produced from fgcm calibration",
141  name="transmission_atmosphere_fgcm",
142  storageClass="TransmissionCurve",
143  dimensions=("instrument",
144  "visit",),
145  multiple=True,
146  )
147 
148  fgcmOffsets = connectionTypes.Output(
149  doc="Per-band offsets computed from doReferenceCalibration",
150  name="fgcmReferenceCalibrationOffsets",
151  storageClass="Catalog",
152  dimensions=("instrument",),
153  multiple=False,
154  )
155 
156  def __init__(self, *, config=None):
157  super().__init__(config=config)
158 
159  if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
160  raise ValueError("cycleNumber must be of integer format")
161  if config.connections.refCat != config.refObjLoader.ref_dataset_name:
162  raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name")
163 
164  if config.doRefcatOutput:
165  raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
166 
167  if not config.doReferenceCalibration:
168  self.prerequisiteInputs.remove("refCat")
169  if not config.doAtmosphereOutput:
170  self.inputs.remove("fgcmAtmosphereParameters")
171  if not config.doZeropointOutput:
172  self.inputs.remove("fgcmZeropoints")
173  if not config.doReferenceCalibration:
174  self.outputs.remove("fgcmOffsets")
175 
176 
177 class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
178  pipelineConnections=FgcmOutputProductsConnections):
179  """Config for FgcmOutputProductsTask"""
180 
181  cycleNumber = pexConfig.Field(
182  doc="Final fit cycle from FGCM fit",
183  dtype=int,
184  default=None,
185  )
186  physicalFilterMap = pexConfig.DictField(
187  doc="Mapping from 'physicalFilter' to band.",
188  keytype=str,
189  itemtype=str,
190  default={},
191  )
192  # The following fields refer to calibrating from a reference
193  # catalog, but in the future this might need to be expanded
194  doReferenceCalibration = pexConfig.Field(
195  doc=("Transfer 'absolute' calibration from reference catalog? "
196  "This afterburner step is unnecessary if reference stars "
197  "were used in the full fit in FgcmFitCycleTask."),
198  dtype=bool,
199  default=False,
200  )
201  doRefcatOutput = pexConfig.Field(
202  doc="Output standard stars in reference catalog format",
203  dtype=bool,
204  default=True,
205  )
206  doAtmosphereOutput = pexConfig.Field(
207  doc="Output atmospheres in transmission_atmosphere_fgcm format",
208  dtype=bool,
209  default=True,
210  )
211  doZeropointOutput = pexConfig.Field(
212  doc="Output zeropoints in fgcm_photoCalib format",
213  dtype=bool,
214  default=True,
215  )
216  doComposeWcsJacobian = pexConfig.Field(
217  doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
218  dtype=bool,
219  default=True,
220  )
221  doApplyMeanChromaticCorrection = pexConfig.Field(
222  doc="Apply the mean chromatic correction to the zeropoints?",
223  dtype=bool,
224  default=True,
225  )
226  refObjLoader = pexConfig.ConfigurableField(
227  target=LoadIndexedReferenceObjectsTask,
228  doc="reference object loader for 'absolute' photometric calibration",
229  )
230  photoCal = pexConfig.ConfigurableField(
231  target=PhotoCalTask,
232  doc="task to perform 'absolute' calibration",
233  )
234  referencePixelizationNside = pexConfig.Field(
235  doc="Healpix nside to pixelize catalog to compare to reference catalog",
236  dtype=int,
237  default=64,
238  )
239  referencePixelizationMinStars = pexConfig.Field(
240  doc=("Minimum number of stars per healpix pixel to select for comparison"
241  "to the specified reference catalog"),
242  dtype=int,
243  default=200,
244  )
245  referenceMinMatch = pexConfig.Field(
246  doc="Minimum number of stars matched to reference catalog to be used in statistics",
247  dtype=int,
248  default=50,
249  )
250  referencePixelizationNPixels = pexConfig.Field(
251  doc=("Number of healpix pixels to sample to do comparison. "
252  "Doing too many will take a long time and not yield any more "
253  "precise results because the final number is the median offset "
254  "(per band) from the set of pixels."),
255  dtype=int,
256  default=100,
257  )
258  datasetConfig = pexConfig.ConfigField(
259  dtype=DatasetConfig,
260  doc="Configuration for writing/reading ingested catalog",
261  )
262 
263  def setDefaults(self):
264  pexConfig.Config.setDefaults(self)
265 
266  # In order to transfer the "absolute" calibration from a reference
267  # catalog to the relatively calibrated FGCM standard stars (one number
268  # per band), we use the PhotoCalTask to match stars in a sample of healpix
269  # pixels. These basic settings ensure that only well-measured, good stars
270  # from the source and reference catalogs are used for the matching.
271 
272  # applyColorTerms needs to be False if doReferenceCalibration is False,
273  # as is the new default after DM-16702
274  self.photoCal.applyColorTerms = False
275  self.photoCal.fluxField = 'instFlux'
276  self.photoCal.magErrFloor = 0.003
277  self.photoCal.match.referenceSelection.doSignalToNoise = True
278  self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
279  self.photoCal.match.sourceSelection.doSignalToNoise = True
280  self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
281  self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
282  self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
283  self.photoCal.match.sourceSelection.doFlags = True
284  self.photoCal.match.sourceSelection.flags.good = []
285  self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
286  self.photoCal.match.sourceSelection.doUnresolved = False
287  self.datasetConfig.ref_dataset_name = 'fgcm_stars'
288  self.datasetConfig.format_version = 1
289 
290  def validate(self):
291  super().validate()
292 
293  # Force the connections to conform with cycleNumber
294  self.connections.cycleNumber = str(self.cycleNumber)
295 
296 
297 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
298  """Subclass of TaskRunner for fgcmOutputProductsTask
299 
300  fgcmOutputProductsTask.run() takes one argument, the butler, and
301  does not run on any data in the repository.
302  This runner does not use any parallelization.
303  """
304 
305  @staticmethod
306  def getTargetList(parsedCmd):
307  """
308  Return a list with one element, the butler.
309  """
310  return [parsedCmd.butler]
311 
312  def __call__(self, butler):
313  """
314  Parameters
315  ----------
316  butler: `lsst.daf.persistence.Butler`
317 
318  Returns
319  -------
320  exitStatus: `list` with `pipeBase.Struct`
321  exitStatus (0: success; 1: failure)
322  if self.doReturnResults also
323  results (`np.array` with absolute zeropoint offsets)
324  """
325  task = self.TaskClass(butler=butler, config=self.config, log=self.log)
326 
327  exitStatus = 0
328  if self.doRaise:
329  results = task.runDataRef(butler)
330  else:
331  try:
332  results = task.runDataRef(butler)
333  except Exception as e:
334  exitStatus = 1
335  task.log.fatal("Failed: %s" % e)
336  if not isinstance(e, pipeBase.TaskError):
337  traceback.print_exc(file=sys.stderr)
338 
339  task.writeMetadata(butler)
340 
341  if self.doReturnResults:
342  # The results here are the zeropoint offsets for each band
343  return [pipeBase.Struct(exitStatus=exitStatus,
344  results=results)]
345  else:
346  return [pipeBase.Struct(exitStatus=exitStatus)]
347 
348  def run(self, parsedCmd):
349  """
350  Run the task, with no multiprocessing
351 
352  Parameters
353  ----------
354  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
355  """
356 
357  resultList = []
358 
359  if self.precall(parsedCmd):
360  targetList = self.getTargetList(parsedCmd)
361  # make sure that we only get 1
362  resultList = self(targetList[0])
363 
364  return resultList
365 
366 
367 class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
368  """
369  Output products from FGCM global calibration.
370  """
371 
372  ConfigClass = FgcmOutputProductsConfig
373  RunnerClass = FgcmOutputProductsRunner
374  _DefaultName = "fgcmOutputProducts"
375 
376  def __init__(self, butler=None, **kwargs):
377  super().__init__(**kwargs)
378 
379  # no saving of metadata for now
380  def _getMetadataName(self):
381  return None
382 
383  def runQuantum(self, butlerQC, inputRefs, outputRefs):
384  dataRefDict = {}
385  dataRefDict['camera'] = butlerQC.get(inputRefs.camera)
386  dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
387  dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
388  dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
389 
390  if self.config.doZeropointOutput:
391  dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
392  photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
393  photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
394 
395  if self.config.doAtmosphereOutput:
396  dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
397  atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
398  atmRef in outputRefs.fgcmTransmissionAtmosphere}
399 
400  if self.config.doReferenceCalibration:
401  refConfig = self.config.refObjLoader
402  self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
403  for ref in inputRefs.refCat],
404  refCats=butlerQC.get(inputRefs.refCat),
405  config=refConfig,
406  log=self.log)
407  else:
408  self.refObjLoader = None
409 
410  struct = self.run(dataRefDict, self.config.physicalFilterMap, returnCatalogs=True)
411 
412  # Output the photoCalib exposure catalogs
413  if struct.photoCalibCatalogs is not None:
414  self.log.info("Outputting photoCalib catalogs.")
415  for visit, expCatalog in struct.photoCalibCatalogs:
416  butlerQC.put(expCatalog, photoCalibRefDict[visit])
417  self.log.info("Done outputting photoCalib catalogs.")
418 
419  # Output the atmospheres
420  if struct.atmospheres is not None:
421  self.log.info("Outputting atmosphere transmission files.")
422  for visit, atm in struct.atmospheres:
423  butlerQC.put(atm, atmRefDict[visit])
424  self.log.info("Done outputting atmosphere files.")
425 
426  if self.config.doReferenceCalibration:
427  # Turn offset into simple catalog for persistence if necessary
428  schema = afwTable.Schema()
429  schema.addField('offset', type=np.float64,
430  doc="Post-process calibration offset (mag)")
431  offsetCat = afwTable.BaseCatalog(schema)
432  offsetCat.resize(len(struct.offsets))
433  offsetCat['offset'][:] = struct.offsets
434 
435  butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
436 
437  return
438 
439  @pipeBase.timeMethod
440  def runDataRef(self, butler):
441  """
442  Make FGCM output products for use in the stack
443 
444  Parameters
445  ----------
446  butler: `lsst.daf.persistence.Butler`
447  cycleNumber: `int`
448  Final fit cycle number, override config.
449 
450  Returns
451  -------
452  offsets: `lsst.pipe.base.Struct`
453  A structure with array of zeropoint offsets
454 
455  Raises
456  ------
457  RuntimeError:
458  Raised if any one of the following is true:
459 
460  - butler cannot find "fgcmBuildStars_config" or
461  "fgcmBuildStarsTable_config".
462  - butler cannot find "fgcmFitCycle_config".
463  - "fgcmFitCycle_config" does not refer to
464  `self.config.cycleNumber`.
465  - butler cannot find "fgcmAtmosphereParameters" and
466  `self.config.doAtmosphereOutput` is `True`.
467  - butler cannot find "fgcmStandardStars" and
468  `self.config.doReferenceCalibration` is `True` or
469  `self.config.doRefcatOutput` is `True`.
470  - butler cannot find "fgcmZeropoints" and
471  `self.config.doZeropointOutput` is `True`.
472  """
473  if self.config.doReferenceCalibration:
474  # We need the ref obj loader to get the flux field
475  self.makeSubtask("refObjLoader", butler=butler)
476 
477  # Check to make sure that the fgcmBuildStars config exists, to retrieve
478  # the visit and ccd dataset tags
479  if not butler.datasetExists('fgcmBuildStarsTable_config') and \
480  not butler.datasetExists('fgcmBuildStars_config'):
481  raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
482  "which is prereq for fgcmOutputProducts")
483 
484  if butler.datasetExists('fgcmBuildStarsTable_config'):
485  fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
486  else:
487  fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
488  visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
489  ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
490  physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
491 
492  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
493  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
494  "in fgcmBuildStarsTask.")
495 
496  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
497  self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
498 
499  # And make sure that the atmosphere was output properly
500  if (self.config.doAtmosphereOutput
501  and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
502  raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
503 
504  if not butler.datasetExists('fgcmStandardStars',
505  fgcmcycle=self.config.cycleNumber):
506  raise RuntimeError("Standard stars are missing for cycle %d." %
507  (self.config.cycleNumber))
508 
509  if (self.config.doZeropointOutput
510  and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
511  raise RuntimeError("Zeropoints are missing for cycle %d." %
512  (self.config.cycleNumber))
513 
514  dataRefDict = {}
515  # This is the _actual_ camera
516  dataRefDict['camera'] = butler.get('camera')
517  dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
518  dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
519  dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars',
520  fgcmcycle=self.config.cycleNumber)
521 
522  if self.config.doZeropointOutput:
523  dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints',
524  fgcmcycle=self.config.cycleNumber)
525  if self.config.doAtmosphereOutput:
526  dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters',
527  fgcmcycle=self.config.cycleNumber)
528 
529  struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False)
530 
531  if struct.photoCalibs is not None:
532  self.log.info("Outputting photoCalib files.")
533 
534  for visit, detector, physicalFilter, photoCalib in struct.photoCalibs:
535  butler.put(photoCalib, 'fgcm_photoCalib',
536  dataId={visitDataRefName: visit,
537  ccdDataRefName: detector,
538  'filter': physicalFilter})
539 
540  self.log.info("Done outputting photoCalib files.")
541 
542  if struct.atmospheres is not None:
543  self.log.info("Outputting atmosphere transmission files.")
544  for visit, atm in struct.atmospheres:
545  butler.put(atm, "transmission_atmosphere_fgcm",
546  dataId={visitDataRefName: visit})
547  self.log.info("Done outputting atmosphere transmissions.")
548 
549  return pipeBase.Struct(offsets=struct.offsets)
550 
551  def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
552  """Run the output products task.
553 
554  Parameters
555  ----------
556  dataRefDict : `dict`
557  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
558  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
559  dataRef dictionary with keys:
560 
561  ``"camera"``
562  Camera object (`lsst.afw.cameraGeom.Camera`)
563  ``"fgcmLookUpTable"``
564  dataRef for the FGCM look-up table.
565  ``"fgcmVisitCatalog"``
566  dataRef for visit summary catalog.
567  ``"fgcmStandardStars"``
568  dataRef for the output standard star catalog.
569  ``"fgcmZeropoints"``
570  dataRef for the zeropoint data catalog.
571  ``"fgcmAtmosphereParameters"``
572  dataRef for the atmosphere parameter catalog.
573  ``"fgcmBuildStarsTableConfig"``
574  Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
575  physicalFilterMap : `dict`
576  Dictionary of mappings from physical filter to FGCM band.
577  returnCatalogs : `bool`, optional
578  Return photoCalibs as per-visit exposure catalogs.
579  butler : `lsst.daf.persistence.Butler`, optional
580  Gen2 butler used for reference star outputs
581 
582  Returns
583  -------
584  retStruct : `lsst.pipe.base.Struct`
585  Output structure with keys:
586 
587  offsets : `np.ndarray`
588  Final reference offsets, per band.
589  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
590  Generator that returns (visit, transmissionCurve) tuples.
591  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
592  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
593  (returned if returnCatalogs is False).
594  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
595  Generator that returns (visit, exposureCatalog) tuples.
596  (returned if returnCatalogs is True).
597  """
598  stdCat = dataRefDict['fgcmStandardStars'].get()
599  md = stdCat.getMetadata()
600  bands = md.getArray('BANDS')
601 
602  if self.config.doReferenceCalibration:
603  lutCat = dataRefDict['fgcmLookUpTable'].get()
604  offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands)
605  else:
606  offsets = np.zeros(len(bands))
607 
608  # This is Gen2 only, and requires the butler.
609  if self.config.doRefcatOutput and butler is not None:
610  self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
611 
612  del stdCat
613 
614  if self.config.doZeropointOutput:
615  zptCat = dataRefDict['fgcmZeropoints'].get()
616  visitCat = dataRefDict['fgcmVisitCatalog'].get()
617 
618  pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
619  physicalFilterMap, returnCatalogs=returnCatalogs)
620  else:
621  pcgen = None
622 
623  if self.config.doAtmosphereOutput:
624  atmCat = dataRefDict['fgcmAtmosphereParameters'].get()
625  atmgen = self._outputAtmospheres(dataRefDict, atmCat)
626  else:
627  atmgen = None
628 
629  retStruct = pipeBase.Struct(offsets=offsets,
630  atmospheres=atmgen)
631  if returnCatalogs:
632  retStruct.photoCalibCatalogs = pcgen
633  else:
634  retStruct.photoCalibs = pcgen
635 
636  return retStruct
637 
638  def generateTractOutputProducts(self, dataRefDict, tract,
639  visitCat, zptCat, atmCat, stdCat,
640  fgcmBuildStarsConfig,
641  returnCatalogs=True,
642  butler=None):
643  """
644  Generate the output products for a given tract, as specified in the config.
645 
646  This method is here to have an alternate entry-point for
647  FgcmCalibrateTract.
648 
649  Parameters
650  ----------
651  dataRefDict : `dict`
652  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
653  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
654  dataRef dictionary with keys:
655 
656  ``"camera"``
657  Camera object (`lsst.afw.cameraGeom.Camera`)
658  ``"fgcmLookUpTable"``
659  dataRef for the FGCM look-up table.
660  tract : `int`
661  Tract number
662  visitCat : `lsst.afw.table.BaseCatalog`
663  FGCM visitCat from `FgcmBuildStarsTask`
664  zptCat : `lsst.afw.table.BaseCatalog`
665  FGCM zeropoint catalog from `FgcmFitCycleTask`
666  atmCat : `lsst.afw.table.BaseCatalog`
667  FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
668  stdCat : `lsst.afw.table.SimpleCatalog`
669  FGCM standard star catalog from `FgcmFitCycleTask`
670  fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
671  Configuration object from `FgcmBuildStarsTask`
672  returnCatalogs : `bool`, optional
673  Return photoCalibs as per-visit exposure catalogs.
674  butler: `lsst.daf.persistence.Butler`, optional
675  Gen2 butler used for reference star outputs
676 
677  Returns
678  -------
679  retStruct : `lsst.pipe.base.Struct`
680  Output structure with keys:
681 
682  offsets : `np.ndarray`
683  Final reference offsets, per band.
684  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
685  Generator that returns (visit, transmissionCurve) tuples.
686  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
687  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
688  (returned if returnCatalogs is False).
689  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
690  Generator that returns (visit, exposureCatalog) tuples.
691  (returned if returnCatalogs is True).
692  """
693  physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
694 
695  md = stdCat.getMetadata()
696  bands = md.getArray('BANDS')
697 
698  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
699  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
700  "in fgcmBuildStarsTask.")
701 
702  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
703  self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
704 
705  if self.config.doReferenceCalibration:
706  lutCat = dataRefDict['fgcmLookUpTable'].get()
707  offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap)
708  else:
709  offsets = np.zeros(len(bands))
710 
711  if self.config.doRefcatOutput and butler is not None:
712  # Create a special config that has the tract number in it
713  datasetConfig = copy.copy(self.config.datasetConfig)
714  datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
715  tract)
716  self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
717 
718  if self.config.doZeropointOutput:
719  pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
720  physicalFilterMap, returnCatalogs=returnCatalogs)
721  else:
722  pcgen = None
723 
724  if self.config.doAtmosphereOutput:
725  atmgen = self._outputAtmospheres(dataRefDict, atmCat)
726  else:
727  atmgen = None
728 
729  retStruct = pipeBase.Struct(offsets=offsets,
730  atmospheres=atmgen)
731  if returnCatalogs:
732  retStruct.photoCalibCatalogs = pcgen
733  else:
734  retStruct.photoCalibs = pcgen
735 
736  return retStruct
737 
738  def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
739  """
740  Compute offsets relative to a reference catalog.
741 
742  This method splits the star catalog into healpix pixels
743  and computes the calibration transfer for a sample of
744  these pixels to approximate the 'absolute' calibration
745  values (on for each band) to apply to transfer the
746  absolute scale.
747 
748  Parameters
749  ----------
750  stdCat : `lsst.afw.table.SimpleCatalog`
751  FGCM standard stars
752  lutCat : `lsst.afw.table.SimpleCatalog`
753  FGCM Look-up table
754  physicalFilterMap : `dict`
755  Dictionary of mappings from physical filter to FGCM band.
756  bands : `list` [`str`]
757  List of band names from FGCM output
758  Returns
759  -------
760  offsets : `numpy.array` of floats
761  Per band zeropoint offsets
762  """
763 
764  # Only use stars that are observed in all the bands that were actually used
765  # This will ensure that we use the same healpix pixels for the absolute
766  # calibration of each band.
767  minObs = stdCat['ngood'].min(axis=1)
768 
769  goodStars = (minObs >= 1)
770  stdCat = stdCat[goodStars]
771 
772  self.log.info("Found %d stars with at least 1 good observation in each band" %
773  (len(stdCat)))
774 
775  # Associate each band with the appropriate physicalFilter and make
776  # filterLabels
777  filterLabels = []
778 
779  lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',')
780  lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',')
781  physicalFilterMapBands = list(physicalFilterMap.values())
782  physicalFilterMapFilters = list(physicalFilterMap.keys())
783  for band in bands:
784  # Find a physical filter associated from the band by doing
785  # a reverse lookup on the physicalFilterMap dict
786  physicalFilterMapIndex = physicalFilterMapBands.index(band)
787  physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
788  # Find the appropriate fgcm standard physicalFilter
789  lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
790  stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
791  filterLabels.append(afwImage.FilterLabel(band=band,
792  physical=stdPhysicalFilter))
793 
794  # We have to make a table for each pixel with flux/fluxErr
795  # This is a temporary table generated for input to the photoCal task.
796  # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
797  # have had chromatic corrections applied to get to the standard system
798  # specified by the atmosphere/instrumental parameters), nor are they
799  # in Jansky (since they don't have a proper absolute calibration: the overall
800  # zeropoint is estimated from the telescope size, etc.)
801  sourceMapper = afwTable.SchemaMapper(stdCat.schema)
802  sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
803  sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
804  doc="instrumental flux (counts)")
805  sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
806  doc="instrumental flux error (counts)")
807  badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
808  type='Flag',
809  doc="bad flag")
810 
811  # Split up the stars
812  # Note that there is an assumption here that the ra/dec coords stored
813  # on-disk are in radians, and therefore that starObs['coord_ra'] /
814  # starObs['coord_dec'] return radians when used as an array of numpy float64s.
815  theta = np.pi/2. - stdCat['coord_dec']
816  phi = stdCat['coord_ra']
817 
818  ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
819  h, rev = esutil.stat.histogram(ipring, rev=True)
820 
821  gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
822 
823  self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
824  (gdpix.size,
825  self.config.referencePixelizationNside,
826  self.config.referencePixelizationMinStars))
827 
828  if gdpix.size < self.config.referencePixelizationNPixels:
829  self.log.warning("Found fewer good pixels (%d) than preferred in configuration (%d)" %
830  (gdpix.size, self.config.referencePixelizationNPixels))
831  else:
832  # Sample out the pixels we want to use
833  gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
834 
835  results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
836  ('nstar', 'i4', len(bands)),
837  ('nmatch', 'i4', len(bands)),
838  ('zp', 'f4', len(bands)),
839  ('zpErr', 'f4', len(bands))])
840  results['hpix'] = ipring[rev[rev[gdpix]]]
841 
842  # We need a boolean index to deal with catalogs...
843  selected = np.zeros(len(stdCat), dtype=bool)
844 
845  refFluxFields = [None]*len(bands)
846 
847  for p_index, pix in enumerate(gdpix):
848  i1a = rev[rev[pix]: rev[pix + 1]]
849 
850  # the stdCat afwTable can only be indexed with boolean arrays,
851  # and not numpy index arrays (see DM-16497). This little trick
852  # converts the index array into a boolean array
853  selected[:] = False
854  selected[i1a] = True
855 
856  for b_index, filterLabel in enumerate(filterLabels):
857  struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index,
858  filterLabel, stdCat,
859  selected, refFluxFields)
860  results['nstar'][p_index, b_index] = len(i1a)
861  results['nmatch'][p_index, b_index] = len(struct.arrays.refMag)
862  results['zp'][p_index, b_index] = struct.zp
863  results['zpErr'][p_index, b_index] = struct.sigma
864 
865  # And compute the summary statistics
866  offsets = np.zeros(len(bands))
867 
868  for b_index, band in enumerate(bands):
869  # make configurable
870  ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch)
871  offsets[b_index] = np.median(results['zp'][ok, b_index])
872  # use median absolute deviation to estimate Normal sigma
873  # see https://en.wikipedia.org/wiki/Median_absolute_deviation
874  madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index]))
875  self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f",
876  band, offsets[b_index], madSigma)
877 
878  return offsets
879 
880  def _computeOffsetOneBand(self, sourceMapper, badStarKey,
881  b_index, filterLabel, stdCat, selected, refFluxFields):
882  """
883  Compute the zeropoint offset between the fgcm stdCat and the reference
884  stars for one pixel in one band
885 
886  Parameters
887  ----------
888  sourceMapper : `lsst.afw.table.SchemaMapper`
889  Mapper to go from stdCat to calibratable catalog
890  badStarKey : `lsst.afw.table.Key`
891  Key for the field with bad stars
892  b_index : `int`
893  Index of the band in the star catalog
894  filterLabel : `lsst.afw.image.FilterLabel`
895  filterLabel with band and physical filter
896  stdCat : `lsst.afw.table.SimpleCatalog`
897  FGCM standard stars
898  selected : `numpy.array(dtype=bool)`
899  Boolean array of which stars are in the pixel
900  refFluxFields : `list`
901  List of names of flux fields for reference catalog
902  """
903 
904  sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
905  sourceCat.reserve(selected.sum())
906  sourceCat.extend(stdCat[selected], mapper=sourceMapper)
907  sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5))
908  sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index]
909  * sourceCat['instFlux'])
910  # Make sure we only use stars that have valid measurements
911  # (This is perhaps redundant with requirements above that the
912  # stars be observed in all bands, but it can't hurt)
913  badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0)
914  for rec in sourceCat[badStar]:
915  rec.set(badStarKey, True)
916 
917  exposure = afwImage.ExposureF()
918  exposure.setFilterLabel(filterLabel)
919 
920  if refFluxFields[b_index] is None:
921  # Need to find the flux field in the reference catalog
922  # to work around limitations of DirectMatch in PhotoCal
923  ctr = stdCat[0].getCoord()
924  rad = 0.05*lsst.geom.degrees
925  refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel)
926  refFluxFields[b_index] = refDataTest.fluxField
927 
928  # Make a copy of the config so that we can modify it
929  calConfig = copy.copy(self.config.photoCal.value)
930  calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index]
931  calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err'
932  calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
933  config=calConfig,
934  schema=sourceCat.getSchema())
935 
936  struct = calTask.run(exposure, sourceCat)
937 
938  return struct
939 
940  def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
941  """
942  Output standard stars in indexed reference catalog format.
943  This is not currently supported in Gen3.
944 
945  Parameters
946  ----------
947  butler : `lsst.daf.persistence.Butler`
948  stdCat : `lsst.afw.table.SimpleCatalog`
949  FGCM standard star catalog from fgcmFitCycleTask
950  offsets : `numpy.array` of floats
951  Per band zeropoint offsets
952  bands : `list` [`str`]
953  List of band names from FGCM output
954  datasetConfig : `lsst.meas.algorithms.DatasetConfig`
955  Config for reference dataset
956  """
957 
958  self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
959 
960  indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
961  self.config.datasetConfig.indexer.active)
962 
963  # We determine the conversion from the native units (typically radians) to
964  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
965  # numpy arrays rather than Angles, which would we approximately 600x slower.
966  # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
967  # (as Angles) for input
968  conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
969  indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv,
970  stdCat['coord_dec']*conv))
971 
972  formattedCat = self._formatCatalog(stdCat, offsets, bands)
973 
974  # Write the master schema
975  dataId = indexer.makeDataId('master_schema',
976  datasetConfig.ref_dataset_name)
977  masterCat = afwTable.SimpleCatalog(formattedCat.schema)
978  addRefCatMetadata(masterCat)
979  butler.put(masterCat, 'ref_cat', dataId=dataId)
980 
981  # Break up the pixels using a histogram
982  h, rev = esutil.stat.histogram(indices, rev=True)
983  gd, = np.where(h > 0)
984  selected = np.zeros(len(formattedCat), dtype=bool)
985  for i in gd:
986  i1a = rev[rev[i]: rev[i + 1]]
987 
988  # the formattedCat afwTable can only be indexed with boolean arrays,
989  # and not numpy index arrays (see DM-16497). This little trick
990  # converts the index array into a boolean array
991  selected[:] = False
992  selected[i1a] = True
993 
994  # Write the individual pixel
995  dataId = indexer.makeDataId(indices[i1a[0]],
996  datasetConfig.ref_dataset_name)
997  butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
998 
999  # And save the dataset configuration
1000  dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name)
1001  butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
1002 
1003  self.log.info("Done outputting standard stars.")
1004 
1005  def _formatCatalog(self, fgcmStarCat, offsets, bands):
1006  """
1007  Turn an FGCM-formatted star catalog, applying zeropoint offsets.
1008 
1009  Parameters
1010  ----------
1011  fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
1012  SimpleCatalog as output by fgcmcal
1013  offsets : `list` with len(self.bands) entries
1014  Zeropoint offsets to apply
1015  bands : `list` [`str`]
1016  List of band names from FGCM output
1017 
1018  Returns
1019  -------
1020  formattedCat: `lsst.afw.table.SimpleCatalog`
1021  SimpleCatalog suitable for using as a reference catalog
1022  """
1023 
1024  sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1025  minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1026  addCentroid=False,
1027  addIsResolved=True,
1028  coordErrDim=0)
1029  sourceMapper.addMinimalSchema(minSchema)
1030  for band in bands:
1031  sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
1032  sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
1033  sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
1034 
1035  formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1036  formattedCat.reserve(len(fgcmStarCat))
1037  formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1038 
1039  # Note that we don't have to set `resolved` because the default is False
1040 
1041  for b, band in enumerate(bands):
1042  mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1043  # We want fluxes in nJy from calibrated AB magnitudes
1044  # (after applying offset). Updated after RFC-549 and RFC-575.
1045  flux = (mag*units.ABmag).to_value(units.nJy)
1046  fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
1047 
1048  formattedCat['%s_flux' % (band)][:] = flux
1049  formattedCat['%s_fluxErr' % (band)][:] = fluxErr
1050  formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
1051  formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
1052  formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
1053 
1054  addRefCatMetadata(formattedCat)
1055 
1056  return formattedCat
1057 
1058  def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1059  physicalFilterMap, returnCatalogs=True,
1060  tract=None):
1061  """Output the zeropoints in fgcm_photoCalib format.
1062 
1063  Parameters
1064  ----------
1065  camera : `lsst.afw.cameraGeom.Camera`
1066  Camera from the butler.
1067  zptCat : `lsst.afw.table.BaseCatalog`
1068  FGCM zeropoint catalog from `FgcmFitCycleTask`.
1069  visitCat : `lsst.afw.table.BaseCatalog`
1070  FGCM visitCat from `FgcmBuildStarsTask`.
1071  offsets : `numpy.array`
1072  Float array of absolute calibration offsets, one for each filter.
1073  bands : `list` [`str`]
1074  List of band names from FGCM output.
1075  physicalFilterMap : `dict`
1076  Dictionary of mappings from physical filter to FGCM band.
1077  returnCatalogs : `bool`, optional
1078  Return photoCalibs as per-visit exposure catalogs.
1079  tract: `int`, optional
1080  Tract number to output. Default is None (global calibration)
1081 
1082  Returns
1083  -------
1084  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1085  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1086  (returned if returnCatalogs is False).
1087  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1088  Generator that returns (visit, exposureCatalog) tuples.
1089  (returned if returnCatalogs is True).
1090  """
1091  # Select visit/ccds where we have a calibration
1092  # This includes ccds where we were able to interpolate from neighboring
1093  # ccds.
1094  cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
1095  selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
1096  & (zptCat['fgcmZptVar'] > 0.0))
1097 
1098  # Log warnings for any visit which has no valid zeropoints
1099  badVisits = np.unique(zptCat['visit'][~selected])
1100  goodVisits = np.unique(zptCat['visit'][selected])
1101  allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1102  for allBadVisit in allBadVisits:
1103  self.log.warning(f'No suitable photoCalib for visit {allBadVisit}')
1104 
1105  # Get a mapping from filtername to the offsets
1106  offsetMapping = {}
1107  for f in physicalFilterMap:
1108  # Not every filter in the map will necesarily have a band.
1109  if physicalFilterMap[f] in bands:
1110  offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1111 
1112  # Get a mapping from "ccd" to the ccd index used for the scaling
1113  ccdMapping = {}
1114  for ccdIndex, detector in enumerate(camera):
1115  ccdMapping[detector.getId()] = ccdIndex
1116 
1117  # And a mapping to get the flat-field scaling values
1118  scalingMapping = {}
1119  for rec in visitCat:
1120  scalingMapping[rec['visit']] = rec['scaling']
1121 
1122  if self.config.doComposeWcsJacobian:
1123  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
1124 
1125  # The zptCat is sorted by visit, which is useful
1126  lastVisit = -1
1127  zptVisitCatalog = None
1128 
1129  metadata = dafBase.PropertyList()
1130  metadata.add("COMMENT", "Catalog id is detector id, sorted.")
1131  metadata.add("COMMENT", "Only detectors with data have entries.")
1132 
1133  for rec in zptCat[selected]:
1134  # Retrieve overall scaling
1135  scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
1136 
1137  # The postCalibrationOffset describe any zeropoint offsets
1138  # to apply after the fgcm calibration. The first part comes
1139  # from the reference catalog match (used in testing). The
1140  # second part comes from the mean chromatic correction
1141  # (if configured).
1142  postCalibrationOffset = offsetMapping[rec['filtername']]
1143  if self.config.doApplyMeanChromaticCorrection:
1144  postCalibrationOffset += rec['fgcmDeltaChrom']
1145 
1146  fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
1147  rec['fgcmfZptChebXyMax'])
1148  # Convert from FGCM AB to nJy
1149  fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
1150  rec['fgcmfZptChebXyMax'],
1151  offset=postCalibrationOffset,
1152  scaling=scaling)
1153 
1154  if self.config.doComposeWcsJacobian:
1155 
1156  fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']],
1157  fgcmSuperStarField,
1158  fgcmZptField])
1159  else:
1160  # The photoCalib is just the product of the fgcmSuperStarField and the
1161  # fgcmZptField
1162  fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1163 
1164  # The "mean" calibration will be set to the center of the ccd for reference
1165  calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1166  calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
1167  photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1168  calibrationErr=calibErr,
1169  calibration=fgcmField,
1170  isConstant=False)
1171 
1172  if not returnCatalogs:
1173  # Return individual photoCalibs
1174  yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib)
1175  else:
1176  # Return full per-visit exposure catalogs
1177  if rec['visit'] != lastVisit:
1178  # This is a new visit. If the last visit was not -1, yield
1179  # the ExposureCatalog
1180  if lastVisit > -1:
1181  # ensure that the detectors are in sorted order, for fast lookups
1182  zptVisitCatalog.sort()
1183  yield (int(lastVisit), zptVisitCatalog)
1184  else:
1185  # We need to create a new schema
1186  zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1187  zptExpCatSchema.addField('visit', type='I', doc='Visit number')
1188 
1189  # And start a new one
1190  zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1191  zptVisitCatalog.setMetadata(metadata)
1192 
1193  lastVisit = int(rec['visit'])
1194 
1195  catRecord = zptVisitCatalog.addNew()
1196  catRecord['id'] = int(rec['detector'])
1197  catRecord['visit'] = rec['visit']
1198  catRecord.setPhotoCalib(photoCalib)
1199 
1200  # Final output of last exposure catalog
1201  if returnCatalogs:
1202  # ensure that the detectors are in sorted order, for fast lookups
1203  zptVisitCatalog.sort()
1204  yield (int(lastVisit), zptVisitCatalog)
1205 
1206  def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1207  """
1208  Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1209  and scaling.
1210 
1211  Parameters
1212  ----------
1213  coefficients: `numpy.array`
1214  Flattened array of chebyshev coefficients
1215  xyMax: `list` of length 2
1216  Maximum x and y of the chebyshev bounding box
1217  offset: `float`, optional
1218  Absolute calibration offset. Default is 0.0
1219  scaling: `float`, optional
1220  Flat scaling value from fgcmBuildStars. Default is 1.0
1221 
1222  Returns
1223  -------
1224  boundedField: `lsst.afw.math.ChebyshevBoundedField`
1225  """
1226 
1227  orderPlus1 = int(np.sqrt(coefficients.size))
1228  pars = np.zeros((orderPlus1, orderPlus1))
1229 
1230  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1231  lsst.geom.Point2I(*xyMax))
1232 
1233  pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1234  * (10.**(offset/-2.5))*scaling)
1235 
1236  boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1237 
1238  return boundedField
1239 
1240  def _outputAtmospheres(self, dataRefDict, atmCat):
1241  """
1242  Output the atmospheres.
1243 
1244  Parameters
1245  ----------
1246  dataRefDict : `dict`
1247  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1248  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1249  dataRef dictionary with keys:
1250 
1251  ``"fgcmLookUpTable"``
1252  dataRef for the FGCM look-up table.
1253  atmCat : `lsst.afw.table.BaseCatalog`
1254  FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1255 
1256  Returns
1257  -------
1258  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1259  Generator that returns (visit, transmissionCurve) tuples.
1260  """
1261  # First, we need to grab the look-up table and key info
1262  lutCat = dataRefDict['fgcmLookUpTable'].get()
1263 
1264  atmosphereTableName = lutCat[0]['tablename']
1265  elevation = lutCat[0]['elevation']
1266  atmLambda = lutCat[0]['atmLambda']
1267  lutCat = None
1268 
1269  # Make the atmosphere table if possible
1270  try:
1271  atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1272  atmTable.loadTable()
1273  except IOError:
1274  atmTable = None
1275 
1276  if atmTable is None:
1277  # Try to use MODTRAN instead
1278  try:
1279  modGen = fgcm.ModtranGenerator(elevation)
1280  lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1281  lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1282  except (ValueError, IOError) as e:
1283  raise RuntimeError("FGCM look-up-table generated with modtran, "
1284  "but modtran not configured to run.") from e
1285 
1286  zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
1287 
1288  for i, visit in enumerate(atmCat['visit']):
1289  if atmTable is not None:
1290  # Interpolate the atmosphere table
1291  atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
1292  pwv=atmCat[i]['pwv'],
1293  o3=atmCat[i]['o3'],
1294  tau=atmCat[i]['tau'],
1295  alpha=atmCat[i]['alpha'],
1296  zenith=zenith[i],
1297  ctranslamstd=[atmCat[i]['cTrans'],
1298  atmCat[i]['lamStd']])
1299  else:
1300  # Run modtran
1301  modAtm = modGen(pmb=atmCat[i]['pmb'],
1302  pwv=atmCat[i]['pwv'],
1303  o3=atmCat[i]['o3'],
1304  tau=atmCat[i]['tau'],
1305  alpha=atmCat[i]['alpha'],
1306  zenith=zenith[i],
1307  lambdaRange=lambdaRange,
1308  lambdaStep=lambdaStep,
1309  ctranslamstd=[atmCat[i]['cTrans'],
1310  atmCat[i]['lamStd']])
1311  atmVals = modAtm['COMBINED']
1312 
1313  # Now need to create something to persist...
1314  curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1315  wavelengths=atmLambda,
1316  throughputAtMin=atmVals[0],
1317  throughputAtMax=atmVals[-1])
1318 
1319  yield (int(visit), curve)
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:494