lsst.fgcmcal  22.0.1+1ef34551f5
fgcmOutputProducts.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Make the final fgcmcal output products.
24 
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
32 offset per band.
33 """
34 import sys
35 import traceback
36 import copy
37 
38 import numpy as np
39 import healpy as hp
40 import esutil
41 from astropy import units
42 
43 import lsst.daf.base as dafBase
44 import lsst.pex.config as pexConfig
45 import lsst.pipe.base as pipeBase
46 from lsst.pipe.base import connectionTypes
47 from lsst.afw.image import TransmissionCurve
48 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
49 from lsst.meas.algorithms import ReferenceObjectLoader
50 from lsst.pipe.tasks.photoCal import PhotoCalTask
51 import lsst.geom
52 import lsst.afw.image as afwImage
53 import lsst.afw.math as afwMath
54 import lsst.afw.table as afwTable
55 from lsst.meas.algorithms import IndexerRegistry
56 from lsst.meas.algorithms import DatasetConfig
57 from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
58 
59 from .utilities import computeApproxPixelAreaFields
60 from .utilities import lookupStaticCalibrations
61 
62 import fgcm
63 
64 __all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
65 
66 
67 class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
68  dimensions=("instrument",),
69  defaultTemplates={"cycleNumber": "0"}):
70  camera = connectionTypes.PrerequisiteInput(
71  doc="Camera instrument",
72  name="camera",
73  storageClass="Camera",
74  dimensions=("instrument",),
75  lookupFunction=lookupStaticCalibrations,
76  isCalibration=True,
77  )
78 
79  fgcmLookUpTable = connectionTypes.PrerequisiteInput(
80  doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
81  "chromatic corrections."),
82  name="fgcmLookUpTable",
83  storageClass="Catalog",
84  dimensions=("instrument",),
85  deferLoad=True,
86  )
87 
88  fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
89  doc="Catalog of visit information for fgcm",
90  name="fgcmVisitCatalog",
91  storageClass="Catalog",
92  dimensions=("instrument",),
93  deferLoad=True,
94  )
95 
96  fgcmStandardStars = connectionTypes.PrerequisiteInput(
97  doc="Catalog of standard star data from fgcm fit",
98  name="fgcmStandardStars{cycleNumber}",
99  storageClass="SimpleCatalog",
100  dimensions=("instrument",),
101  deferLoad=True,
102  )
103 
104  fgcmZeropoints = connectionTypes.PrerequisiteInput(
105  doc="Catalog of zeropoints from fgcm fit",
106  name="fgcmZeropoints{cycleNumber}",
107  storageClass="Catalog",
108  dimensions=("instrument",),
109  deferLoad=True,
110  )
111 
112  fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
113  doc="Catalog of atmosphere parameters from fgcm fit",
114  name="fgcmAtmosphereParameters{cycleNumber}",
115  storageClass="Catalog",
116  dimensions=("instrument",),
117  deferLoad=True,
118  )
119 
120  refCat = connectionTypes.PrerequisiteInput(
121  doc="Reference catalog to use for photometric calibration",
122  name="cal_ref_cat",
123  storageClass="SimpleCatalog",
124  dimensions=("skypix",),
125  deferLoad=True,
126  multiple=True,
127  )
128 
129  fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
130  doc="Config used to build FGCM input stars",
131  name="fgcmBuildStarsTable_config",
132  storageClass="Config",
133  )
134 
135  fgcmPhotoCalib = connectionTypes.Output(
136  doc=("Per-visit photometric calibrations derived from fgcm calibration. "
137  "These catalogs use detector id for the id and are sorted for "
138  "fast lookups of a detector."),
139  name="fgcmPhotoCalibCatalog",
140  storageClass="ExposureCatalog",
141  dimensions=("instrument", "visit",),
142  multiple=True,
143  )
144 
145  fgcmTransmissionAtmosphere = connectionTypes.Output(
146  doc="Per-visit atmosphere transmission files produced from fgcm calibration",
147  name="transmission_atmosphere_fgcm",
148  storageClass="TransmissionCurve",
149  dimensions=("instrument",
150  "visit",),
151  multiple=True,
152  )
153 
154  fgcmOffsets = connectionTypes.Output(
155  doc="Per-band offsets computed from doReferenceCalibration",
156  name="fgcmReferenceCalibrationOffsets",
157  storageClass="Catalog",
158  dimensions=("instrument",),
159  multiple=False,
160  )
161 
162  def __init__(self, *, config=None):
163  super().__init__(config=config)
164 
165  if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
166  raise ValueError("cycleNumber must be of integer format")
167  if config.connections.refCat != config.refObjLoader.ref_dataset_name:
168  raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name")
169 
170  if config.doRefcatOutput:
171  raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
172 
173  if not config.doReferenceCalibration:
174  self.prerequisiteInputs.remove("refCat")
175  if not config.doAtmosphereOutput:
176  self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
177  if not config.doZeropointOutput:
178  self.prerequisiteInputs.remove("fgcmZeropoints")
179  if not config.doReferenceCalibration:
180  self.outputs.remove("fgcmOffsets")
181 
182 
183 class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
184  pipelineConnections=FgcmOutputProductsConnections):
185  """Config for FgcmOutputProductsTask"""
186 
187  cycleNumber = pexConfig.Field(
188  doc="Final fit cycle from FGCM fit",
189  dtype=int,
190  default=None,
191  )
192 
193  # The following fields refer to calibrating from a reference
194  # catalog, but in the future this might need to be expanded
195  doReferenceCalibration = pexConfig.Field(
196  doc=("Transfer 'absolute' calibration from reference catalog? "
197  "This afterburner step is unnecessary if reference stars "
198  "were used in the full fit in FgcmFitCycleTask."),
199  dtype=bool,
200  default=False,
201  )
202  doRefcatOutput = pexConfig.Field(
203  doc="Output standard stars in reference catalog format",
204  dtype=bool,
205  default=True,
206  )
207  doAtmosphereOutput = pexConfig.Field(
208  doc="Output atmospheres in transmission_atmosphere_fgcm format",
209  dtype=bool,
210  default=True,
211  )
212  doZeropointOutput = pexConfig.Field(
213  doc="Output zeropoints in fgcm_photoCalib format",
214  dtype=bool,
215  default=True,
216  )
217  doComposeWcsJacobian = pexConfig.Field(
218  doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
219  dtype=bool,
220  default=True,
221  )
222  doApplyMeanChromaticCorrection = pexConfig.Field(
223  doc="Apply the mean chromatic correction to the zeropoints?",
224  dtype=bool,
225  default=True,
226  )
227  refObjLoader = pexConfig.ConfigurableField(
228  target=LoadIndexedReferenceObjectsTask,
229  doc="reference object loader for 'absolute' photometric calibration",
230  )
231  photoCal = pexConfig.ConfigurableField(
232  target=PhotoCalTask,
233  doc="task to perform 'absolute' calibration",
234  )
235  referencePixelizationNside = pexConfig.Field(
236  doc="Healpix nside to pixelize catalog to compare to reference catalog",
237  dtype=int,
238  default=64,
239  )
240  referencePixelizationMinStars = pexConfig.Field(
241  doc=("Minimum number of stars per healpix pixel to select for comparison"
242  "to the specified reference catalog"),
243  dtype=int,
244  default=200,
245  )
246  referenceMinMatch = pexConfig.Field(
247  doc="Minimum number of stars matched to reference catalog to be used in statistics",
248  dtype=int,
249  default=50,
250  )
251  referencePixelizationNPixels = pexConfig.Field(
252  doc=("Number of healpix pixels to sample to do comparison. "
253  "Doing too many will take a long time and not yield any more "
254  "precise results because the final number is the median offset "
255  "(per band) from the set of pixels."),
256  dtype=int,
257  default=100,
258  )
259  datasetConfig = pexConfig.ConfigField(
260  dtype=DatasetConfig,
261  doc="Configuration for writing/reading ingested catalog",
262  )
263 
264  def setDefaults(self):
265  pexConfig.Config.setDefaults(self)
266 
267  # In order to transfer the "absolute" calibration from a reference
268  # catalog to the relatively calibrated FGCM standard stars (one number
269  # per band), we use the PhotoCalTask to match stars in a sample of healpix
270  # pixels. These basic settings ensure that only well-measured, good stars
271  # from the source and reference catalogs are used for the matching.
272 
273  # applyColorTerms needs to be False if doReferenceCalibration is False,
274  # as is the new default after DM-16702
275  self.photoCal.applyColorTerms = False
276  self.photoCal.fluxField = 'instFlux'
277  self.photoCal.magErrFloor = 0.003
278  self.photoCal.match.referenceSelection.doSignalToNoise = True
279  self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
280  self.photoCal.match.sourceSelection.doSignalToNoise = True
281  self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
282  self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
283  self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
284  self.photoCal.match.sourceSelection.doFlags = True
285  self.photoCal.match.sourceSelection.flags.good = []
286  self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
287  self.photoCal.match.sourceSelection.doUnresolved = False
288  self.datasetConfig.ref_dataset_name = 'fgcm_stars'
289  self.datasetConfig.format_version = 1
290 
291  def validate(self):
292  super().validate()
293 
294  # Force the connections to conform with cycleNumber
295  self.connections.cycleNumber = str(self.cycleNumber)
296 
297 
298 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
299  """Subclass of TaskRunner for fgcmOutputProductsTask
300 
301  fgcmOutputProductsTask.run() takes one argument, the butler, and
302  does not run on any data in the repository.
303  This runner does not use any parallelization.
304  """
305 
306  @staticmethod
307  def getTargetList(parsedCmd):
308  """
309  Return a list with one element, the butler.
310  """
311  return [parsedCmd.butler]
312 
313  def __call__(self, butler):
314  """
315  Parameters
316  ----------
317  butler: `lsst.daf.persistence.Butler`
318 
319  Returns
320  -------
321  exitStatus: `list` with `pipeBase.Struct`
322  exitStatus (0: success; 1: failure)
323  if self.doReturnResults also
324  results (`np.array` with absolute zeropoint offsets)
325  """
326  task = self.TaskClass(butler=butler, config=self.config, log=self.log)
327 
328  exitStatus = 0
329  if self.doRaise:
330  results = task.runDataRef(butler)
331  else:
332  try:
333  results = task.runDataRef(butler)
334  except Exception as e:
335  exitStatus = 1
336  task.log.fatal("Failed: %s" % e)
337  if not isinstance(e, pipeBase.TaskError):
338  traceback.print_exc(file=sys.stderr)
339 
340  task.writeMetadata(butler)
341 
342  if self.doReturnResults:
343  # The results here are the zeropoint offsets for each band
344  return [pipeBase.Struct(exitStatus=exitStatus,
345  results=results)]
346  else:
347  return [pipeBase.Struct(exitStatus=exitStatus)]
348 
349  def run(self, parsedCmd):
350  """
351  Run the task, with no multiprocessing
352 
353  Parameters
354  ----------
355  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
356  """
357 
358  resultList = []
359 
360  if self.precall(parsedCmd):
361  targetList = self.getTargetList(parsedCmd)
362  # make sure that we only get 1
363  resultList = self(targetList[0])
364 
365  return resultList
366 
367 
368 class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
369  """
370  Output products from FGCM global calibration.
371  """
372 
373  ConfigClass = FgcmOutputProductsConfig
374  RunnerClass = FgcmOutputProductsRunner
375  _DefaultName = "fgcmOutputProducts"
376 
377  def __init__(self, butler=None, **kwargs):
378  super().__init__(**kwargs)
379 
380  # no saving of metadata for now
381  def _getMetadataName(self):
382  return None
383 
384  def runQuantum(self, butlerQC, inputRefs, outputRefs):
385  dataRefDict = {}
386  dataRefDict['camera'] = butlerQC.get(inputRefs.camera)
387  dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
388  dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
389  dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
390 
391  if self.config.doZeropointOutput:
392  dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
393  photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
394  photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
395 
396  if self.config.doAtmosphereOutput:
397  dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
398  atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
399  atmRef in outputRefs.fgcmTransmissionAtmosphere}
400 
401  if self.config.doReferenceCalibration:
402  refConfig = self.config.refObjLoader
403  self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
404  for ref in inputRefs.refCat],
405  refCats=butlerQC.get(inputRefs.refCat),
406  config=refConfig,
407  log=self.log)
408  else:
409  self.refObjLoader = None
410 
411  dataRefDict['fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
412 
413  fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
414  physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
415 
416  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
417  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
418  "in fgcmBuildStarsTask.")
419  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
420  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
421 
422  struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=True)
423 
424  # Output the photoCalib exposure catalogs
425  if struct.photoCalibCatalogs is not None:
426  self.log.info("Outputting photoCalib catalogs.")
427  for visit, expCatalog in struct.photoCalibCatalogs:
428  butlerQC.put(expCatalog, photoCalibRefDict[visit])
429  self.log.info("Done outputting photoCalib catalogs.")
430 
431  # Output the atmospheres
432  if struct.atmospheres is not None:
433  self.log.info("Outputting atmosphere transmission files.")
434  for visit, atm in struct.atmospheres:
435  butlerQC.put(atm, atmRefDict[visit])
436  self.log.info("Done outputting atmosphere files.")
437 
438  if self.config.doReferenceCalibration:
439  # Turn offset into simple catalog for persistence if necessary
440  schema = afwTable.Schema()
441  schema.addField('offset', type=np.float64,
442  doc="Post-process calibration offset (mag)")
443  offsetCat = afwTable.BaseCatalog(schema)
444  offsetCat.resize(len(struct.offsets))
445  offsetCat['offset'][:] = struct.offsets
446 
447  butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
448 
449  return
450 
451  @pipeBase.timeMethod
452  def runDataRef(self, butler):
453  """
454  Make FGCM output products for use in the stack
455 
456  Parameters
457  ----------
458  butler: `lsst.daf.persistence.Butler`
459  cycleNumber: `int`
460  Final fit cycle number, override config.
461 
462  Returns
463  -------
464  offsets: `lsst.pipe.base.Struct`
465  A structure with array of zeropoint offsets
466 
467  Raises
468  ------
469  RuntimeError:
470  Raised if any one of the following is true:
471 
472  - butler cannot find "fgcmBuildStars_config" or
473  "fgcmBuildStarsTable_config".
474  - butler cannot find "fgcmFitCycle_config".
475  - "fgcmFitCycle_config" does not refer to
476  `self.config.cycleNumber`.
477  - butler cannot find "fgcmAtmosphereParameters" and
478  `self.config.doAtmosphereOutput` is `True`.
479  - butler cannot find "fgcmStandardStars" and
480  `self.config.doReferenceCalibration` is `True` or
481  `self.config.doRefcatOutput` is `True`.
482  - butler cannot find "fgcmZeropoints" and
483  `self.config.doZeropointOutput` is `True`.
484  """
485  if self.config.doReferenceCalibration:
486  # We need the ref obj loader to get the flux field
487  self.makeSubtask("refObjLoader", butler=butler)
488 
489  # Check to make sure that the fgcmBuildStars config exists, to retrieve
490  # the visit and ccd dataset tags
491  if not butler.datasetExists('fgcmBuildStarsTable_config') and \
492  not butler.datasetExists('fgcmBuildStars_config'):
493  raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
494  "which is prereq for fgcmOutputProducts")
495 
496  if butler.datasetExists('fgcmBuildStarsTable_config'):
497  fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
498  else:
499  fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
500  visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
501  ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
502  physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
503 
504  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
505  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
506  "in fgcmBuildStarsTask.")
507 
508  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
509  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
510 
511  # And make sure that the atmosphere was output properly
512  if (self.config.doAtmosphereOutput
513  and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
514  raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
515 
516  if not butler.datasetExists('fgcmStandardStars',
517  fgcmcycle=self.config.cycleNumber):
518  raise RuntimeError("Standard stars are missing for cycle %d." %
519  (self.config.cycleNumber))
520 
521  if (self.config.doZeropointOutput
522  and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
523  raise RuntimeError("Zeropoints are missing for cycle %d." %
524  (self.config.cycleNumber))
525 
526  dataRefDict = {}
527  # This is the _actual_ camera
528  dataRefDict['camera'] = butler.get('camera')
529  dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
530  dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
531  dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars',
532  fgcmcycle=self.config.cycleNumber)
533 
534  if self.config.doZeropointOutput:
535  dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints',
536  fgcmcycle=self.config.cycleNumber)
537  if self.config.doAtmosphereOutput:
538  dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters',
539  fgcmcycle=self.config.cycleNumber)
540 
541  struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False)
542 
543  if struct.photoCalibs is not None:
544  self.log.info("Outputting photoCalib files.")
545 
546  for visit, detector, physicalFilter, photoCalib in struct.photoCalibs:
547  butler.put(photoCalib, 'fgcm_photoCalib',
548  dataId={visitDataRefName: visit,
549  ccdDataRefName: detector,
550  'filter': physicalFilter})
551 
552  self.log.info("Done outputting photoCalib files.")
553 
554  if struct.atmospheres is not None:
555  self.log.info("Outputting atmosphere transmission files.")
556  for visit, atm in struct.atmospheres:
557  butler.put(atm, "transmission_atmosphere_fgcm",
558  dataId={visitDataRefName: visit})
559  self.log.info("Done outputting atmosphere transmissions.")
560 
561  return pipeBase.Struct(offsets=struct.offsets)
562 
563  def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
564  """Run the output products task.
565 
566  Parameters
567  ----------
568  dataRefDict : `dict`
569  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
570  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
571  dataRef dictionary with keys:
572 
573  ``"camera"``
574  Camera object (`lsst.afw.cameraGeom.Camera`)
575  ``"fgcmLookUpTable"``
576  dataRef for the FGCM look-up table.
577  ``"fgcmVisitCatalog"``
578  dataRef for visit summary catalog.
579  ``"fgcmStandardStars"``
580  dataRef for the output standard star catalog.
581  ``"fgcmZeropoints"``
582  dataRef for the zeropoint data catalog.
583  ``"fgcmAtmosphereParameters"``
584  dataRef for the atmosphere parameter catalog.
585  ``"fgcmBuildStarsTableConfig"``
586  Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
587  physicalFilterMap : `dict`
588  Dictionary of mappings from physical filter to FGCM band.
589  returnCatalogs : `bool`, optional
590  Return photoCalibs as per-visit exposure catalogs.
591  butler : `lsst.daf.persistence.Butler`, optional
592  Gen2 butler used for reference star outputs
593 
594  Returns
595  -------
596  retStruct : `lsst.pipe.base.Struct`
597  Output structure with keys:
598 
599  offsets : `np.ndarray`
600  Final reference offsets, per band.
601  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
602  Generator that returns (visit, transmissionCurve) tuples.
603  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
604  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
605  (returned if returnCatalogs is False).
606  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
607  Generator that returns (visit, exposureCatalog) tuples.
608  (returned if returnCatalogs is True).
609  """
610  stdCat = dataRefDict['fgcmStandardStars'].get()
611  md = stdCat.getMetadata()
612  bands = md.getArray('BANDS')
613 
614  if self.config.doReferenceCalibration:
615  lutCat = dataRefDict['fgcmLookUpTable'].get()
616  offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands)
617  else:
618  offsets = np.zeros(len(bands))
619 
620  # This is Gen2 only, and requires the butler.
621  if self.config.doRefcatOutput and butler is not None:
622  self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
623 
624  del stdCat
625 
626  if self.config.doZeropointOutput:
627  zptCat = dataRefDict['fgcmZeropoints'].get()
628  visitCat = dataRefDict['fgcmVisitCatalog'].get()
629 
630  pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
631  physicalFilterMap, returnCatalogs=returnCatalogs)
632  else:
633  pcgen = None
634 
635  if self.config.doAtmosphereOutput:
636  atmCat = dataRefDict['fgcmAtmosphereParameters'].get()
637  atmgen = self._outputAtmospheres(dataRefDict, atmCat)
638  else:
639  atmgen = None
640 
641  retStruct = pipeBase.Struct(offsets=offsets,
642  atmospheres=atmgen)
643  if returnCatalogs:
644  retStruct.photoCalibCatalogs = pcgen
645  else:
646  retStruct.photoCalibs = pcgen
647 
648  return retStruct
649 
650  def generateTractOutputProducts(self, dataRefDict, tract,
651  visitCat, zptCat, atmCat, stdCat,
652  fgcmBuildStarsConfig,
653  returnCatalogs=True,
654  butler=None):
655  """
656  Generate the output products for a given tract, as specified in the config.
657 
658  This method is here to have an alternate entry-point for
659  FgcmCalibrateTract.
660 
661  Parameters
662  ----------
663  dataRefDict : `dict`
664  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
665  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
666  dataRef dictionary with keys:
667 
668  ``"camera"``
669  Camera object (`lsst.afw.cameraGeom.Camera`)
670  ``"fgcmLookUpTable"``
671  dataRef for the FGCM look-up table.
672  tract : `int`
673  Tract number
674  visitCat : `lsst.afw.table.BaseCatalog`
675  FGCM visitCat from `FgcmBuildStarsTask`
676  zptCat : `lsst.afw.table.BaseCatalog`
677  FGCM zeropoint catalog from `FgcmFitCycleTask`
678  atmCat : `lsst.afw.table.BaseCatalog`
679  FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
680  stdCat : `lsst.afw.table.SimpleCatalog`
681  FGCM standard star catalog from `FgcmFitCycleTask`
682  fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
683  Configuration object from `FgcmBuildStarsTask`
684  returnCatalogs : `bool`, optional
685  Return photoCalibs as per-visit exposure catalogs.
686  butler: `lsst.daf.persistence.Butler`, optional
687  Gen2 butler used for reference star outputs
688 
689  Returns
690  -------
691  retStruct : `lsst.pipe.base.Struct`
692  Output structure with keys:
693 
694  offsets : `np.ndarray`
695  Final reference offsets, per band.
696  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
697  Generator that returns (visit, transmissionCurve) tuples.
698  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
699  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
700  (returned if returnCatalogs is False).
701  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
702  Generator that returns (visit, exposureCatalog) tuples.
703  (returned if returnCatalogs is True).
704  """
705  physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
706 
707  md = stdCat.getMetadata()
708  bands = md.getArray('BANDS')
709 
710  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
711  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
712  "in fgcmBuildStarsTask.")
713 
714  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
715  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
716 
717  if self.config.doReferenceCalibration:
718  lutCat = dataRefDict['fgcmLookUpTable'].get()
719  offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap)
720  else:
721  offsets = np.zeros(len(bands))
722 
723  if self.config.doRefcatOutput and butler is not None:
724  # Create a special config that has the tract number in it
725  datasetConfig = copy.copy(self.config.datasetConfig)
726  datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
727  tract)
728  self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
729 
730  if self.config.doZeropointOutput:
731  pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
732  physicalFilterMap, returnCatalogs=returnCatalogs)
733  else:
734  pcgen = None
735 
736  if self.config.doAtmosphereOutput:
737  atmgen = self._outputAtmospheres(dataRefDict, atmCat)
738  else:
739  atmgen = None
740 
741  retStruct = pipeBase.Struct(offsets=offsets,
742  atmospheres=atmgen)
743  if returnCatalogs:
744  retStruct.photoCalibCatalogs = pcgen
745  else:
746  retStruct.photoCalibs = pcgen
747 
748  return retStruct
749 
750  def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
751  """
752  Compute offsets relative to a reference catalog.
753 
754  This method splits the star catalog into healpix pixels
755  and computes the calibration transfer for a sample of
756  these pixels to approximate the 'absolute' calibration
757  values (on for each band) to apply to transfer the
758  absolute scale.
759 
760  Parameters
761  ----------
762  stdCat : `lsst.afw.table.SimpleCatalog`
763  FGCM standard stars
764  lutCat : `lsst.afw.table.SimpleCatalog`
765  FGCM Look-up table
766  physicalFilterMap : `dict`
767  Dictionary of mappings from physical filter to FGCM band.
768  bands : `list` [`str`]
769  List of band names from FGCM output
770  Returns
771  -------
772  offsets : `numpy.array` of floats
773  Per band zeropoint offsets
774  """
775 
776  # Only use stars that are observed in all the bands that were actually used
777  # This will ensure that we use the same healpix pixels for the absolute
778  # calibration of each band.
779  minObs = stdCat['ngood'].min(axis=1)
780 
781  goodStars = (minObs >= 1)
782  stdCat = stdCat[goodStars]
783 
784  self.log.info("Found %d stars with at least 1 good observation in each band" %
785  (len(stdCat)))
786 
787  # Associate each band with the appropriate physicalFilter and make
788  # filterLabels
789  filterLabels = []
790 
791  lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',')
792  lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',')
793  physicalFilterMapBands = list(physicalFilterMap.values())
794  physicalFilterMapFilters = list(physicalFilterMap.keys())
795  for band in bands:
796  # Find a physical filter associated from the band by doing
797  # a reverse lookup on the physicalFilterMap dict
798  physicalFilterMapIndex = physicalFilterMapBands.index(band)
799  physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
800  # Find the appropriate fgcm standard physicalFilter
801  lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
802  stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
803  filterLabels.append(afwImage.FilterLabel(band=band,
804  physical=stdPhysicalFilter))
805 
806  # We have to make a table for each pixel with flux/fluxErr
807  # This is a temporary table generated for input to the photoCal task.
808  # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
809  # have had chromatic corrections applied to get to the standard system
810  # specified by the atmosphere/instrumental parameters), nor are they
811  # in Jansky (since they don't have a proper absolute calibration: the overall
812  # zeropoint is estimated from the telescope size, etc.)
813  sourceMapper = afwTable.SchemaMapper(stdCat.schema)
814  sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
815  sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
816  doc="instrumental flux (counts)")
817  sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
818  doc="instrumental flux error (counts)")
819  badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
820  type='Flag',
821  doc="bad flag")
822 
823  # Split up the stars
824  # Note that there is an assumption here that the ra/dec coords stored
825  # on-disk are in radians, and therefore that starObs['coord_ra'] /
826  # starObs['coord_dec'] return radians when used as an array of numpy float64s.
827  theta = np.pi/2. - stdCat['coord_dec']
828  phi = stdCat['coord_ra']
829 
830  ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
831  h, rev = esutil.stat.histogram(ipring, rev=True)
832 
833  gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
834 
835  self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
836  (gdpix.size,
837  self.config.referencePixelizationNside,
838  self.config.referencePixelizationMinStars))
839 
840  if gdpix.size < self.config.referencePixelizationNPixels:
841  self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
842  (gdpix.size, self.config.referencePixelizationNPixels))
843  else:
844  # Sample out the pixels we want to use
845  gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
846 
847  results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
848  ('nstar', 'i4', len(bands)),
849  ('nmatch', 'i4', len(bands)),
850  ('zp', 'f4', len(bands)),
851  ('zpErr', 'f4', len(bands))])
852  results['hpix'] = ipring[rev[rev[gdpix]]]
853 
854  # We need a boolean index to deal with catalogs...
855  selected = np.zeros(len(stdCat), dtype=bool)
856 
857  refFluxFields = [None]*len(bands)
858 
859  for p_index, pix in enumerate(gdpix):
860  i1a = rev[rev[pix]: rev[pix + 1]]
861 
862  # the stdCat afwTable can only be indexed with boolean arrays,
863  # and not numpy index arrays (see DM-16497). This little trick
864  # converts the index array into a boolean array
865  selected[:] = False
866  selected[i1a] = True
867 
868  for b_index, filterLabel in enumerate(filterLabels):
869  struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index,
870  filterLabel, stdCat,
871  selected, refFluxFields)
872  results['nstar'][p_index, b_index] = len(i1a)
873  results['nmatch'][p_index, b_index] = len(struct.arrays.refMag)
874  results['zp'][p_index, b_index] = struct.zp
875  results['zpErr'][p_index, b_index] = struct.sigma
876 
877  # And compute the summary statistics
878  offsets = np.zeros(len(bands))
879 
880  for b_index, band in enumerate(bands):
881  # make configurable
882  ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch)
883  offsets[b_index] = np.median(results['zp'][ok, b_index])
884  # use median absolute deviation to estimate Normal sigma
885  # see https://en.wikipedia.org/wiki/Median_absolute_deviation
886  madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index]))
887  self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f",
888  band, offsets[b_index], madSigma)
889 
890  return offsets
891 
892  def _computeOffsetOneBand(self, sourceMapper, badStarKey,
893  b_index, filterLabel, stdCat, selected, refFluxFields):
894  """
895  Compute the zeropoint offset between the fgcm stdCat and the reference
896  stars for one pixel in one band
897 
898  Parameters
899  ----------
900  sourceMapper : `lsst.afw.table.SchemaMapper`
901  Mapper to go from stdCat to calibratable catalog
902  badStarKey : `lsst.afw.table.Key`
903  Key for the field with bad stars
904  b_index : `int`
905  Index of the band in the star catalog
906  filterLabel : `lsst.afw.image.FilterLabel`
907  filterLabel with band and physical filter
908  stdCat : `lsst.afw.table.SimpleCatalog`
909  FGCM standard stars
910  selected : `numpy.array(dtype=bool)`
911  Boolean array of which stars are in the pixel
912  refFluxFields : `list`
913  List of names of flux fields for reference catalog
914  """
915 
916  sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
917  sourceCat.reserve(selected.sum())
918  sourceCat.extend(stdCat[selected], mapper=sourceMapper)
919  sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5))
920  sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index]
921  * sourceCat['instFlux'])
922  # Make sure we only use stars that have valid measurements
923  # (This is perhaps redundant with requirements above that the
924  # stars be observed in all bands, but it can't hurt)
925  badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0)
926  for rec in sourceCat[badStar]:
927  rec.set(badStarKey, True)
928 
929  exposure = afwImage.ExposureF()
930  exposure.setFilterLabel(filterLabel)
931 
932  if refFluxFields[b_index] is None:
933  # Need to find the flux field in the reference catalog
934  # to work around limitations of DirectMatch in PhotoCal
935  ctr = stdCat[0].getCoord()
936  rad = 0.05*lsst.geom.degrees
937  refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel)
938  refFluxFields[b_index] = refDataTest.fluxField
939 
940  # Make a copy of the config so that we can modify it
941  calConfig = copy.copy(self.config.photoCal.value)
942  calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index]
943  calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err'
944  calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
945  config=calConfig,
946  schema=sourceCat.getSchema())
947 
948  struct = calTask.run(exposure, sourceCat)
949 
950  return struct
951 
952  def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
953  """
954  Output standard stars in indexed reference catalog format.
955  This is not currently supported in Gen3.
956 
957  Parameters
958  ----------
959  butler : `lsst.daf.persistence.Butler`
960  stdCat : `lsst.afw.table.SimpleCatalog`
961  FGCM standard star catalog from fgcmFitCycleTask
962  offsets : `numpy.array` of floats
963  Per band zeropoint offsets
964  bands : `list` [`str`]
965  List of band names from FGCM output
966  datasetConfig : `lsst.meas.algorithms.DatasetConfig`
967  Config for reference dataset
968  """
969 
970  self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
971 
972  indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
973  self.config.datasetConfig.indexer.active)
974 
975  # We determine the conversion from the native units (typically radians) to
976  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
977  # numpy arrays rather than Angles, which would we approximately 600x slower.
978  # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
979  # (as Angles) for input
980  conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
981  indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv,
982  stdCat['coord_dec']*conv))
983 
984  formattedCat = self._formatCatalog(stdCat, offsets, bands)
985 
986  # Write the master schema
987  dataId = indexer.makeDataId('master_schema',
988  datasetConfig.ref_dataset_name)
989  masterCat = afwTable.SimpleCatalog(formattedCat.schema)
990  addRefCatMetadata(masterCat)
991  butler.put(masterCat, 'ref_cat', dataId=dataId)
992 
993  # Break up the pixels using a histogram
994  h, rev = esutil.stat.histogram(indices, rev=True)
995  gd, = np.where(h > 0)
996  selected = np.zeros(len(formattedCat), dtype=bool)
997  for i in gd:
998  i1a = rev[rev[i]: rev[i + 1]]
999 
1000  # the formattedCat afwTable can only be indexed with boolean arrays,
1001  # and not numpy index arrays (see DM-16497). This little trick
1002  # converts the index array into a boolean array
1003  selected[:] = False
1004  selected[i1a] = True
1005 
1006  # Write the individual pixel
1007  dataId = indexer.makeDataId(indices[i1a[0]],
1008  datasetConfig.ref_dataset_name)
1009  butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
1010 
1011  # And save the dataset configuration
1012  dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name)
1013  butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
1014 
1015  self.log.info("Done outputting standard stars.")
1016 
1017  def _formatCatalog(self, fgcmStarCat, offsets, bands):
1018  """
1019  Turn an FGCM-formatted star catalog, applying zeropoint offsets.
1020 
1021  Parameters
1022  ----------
1023  fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
1024  SimpleCatalog as output by fgcmcal
1025  offsets : `list` with len(self.bands) entries
1026  Zeropoint offsets to apply
1027  bands : `list` [`str`]
1028  List of band names from FGCM output
1029 
1030  Returns
1031  -------
1032  formattedCat: `lsst.afw.table.SimpleCatalog`
1033  SimpleCatalog suitable for using as a reference catalog
1034  """
1035 
1036  sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1037  minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1038  addCentroid=False,
1039  addIsResolved=True,
1040  coordErrDim=0)
1041  sourceMapper.addMinimalSchema(minSchema)
1042  for band in bands:
1043  sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
1044  sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
1045  sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
1046 
1047  formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1048  formattedCat.reserve(len(fgcmStarCat))
1049  formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1050 
1051  # Note that we don't have to set `resolved` because the default is False
1052 
1053  for b, band in enumerate(bands):
1054  mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1055  # We want fluxes in nJy from calibrated AB magnitudes
1056  # (after applying offset). Updated after RFC-549 and RFC-575.
1057  flux = (mag*units.ABmag).to_value(units.nJy)
1058  fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
1059 
1060  formattedCat['%s_flux' % (band)][:] = flux
1061  formattedCat['%s_fluxErr' % (band)][:] = fluxErr
1062  formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
1063  formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
1064  formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
1065 
1066  addRefCatMetadata(formattedCat)
1067 
1068  return formattedCat
1069 
1070  def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1071  physicalFilterMap, returnCatalogs=True,
1072  tract=None):
1073  """Output the zeropoints in fgcm_photoCalib format.
1074 
1075  Parameters
1076  ----------
1077  camera : `lsst.afw.cameraGeom.Camera`
1078  Camera from the butler.
1079  zptCat : `lsst.afw.table.BaseCatalog`
1080  FGCM zeropoint catalog from `FgcmFitCycleTask`.
1081  visitCat : `lsst.afw.table.BaseCatalog`
1082  FGCM visitCat from `FgcmBuildStarsTask`.
1083  offsets : `numpy.array`
1084  Float array of absolute calibration offsets, one for each filter.
1085  bands : `list` [`str`]
1086  List of band names from FGCM output.
1087  physicalFilterMap : `dict`
1088  Dictionary of mappings from physical filter to FGCM band.
1089  returnCatalogs : `bool`, optional
1090  Return photoCalibs as per-visit exposure catalogs.
1091  tract: `int`, optional
1092  Tract number to output. Default is None (global calibration)
1093 
1094  Returns
1095  -------
1096  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1097  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1098  (returned if returnCatalogs is False).
1099  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1100  Generator that returns (visit, exposureCatalog) tuples.
1101  (returned if returnCatalogs is True).
1102  """
1103  # Select visit/ccds where we have a calibration
1104  # This includes ccds where we were able to interpolate from neighboring
1105  # ccds.
1106  cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
1107  selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
1108  & (zptCat['fgcmZptVar'] > 0.0))
1109 
1110  # Log warnings for any visit which has no valid zeropoints
1111  badVisits = np.unique(zptCat['visit'][~selected])
1112  goodVisits = np.unique(zptCat['visit'][selected])
1113  allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1114  for allBadVisit in allBadVisits:
1115  self.log.warn(f'No suitable photoCalib for visit {allBadVisit}')
1116 
1117  # Get a mapping from filtername to the offsets
1118  offsetMapping = {}
1119  for f in physicalFilterMap:
1120  # Not every filter in the map will necesarily have a band.
1121  if physicalFilterMap[f] in bands:
1122  offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1123 
1124  # Get a mapping from "ccd" to the ccd index used for the scaling
1125  ccdMapping = {}
1126  for ccdIndex, detector in enumerate(camera):
1127  ccdMapping[detector.getId()] = ccdIndex
1128 
1129  # And a mapping to get the flat-field scaling values
1130  scalingMapping = {}
1131  for rec in visitCat:
1132  scalingMapping[rec['visit']] = rec['scaling']
1133 
1134  if self.config.doComposeWcsJacobian:
1135  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
1136 
1137  # The zptCat is sorted by visit, which is useful
1138  lastVisit = -1
1139  zptVisitCatalog = None
1140 
1141  metadata = dafBase.PropertyList()
1142  metadata.add("COMMENT", "Catalog id is detector id, sorted.")
1143  metadata.add("COMMENT", "Only detectors with data have entries.")
1144 
1145  for rec in zptCat[selected]:
1146  # Retrieve overall scaling
1147  scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
1148 
1149  # The postCalibrationOffset describe any zeropoint offsets
1150  # to apply after the fgcm calibration. The first part comes
1151  # from the reference catalog match (used in testing). The
1152  # second part comes from the mean chromatic correction
1153  # (if configured).
1154  postCalibrationOffset = offsetMapping[rec['filtername']]
1155  if self.config.doApplyMeanChromaticCorrection:
1156  postCalibrationOffset += rec['fgcmDeltaChrom']
1157 
1158  fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
1159  rec['fgcmfZptChebXyMax'])
1160  # Convert from FGCM AB to nJy
1161  fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
1162  rec['fgcmfZptChebXyMax'],
1163  offset=postCalibrationOffset,
1164  scaling=scaling)
1165 
1166  if self.config.doComposeWcsJacobian:
1167 
1168  fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']],
1169  fgcmSuperStarField,
1170  fgcmZptField])
1171  else:
1172  # The photoCalib is just the product of the fgcmSuperStarField and the
1173  # fgcmZptField
1174  fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1175 
1176  # The "mean" calibration will be set to the center of the ccd for reference
1177  calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1178  calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
1179  photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1180  calibrationErr=calibErr,
1181  calibration=fgcmField,
1182  isConstant=False)
1183 
1184  if not returnCatalogs:
1185  # Return individual photoCalibs
1186  yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib)
1187  else:
1188  # Return full per-visit exposure catalogs
1189  if rec['visit'] != lastVisit:
1190  # This is a new visit. If the last visit was not -1, yield
1191  # the ExposureCatalog
1192  if lastVisit > -1:
1193  # ensure that the detectors are in sorted order, for fast lookups
1194  zptVisitCatalog.sort()
1195  yield (int(lastVisit), zptVisitCatalog)
1196  else:
1197  # We need to create a new schema
1198  zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1199  zptExpCatSchema.addField('visit', type='I', doc='Visit number')
1200 
1201  # And start a new one
1202  zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1203  zptVisitCatalog.setMetadata(metadata)
1204 
1205  lastVisit = int(rec['visit'])
1206 
1207  catRecord = zptVisitCatalog.addNew()
1208  catRecord['id'] = int(rec['detector'])
1209  catRecord['visit'] = rec['visit']
1210  catRecord.setPhotoCalib(photoCalib)
1211 
1212  # Final output of last exposure catalog
1213  if returnCatalogs:
1214  # ensure that the detectors are in sorted order, for fast lookups
1215  zptVisitCatalog.sort()
1216  yield (int(lastVisit), zptVisitCatalog)
1217 
1218  def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1219  """
1220  Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1221  and scaling.
1222 
1223  Parameters
1224  ----------
1225  coefficients: `numpy.array`
1226  Flattened array of chebyshev coefficients
1227  xyMax: `list` of length 2
1228  Maximum x and y of the chebyshev bounding box
1229  offset: `float`, optional
1230  Absolute calibration offset. Default is 0.0
1231  scaling: `float`, optional
1232  Flat scaling value from fgcmBuildStars. Default is 1.0
1233 
1234  Returns
1235  -------
1236  boundedField: `lsst.afw.math.ChebyshevBoundedField`
1237  """
1238 
1239  orderPlus1 = int(np.sqrt(coefficients.size))
1240  pars = np.zeros((orderPlus1, orderPlus1))
1241 
1242  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1243  lsst.geom.Point2I(*xyMax))
1244 
1245  pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1246  * (10.**(offset/-2.5))*scaling)
1247 
1248  boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1249 
1250  return boundedField
1251 
1252  def _outputAtmospheres(self, dataRefDict, atmCat):
1253  """
1254  Output the atmospheres.
1255 
1256  Parameters
1257  ----------
1258  dataRefDict : `dict`
1259  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1260  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1261  dataRef dictionary with keys:
1262 
1263  ``"fgcmLookUpTable"``
1264  dataRef for the FGCM look-up table.
1265  atmCat : `lsst.afw.table.BaseCatalog`
1266  FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1267 
1268  Returns
1269  -------
1270  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1271  Generator that returns (visit, transmissionCurve) tuples.
1272  """
1273  # First, we need to grab the look-up table and key info
1274  lutCat = dataRefDict['fgcmLookUpTable'].get()
1275 
1276  atmosphereTableName = lutCat[0]['tablename']
1277  elevation = lutCat[0]['elevation']
1278  atmLambda = lutCat[0]['atmLambda']
1279  lutCat = None
1280 
1281  # Make the atmosphere table if possible
1282  try:
1283  atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1284  atmTable.loadTable()
1285  except IOError:
1286  atmTable = None
1287 
1288  if atmTable is None:
1289  # Try to use MODTRAN instead
1290  try:
1291  modGen = fgcm.ModtranGenerator(elevation)
1292  lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1293  lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1294  except (ValueError, IOError) as e:
1295  raise RuntimeError("FGCM look-up-table generated with modtran, "
1296  "but modtran not configured to run.") from e
1297 
1298  zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
1299 
1300  for i, visit in enumerate(atmCat['visit']):
1301  if atmTable is not None:
1302  # Interpolate the atmosphere table
1303  atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
1304  pwv=atmCat[i]['pwv'],
1305  o3=atmCat[i]['o3'],
1306  tau=atmCat[i]['tau'],
1307  alpha=atmCat[i]['alpha'],
1308  zenith=zenith[i],
1309  ctranslamstd=[atmCat[i]['cTrans'],
1310  atmCat[i]['lamStd']])
1311  else:
1312  # Run modtran
1313  modAtm = modGen(pmb=atmCat[i]['pmb'],
1314  pwv=atmCat[i]['pwv'],
1315  o3=atmCat[i]['o3'],
1316  tau=atmCat[i]['tau'],
1317  alpha=atmCat[i]['alpha'],
1318  zenith=zenith[i],
1319  lambdaRange=lambdaRange,
1320  lambdaStep=lambdaStep,
1321  ctranslamstd=[atmCat[i]['cTrans'],
1322  atmCat[i]['lamStd']])
1323  atmVals = modAtm['COMBINED']
1324 
1325  # Now need to create something to persist...
1326  curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1327  wavelengths=atmLambda,
1328  throughputAtMin=atmVals[0],
1329  throughputAtMax=atmVals[-1])
1330 
1331  yield (int(visit), curve)
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:492