lsst.fgcmcal  21.0.0-6-g5ef7dad+4799c432d5
fgcmOutputProducts.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Make the final fgcmcal output products.
24 
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
32 offset per band.
33 """
34 import sys
35 import traceback
36 import copy
37 
38 import numpy as np
39 import healpy as hp
40 import esutil
41 from astropy import units
42 
43 import lsst.pex.config as pexConfig
44 import lsst.pipe.base as pipeBase
45 from lsst.pipe.base import connectionTypes
46 from lsst.afw.image import TransmissionCurve
47 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
48 from lsst.meas.algorithms import ReferenceObjectLoader
49 from lsst.pipe.tasks.photoCal import PhotoCalTask
50 import lsst.geom
51 import lsst.afw.image as afwImage
52 import lsst.afw.math as afwMath
53 import lsst.afw.table as afwTable
54 from lsst.meas.algorithms import IndexerRegistry
55 from lsst.meas.algorithms import DatasetConfig
56 from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
57 
58 from .utilities import computeApproxPixelAreaFields
59 from .utilities import lookupStaticCalibrations
60 
61 import fgcm
62 
63 __all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
64 
65 
66 class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
67  dimensions=("instrument",),
68  defaultTemplates={"cycleNumber": "0"}):
69  camera = connectionTypes.PrerequisiteInput(
70  doc="Camera instrument",
71  name="camera",
72  storageClass="Camera",
73  dimensions=("instrument",),
74  lookupFunction=lookupStaticCalibrations,
75  isCalibration=True,
76  )
77 
78  fgcmLookUpTable = connectionTypes.PrerequisiteInput(
79  doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
80  "chromatic corrections."),
81  name="fgcmLookUpTable",
82  storageClass="Catalog",
83  dimensions=("instrument",),
84  deferLoad=True,
85  )
86 
87  fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
88  doc="Catalog of visit information for fgcm",
89  name="fgcmVisitCatalog",
90  storageClass="Catalog",
91  dimensions=("instrument",),
92  deferLoad=True,
93  )
94 
95  fgcmStandardStars = connectionTypes.PrerequisiteInput(
96  doc="Catalog of standard star data from fgcm fit",
97  name="fgcmStandardStars{cycleNumber}",
98  storageClass="SimpleCatalog",
99  dimensions=("instrument",),
100  deferLoad=True,
101  )
102 
103  fgcmZeropoints = connectionTypes.PrerequisiteInput(
104  doc="Catalog of zeropoints from fgcm fit",
105  name="fgcmZeropoints{cycleNumber}",
106  storageClass="Catalog",
107  dimensions=("instrument",),
108  deferLoad=True,
109  )
110 
111  fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
112  doc="Catalog of atmosphere parameters from fgcm fit",
113  name="fgcmAtmosphereParameters{cycleNumber}",
114  storageClass="Catalog",
115  dimensions=("instrument",),
116  deferLoad=True,
117  )
118 
119  refCat = connectionTypes.PrerequisiteInput(
120  doc="Reference catalog to use for photometric calibration",
121  name="cal_ref_cat",
122  storageClass="SimpleCatalog",
123  dimensions=("skypix",),
124  deferLoad=True,
125  multiple=True,
126  )
127 
128  fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
129  doc="Config used to build FGCM input stars",
130  name="fgcmBuildStarsTable_config",
131  storageClass="Config",
132  )
133 
134  fgcmPhotoCalib = connectionTypes.Output(
135  doc="Per-visit photoCalib exposure catalogs produced from fgcm calibration",
136  name="fgcmPhotoCalibCatalog",
137  storageClass="ExposureCatalog",
138  dimensions=("instrument", "visit",),
139  multiple=True,
140  )
141 
142  fgcmTransmissionAtmosphere = connectionTypes.Output(
143  doc="Per-visit atmosphere transmission files produced from fgcm calibration",
144  name="transmission_atmosphere_fgcm",
145  storageClass="TransmissionCurve",
146  dimensions=("instrument",
147  "visit",),
148  multiple=True,
149  )
150 
151  fgcmOffsets = connectionTypes.Output(
152  doc="Per-band offsets computed from doReferenceCalibration",
153  name="fgcmReferenceCalibrationOffsets",
154  storageClass="Catalog",
155  dimensions=("instrument",),
156  multiple=False,
157  )
158 
159  def __init__(self, *, config=None):
160  super().__init__(config=config)
161 
162  if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
163  raise ValueError("cycleNumber must be of integer format")
164  if config.connections.refCat != config.refObjLoader.ref_dataset_name:
165  raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name")
166 
167  if config.doRefcatOutput:
168  raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput")
169 
170  if not config.doReferenceCalibration:
171  self.prerequisiteInputs.remove("refCat")
172  if not config.doAtmosphereOutput:
173  self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
174  if not config.doZeropointOutput:
175  self.prerequisiteInputs.remove("fgcmZeropoints")
176  if not config.doReferenceCalibration:
177  self.outputs.remove("fgcmOffsets")
178 
179 
180 class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
181  pipelineConnections=FgcmOutputProductsConnections):
182  """Config for FgcmOutputProductsTask"""
183 
184  cycleNumber = pexConfig.Field(
185  doc="Final fit cycle from FGCM fit",
186  dtype=int,
187  default=None,
188  )
189 
190  # The following fields refer to calibrating from a reference
191  # catalog, but in the future this might need to be expanded
192  doReferenceCalibration = pexConfig.Field(
193  doc=("Transfer 'absolute' calibration from reference catalog? "
194  "This afterburner step is unnecessary if reference stars "
195  "were used in the full fit in FgcmFitCycleTask."),
196  dtype=bool,
197  default=False,
198  )
199  doRefcatOutput = pexConfig.Field(
200  doc="Output standard stars in reference catalog format",
201  dtype=bool,
202  default=True,
203  )
204  doAtmosphereOutput = pexConfig.Field(
205  doc="Output atmospheres in transmission_atmosphere_fgcm format",
206  dtype=bool,
207  default=True,
208  )
209  doZeropointOutput = pexConfig.Field(
210  doc="Output zeropoints in fgcm_photoCalib format",
211  dtype=bool,
212  default=True,
213  )
214  doComposeWcsJacobian = pexConfig.Field(
215  doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
216  dtype=bool,
217  default=True,
218  )
219  doApplyMeanChromaticCorrection = pexConfig.Field(
220  doc="Apply the mean chromatic correction to the zeropoints?",
221  dtype=bool,
222  default=True,
223  )
224  refObjLoader = pexConfig.ConfigurableField(
225  target=LoadIndexedReferenceObjectsTask,
226  doc="reference object loader for 'absolute' photometric calibration",
227  )
228  photoCal = pexConfig.ConfigurableField(
229  target=PhotoCalTask,
230  doc="task to perform 'absolute' calibration",
231  )
232  referencePixelizationNside = pexConfig.Field(
233  doc="Healpix nside to pixelize catalog to compare to reference catalog",
234  dtype=int,
235  default=64,
236  )
237  referencePixelizationMinStars = pexConfig.Field(
238  doc=("Minimum number of stars per healpix pixel to select for comparison"
239  "to the specified reference catalog"),
240  dtype=int,
241  default=200,
242  )
243  referenceMinMatch = pexConfig.Field(
244  doc="Minimum number of stars matched to reference catalog to be used in statistics",
245  dtype=int,
246  default=50,
247  )
248  referencePixelizationNPixels = pexConfig.Field(
249  doc=("Number of healpix pixels to sample to do comparison. "
250  "Doing too many will take a long time and not yield any more "
251  "precise results because the final number is the median offset "
252  "(per band) from the set of pixels."),
253  dtype=int,
254  default=100,
255  )
256  datasetConfig = pexConfig.ConfigField(
257  dtype=DatasetConfig,
258  doc="Configuration for writing/reading ingested catalog",
259  )
260 
261  def setDefaults(self):
262  pexConfig.Config.setDefaults(self)
263 
264  # In order to transfer the "absolute" calibration from a reference
265  # catalog to the relatively calibrated FGCM standard stars (one number
266  # per band), we use the PhotoCalTask to match stars in a sample of healpix
267  # pixels. These basic settings ensure that only well-measured, good stars
268  # from the source and reference catalogs are used for the matching.
269 
270  # applyColorTerms needs to be False if doReferenceCalibration is False,
271  # as is the new default after DM-16702
272  self.photoCal.applyColorTerms = False
273  self.photoCal.fluxField = 'instFlux'
274  self.photoCal.magErrFloor = 0.003
275  self.photoCal.match.referenceSelection.doSignalToNoise = True
276  self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
277  self.photoCal.match.sourceSelection.doSignalToNoise = True
278  self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
279  self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
280  self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
281  self.photoCal.match.sourceSelection.doFlags = True
282  self.photoCal.match.sourceSelection.flags.good = []
283  self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
284  self.photoCal.match.sourceSelection.doUnresolved = False
285  self.datasetConfig.ref_dataset_name = 'fgcm_stars'
286  self.datasetConfig.format_version = 1
287 
288  def validate(self):
289  super().validate()
290 
291  # Force the connections to conform with cycleNumber
292  self.connections.cycleNumber = str(self.cycleNumber)
293 
294 
295 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
296  """Subclass of TaskRunner for fgcmOutputProductsTask
297 
298  fgcmOutputProductsTask.run() takes one argument, the butler, and
299  does not run on any data in the repository.
300  This runner does not use any parallelization.
301  """
302 
303  @staticmethod
304  def getTargetList(parsedCmd):
305  """
306  Return a list with one element, the butler.
307  """
308  return [parsedCmd.butler]
309 
310  def __call__(self, butler):
311  """
312  Parameters
313  ----------
314  butler: `lsst.daf.persistence.Butler`
315 
316  Returns
317  -------
318  exitStatus: `list` with `pipeBase.Struct`
319  exitStatus (0: success; 1: failure)
320  if self.doReturnResults also
321  results (`np.array` with absolute zeropoint offsets)
322  """
323  task = self.TaskClass(butler=butler, config=self.config, log=self.log)
324 
325  exitStatus = 0
326  if self.doRaise:
327  results = task.runDataRef(butler)
328  else:
329  try:
330  results = task.runDataRef(butler)
331  except Exception as e:
332  exitStatus = 1
333  task.log.fatal("Failed: %s" % e)
334  if not isinstance(e, pipeBase.TaskError):
335  traceback.print_exc(file=sys.stderr)
336 
337  task.writeMetadata(butler)
338 
339  if self.doReturnResults:
340  # The results here are the zeropoint offsets for each band
341  return [pipeBase.Struct(exitStatus=exitStatus,
342  results=results)]
343  else:
344  return [pipeBase.Struct(exitStatus=exitStatus)]
345 
346  def run(self, parsedCmd):
347  """
348  Run the task, with no multiprocessing
349 
350  Parameters
351  ----------
352  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
353  """
354 
355  resultList = []
356 
357  if self.precall(parsedCmd):
358  targetList = self.getTargetList(parsedCmd)
359  # make sure that we only get 1
360  resultList = self(targetList[0])
361 
362  return resultList
363 
364 
365 class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
366  """
367  Output products from FGCM global calibration.
368  """
369 
370  ConfigClass = FgcmOutputProductsConfig
371  RunnerClass = FgcmOutputProductsRunner
372  _DefaultName = "fgcmOutputProducts"
373 
374  def __init__(self, butler=None, **kwargs):
375  super().__init__(**kwargs)
376 
377  # no saving of metadata for now
378  def _getMetadataName(self):
379  return None
380 
381  def runQuantum(self, butlerQC, inputRefs, outputRefs):
382  dataRefDict = {}
383  dataRefDict['camera'] = butlerQC.get(inputRefs.camera)
384  dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
385  dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
386  dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
387 
388  if self.config.doZeropointOutput:
389  dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
390  photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
391  photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
392 
393  if self.config.doAtmosphereOutput:
394  dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
395  atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
396  atmRef in outputRefs.fgcmTransmissionAtmosphere}
397 
398  if self.config.doReferenceCalibration:
399  refConfig = self.config.refObjLoader
400  self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
401  for ref in inputRefs.refCat],
402  refCats=butlerQC.get(inputRefs.refCat),
403  config=refConfig,
404  log=self.log)
405  else:
406  self.refObjLoader = None
407 
408  dataRefDict['fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
409 
410  fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
411  physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
412 
413  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
414  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
415  "in fgcmBuildStarsTask.")
416  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
417  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
418 
419  struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=True)
420 
421  # Output the photoCalib exposure catalogs
422  if struct.photoCalibCatalogs is not None:
423  self.log.info("Outputting photoCalib catalogs.")
424  for visit, expCatalog in struct.photoCalibCatalogs:
425  butlerQC.put(expCatalog, photoCalibRefDict[visit])
426  self.log.info("Done outputting photoCalib catalogs.")
427 
428  # Output the atmospheres
429  if struct.atmospheres is not None:
430  self.log.info("Outputting atmosphere transmission files.")
431  for visit, atm in struct.atmospheres:
432  butlerQC.put(atm, atmRefDict[visit])
433  self.log.info("Done outputting atmosphere files.")
434 
435  if self.config.doReferenceCalibration:
436  # Turn offset into simple catalog for persistence if necessary
437  schema = afwTable.Schema()
438  schema.addField('offset', type=np.float64,
439  doc="Post-process calibration offset (mag)")
440  offsetCat = afwTable.BaseCatalog(schema)
441  offsetCat.resize(len(struct.offsets))
442  offsetCat['offset'][:] = struct.offsets
443 
444  butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
445 
446  return
447 
448  @pipeBase.timeMethod
449  def runDataRef(self, butler):
450  """
451  Make FGCM output products for use in the stack
452 
453  Parameters
454  ----------
455  butler: `lsst.daf.persistence.Butler`
456  cycleNumber: `int`
457  Final fit cycle number, override config.
458 
459  Returns
460  -------
461  offsets: `lsst.pipe.base.Struct`
462  A structure with array of zeropoint offsets
463 
464  Raises
465  ------
466  RuntimeError:
467  Raised if any one of the following is true:
468 
469  - butler cannot find "fgcmBuildStars_config" or
470  "fgcmBuildStarsTable_config".
471  - butler cannot find "fgcmFitCycle_config".
472  - "fgcmFitCycle_config" does not refer to
473  `self.config.cycleNumber`.
474  - butler cannot find "fgcmAtmosphereParameters" and
475  `self.config.doAtmosphereOutput` is `True`.
476  - butler cannot find "fgcmStandardStars" and
477  `self.config.doReferenceCalibration` is `True` or
478  `self.config.doRefcatOutput` is `True`.
479  - butler cannot find "fgcmZeropoints" and
480  `self.config.doZeropointOutput` is `True`.
481  """
482  if self.config.doReferenceCalibration:
483  # We need the ref obj loader to get the flux field
484  self.makeSubtask("refObjLoader", butler=butler)
485 
486  # Check to make sure that the fgcmBuildStars config exists, to retrieve
487  # the visit and ccd dataset tags
488  if not butler.datasetExists('fgcmBuildStarsTable_config') and \
489  not butler.datasetExists('fgcmBuildStars_config'):
490  raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
491  "which is prereq for fgcmOutputProducts")
492 
493  if butler.datasetExists('fgcmBuildStarsTable_config'):
494  fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
495  else:
496  fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
497  visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
498  ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
499  physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
500 
501  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
502  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
503  "in fgcmBuildStarsTask.")
504 
505  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
506  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
507 
508  # And make sure that the atmosphere was output properly
509  if (self.config.doAtmosphereOutput
510  and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
511  raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.")
512 
513  if not butler.datasetExists('fgcmStandardStars',
514  fgcmcycle=self.config.cycleNumber):
515  raise RuntimeError("Standard stars are missing for cycle %d." %
516  (self.config.cycleNumber))
517 
518  if (self.config.doZeropointOutput
519  and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
520  raise RuntimeError("Zeropoints are missing for cycle %d." %
521  (self.config.cycleNumber))
522 
523  dataRefDict = {}
524  # This is the _actual_ camera
525  dataRefDict['camera'] = butler.get('camera')
526  dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
527  dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
528  dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars',
529  fgcmcycle=self.config.cycleNumber)
530 
531  if self.config.doZeropointOutput:
532  dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints',
533  fgcmcycle=self.config.cycleNumber)
534  if self.config.doAtmosphereOutput:
535  dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters',
536  fgcmcycle=self.config.cycleNumber)
537 
538  struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False)
539 
540  if struct.photoCalibs is not None:
541  self.log.info("Outputting photoCalib files.")
542 
543  for visit, detector, physicalFilter, photoCalib in struct.photoCalibs:
544  butler.put(photoCalib, 'fgcm_photoCalib',
545  dataId={visitDataRefName: visit,
546  ccdDataRefName: detector,
547  'filter': physicalFilter})
548 
549  self.log.info("Done outputting photoCalib files.")
550 
551  if struct.atmospheres is not None:
552  self.log.info("Outputting atmosphere transmission files.")
553  for visit, atm in struct.atmospheres:
554  butler.put(atm, "transmission_atmosphere_fgcm",
555  dataId={visitDataRefName: visit})
556  self.log.info("Done outputting atmosphere transmissions.")
557 
558  return pipeBase.Struct(offsets=struct.offsets)
559 
560  def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None):
561  """Run the output products task.
562 
563  Parameters
564  ----------
565  dataRefDict : `dict`
566  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
567  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
568  dataRef dictionary with keys:
569 
570  ``"camera"``
571  Camera object (`lsst.afw.cameraGeom.Camera`)
572  ``"fgcmLookUpTable"``
573  dataRef for the FGCM look-up table.
574  ``"fgcmVisitCatalog"``
575  dataRef for visit summary catalog.
576  ``"fgcmStandardStars"``
577  dataRef for the output standard star catalog.
578  ``"fgcmZeropoints"``
579  dataRef for the zeropoint data catalog.
580  ``"fgcmAtmosphereParameters"``
581  dataRef for the atmosphere parameter catalog.
582  ``"fgcmBuildStarsTableConfig"``
583  Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
584  physicalFilterMap : `dict`
585  Dictionary of mappings from physical filter to FGCM band.
586  returnCatalogs : `bool`, optional
587  Return photoCalibs as per-visit exposure catalogs.
588  butler : `lsst.daf.persistence.Butler`, optional
589  Gen2 butler used for reference star outputs
590 
591  Returns
592  -------
593  retStruct : `lsst.pipe.base.Struct`
594  Output structure with keys:
595 
596  offsets : `np.ndarray`
597  Final reference offsets, per band.
598  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
599  Generator that returns (visit, transmissionCurve) tuples.
600  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
601  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
602  (returned if returnCatalogs is False).
603  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
604  Generator that returns (visit, exposureCatalog) tuples.
605  (returned if returnCatalogs is True).
606  """
607  stdCat = dataRefDict['fgcmStandardStars'].get()
608  md = stdCat.getMetadata()
609  bands = md.getArray('BANDS')
610 
611  if self.config.doReferenceCalibration:
612  offsets = self._computeReferenceOffsets(stdCat, bands)
613  else:
614  offsets = np.zeros(len(bands))
615 
616  # This is Gen2 only, and requires the butler.
617  if self.config.doRefcatOutput and butler is not None:
618  self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig)
619 
620  del stdCat
621 
622  if self.config.doZeropointOutput:
623  zptCat = dataRefDict['fgcmZeropoints'].get()
624  visitCat = dataRefDict['fgcmVisitCatalog'].get()
625 
626  pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
627  physicalFilterMap, returnCatalogs=returnCatalogs)
628  else:
629  pcgen = None
630 
631  if self.config.doAtmosphereOutput:
632  atmCat = dataRefDict['fgcmAtmosphereParameters'].get()
633  atmgen = self._outputAtmospheres(dataRefDict, atmCat)
634  else:
635  atmgen = None
636 
637  retStruct = pipeBase.Struct(offsets=offsets,
638  atmospheres=atmgen)
639  if returnCatalogs:
640  retStruct.photoCalibCatalogs = pcgen
641  else:
642  retStruct.photoCalibs = pcgen
643 
644  return retStruct
645 
646  def generateTractOutputProducts(self, dataRefDict, tract,
647  visitCat, zptCat, atmCat, stdCat,
648  fgcmBuildStarsConfig,
649  returnCatalogs=True,
650  butler=None):
651  """
652  Generate the output products for a given tract, as specified in the config.
653 
654  This method is here to have an alternate entry-point for
655  FgcmCalibrateTract.
656 
657  Parameters
658  ----------
659  dataRefDict : `dict`
660  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
661  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
662  dataRef dictionary with keys:
663 
664  ``"camera"``
665  Camera object (`lsst.afw.cameraGeom.Camera`)
666  ``"fgcmLookUpTable"``
667  dataRef for the FGCM look-up table.
668  tract : `int`
669  Tract number
670  visitCat : `lsst.afw.table.BaseCatalog`
671  FGCM visitCat from `FgcmBuildStarsTask`
672  zptCat : `lsst.afw.table.BaseCatalog`
673  FGCM zeropoint catalog from `FgcmFitCycleTask`
674  atmCat : `lsst.afw.table.BaseCatalog`
675  FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
676  stdCat : `lsst.afw.table.SimpleCatalog`
677  FGCM standard star catalog from `FgcmFitCycleTask`
678  fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
679  Configuration object from `FgcmBuildStarsTask`
680  returnCatalogs : `bool`, optional
681  Return photoCalibs as per-visit exposure catalogs.
682  butler: `lsst.daf.persistence.Butler`, optional
683  Gen2 butler used for reference star outputs
684 
685  Returns
686  -------
687  retStruct : `lsst.pipe.base.Struct`
688  Output structure with keys:
689 
690  offsets : `np.ndarray`
691  Final reference offsets, per band.
692  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
693  Generator that returns (visit, transmissionCurve) tuples.
694  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
695  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
696  (returned if returnCatalogs is False).
697  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
698  Generator that returns (visit, exposureCatalog) tuples.
699  (returned if returnCatalogs is True).
700  """
701  physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
702 
703  md = stdCat.getMetadata()
704  bands = md.getArray('BANDS')
705 
706  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
707  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
708  "in fgcmBuildStarsTask.")
709 
710  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
711  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
712 
713  if self.config.doReferenceCalibration:
714  offsets = self._computeReferenceOffsets(stdCat, bands)
715  else:
716  offsets = np.zeros(len(bands))
717 
718  if self.config.doRefcatOutput and butler is not None:
719  # Create a special config that has the tract number in it
720  datasetConfig = copy.copy(self.config.datasetConfig)
721  datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
722  tract)
723  self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig)
724 
725  if self.config.doZeropointOutput:
726  pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands,
727  physicalFilterMap, returnCatalogs=returnCatalogs)
728  else:
729  pcgen = None
730 
731  if self.config.doAtmosphereOutput:
732  atmgen = self._outputAtmospheres(dataRefDict, atmCat)
733  else:
734  atmgen = None
735 
736  retStruct = pipeBase.Struct(offsets=offsets,
737  atmospheres=atmgen)
738  if returnCatalogs:
739  retStruct.photoCalibCatalogs = pcgen
740  else:
741  retStruct.photoCalibs = pcgen
742 
743  return retStruct
744 
745  def _computeReferenceOffsets(self, stdCat, bands):
746  """
747  Compute offsets relative to a reference catalog.
748 
749  This method splits the star catalog into healpix pixels
750  and computes the calibration transfer for a sample of
751  these pixels to approximate the 'absolute' calibration
752  values (on for each band) to apply to transfer the
753  absolute scale.
754 
755  Parameters
756  ----------
757  stdCat : `lsst.afw.table.SimpleCatalog`
758  FGCM standard stars
759  bands : `list` [`str`]
760  List of band names from FGCM output
761  Returns
762  -------
763  offsets : `numpy.array` of floats
764  Per band zeropoint offsets
765  """
766 
767  # Only use stars that are observed in all the bands that were actually used
768  # This will ensure that we use the same healpix pixels for the absolute
769  # calibration of each band.
770  minObs = stdCat['ngood'].min(axis=1)
771 
772  goodStars = (minObs >= 1)
773  stdCat = stdCat[goodStars]
774 
775  self.log.info("Found %d stars with at least 1 good observation in each band" %
776  (len(stdCat)))
777 
778  # We have to make a table for each pixel with flux/fluxErr
779  # This is a temporary table generated for input to the photoCal task.
780  # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
781  # have had chromatic corrections applied to get to the standard system
782  # specified by the atmosphere/instrumental parameters), nor are they
783  # in Jansky (since they don't have a proper absolute calibration: the overall
784  # zeropoint is estimated from the telescope size, etc.)
785  sourceMapper = afwTable.SchemaMapper(stdCat.schema)
786  sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
787  sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
788  doc="instrumental flux (counts)")
789  sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
790  doc="instrumental flux error (counts)")
791  badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
792  type='Flag',
793  doc="bad flag")
794 
795  # Split up the stars
796  # Note that there is an assumption here that the ra/dec coords stored
797  # on-disk are in radians, and therefore that starObs['coord_ra'] /
798  # starObs['coord_dec'] return radians when used as an array of numpy float64s.
799  theta = np.pi/2. - stdCat['coord_dec']
800  phi = stdCat['coord_ra']
801 
802  ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
803  h, rev = esutil.stat.histogram(ipring, rev=True)
804 
805  gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
806 
807  self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
808  (gdpix.size,
809  self.config.referencePixelizationNside,
810  self.config.referencePixelizationMinStars))
811 
812  if gdpix.size < self.config.referencePixelizationNPixels:
813  self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
814  (gdpix.size, self.config.referencePixelizationNPixels))
815  else:
816  # Sample out the pixels we want to use
817  gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
818 
819  results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
820  ('nstar', 'i4', len(bands)),
821  ('nmatch', 'i4', len(bands)),
822  ('zp', 'f4', len(bands)),
823  ('zpErr', 'f4', len(bands))])
824  results['hpix'] = ipring[rev[rev[gdpix]]]
825 
826  # We need a boolean index to deal with catalogs...
827  selected = np.zeros(len(stdCat), dtype=np.bool)
828 
829  refFluxFields = [None]*len(bands)
830 
831  for p, pix in enumerate(gdpix):
832  i1a = rev[rev[pix]: rev[pix + 1]]
833 
834  # the stdCat afwTable can only be indexed with boolean arrays,
835  # and not numpy index arrays (see DM-16497). This little trick
836  # converts the index array into a boolean array
837  selected[:] = False
838  selected[i1a] = True
839 
840  for b, band in enumerate(bands):
841 
842  struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
843  selected, refFluxFields)
844  results['nstar'][p, b] = len(i1a)
845  results['nmatch'][p, b] = len(struct.arrays.refMag)
846  results['zp'][p, b] = struct.zp
847  results['zpErr'][p, b] = struct.sigma
848 
849  # And compute the summary statistics
850  offsets = np.zeros(len(bands))
851 
852  for b, band in enumerate(bands):
853  # make configurable
854  ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)
855  offsets[b] = np.median(results['zp'][ok, b])
856  # use median absolute deviation to estimate Normal sigma
857  # see https://en.wikipedia.org/wiki/Median_absolute_deviation
858  madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b]))
859  self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" %
860  (band, offsets[b], madSigma))
861 
862  return offsets
863 
864  def _computeOffsetOneBand(self, sourceMapper, badStarKey,
865  b, band, stdCat, selected, refFluxFields):
866  """
867  Compute the zeropoint offset between the fgcm stdCat and the reference
868  stars for one pixel in one band
869 
870  Parameters
871  ----------
872  sourceMapper: `lsst.afw.table.SchemaMapper`
873  Mapper to go from stdCat to calibratable catalog
874  badStarKey: `lsst.afw.table.Key`
875  Key for the field with bad stars
876  b: `int`
877  Index of the band in the star catalog
878  band: `str`
879  Name of band for reference catalog
880  stdCat: `lsst.afw.table.SimpleCatalog`
881  FGCM standard stars
882  selected: `numpy.array(dtype=np.bool)`
883  Boolean array of which stars are in the pixel
884  refFluxFields: `list`
885  List of names of flux fields for reference catalog
886  """
887 
888  sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
889  sourceCat.reserve(selected.sum())
890  sourceCat.extend(stdCat[selected], mapper=sourceMapper)
891  sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5))
892  sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b]
893  * sourceCat['instFlux'])
894  # Make sure we only use stars that have valid measurements
895  # (This is perhaps redundant with requirements above that the
896  # stars be observed in all bands, but it can't hurt)
897  badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0)
898  for rec in sourceCat[badStar]:
899  rec.set(badStarKey, True)
900 
901  exposure = afwImage.ExposureF()
902  exposure.setFilter(afwImage.Filter(band))
903 
904  if refFluxFields[b] is None:
905  # Need to find the flux field in the reference catalog
906  # to work around limitations of DirectMatch in PhotoCal
907  ctr = stdCat[0].getCoord()
908  rad = 0.05*lsst.geom.degrees
909  refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
910  refFluxFields[b] = refDataTest.fluxField
911 
912  # Make a copy of the config so that we can modify it
913  calConfig = copy.copy(self.config.photoCal.value)
914  calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
915  calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'
916  calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
917  config=calConfig,
918  schema=sourceCat.getSchema())
919 
920  struct = calTask.run(exposure, sourceCat)
921 
922  return struct
923 
924  def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
925  """
926  Output standard stars in indexed reference catalog format.
927  This is not currently supported in Gen3.
928 
929  Parameters
930  ----------
931  butler : `lsst.daf.persistence.Butler`
932  stdCat : `lsst.afw.table.SimpleCatalog`
933  FGCM standard star catalog from fgcmFitCycleTask
934  offsets : `numpy.array` of floats
935  Per band zeropoint offsets
936  bands : `list` [`str`]
937  List of band names from FGCM output
938  datasetConfig : `lsst.meas.algorithms.DatasetConfig`
939  Config for reference dataset
940  """
941 
942  self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
943 
944  indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
945  self.config.datasetConfig.indexer.active)
946 
947  # We determine the conversion from the native units (typically radians) to
948  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
949  # numpy arrays rather than Angles, which would we approximately 600x slower.
950  # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
951  # (as Angles) for input
952  conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
953  indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv,
954  stdCat['coord_dec']*conv))
955 
956  formattedCat = self._formatCatalog(stdCat, offsets, bands)
957 
958  # Write the master schema
959  dataId = indexer.makeDataId('master_schema',
960  datasetConfig.ref_dataset_name)
961  masterCat = afwTable.SimpleCatalog(formattedCat.schema)
962  addRefCatMetadata(masterCat)
963  butler.put(masterCat, 'ref_cat', dataId=dataId)
964 
965  # Break up the pixels using a histogram
966  h, rev = esutil.stat.histogram(indices, rev=True)
967  gd, = np.where(h > 0)
968  selected = np.zeros(len(formattedCat), dtype=np.bool)
969  for i in gd:
970  i1a = rev[rev[i]: rev[i + 1]]
971 
972  # the formattedCat afwTable can only be indexed with boolean arrays,
973  # and not numpy index arrays (see DM-16497). This little trick
974  # converts the index array into a boolean array
975  selected[:] = False
976  selected[i1a] = True
977 
978  # Write the individual pixel
979  dataId = indexer.makeDataId(indices[i1a[0]],
980  datasetConfig.ref_dataset_name)
981  butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
982 
983  # And save the dataset configuration
984  dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name)
985  butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
986 
987  self.log.info("Done outputting standard stars.")
988 
989  def _formatCatalog(self, fgcmStarCat, offsets, bands):
990  """
991  Turn an FGCM-formatted star catalog, applying zeropoint offsets.
992 
993  Parameters
994  ----------
995  fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
996  SimpleCatalog as output by fgcmcal
997  offsets : `list` with len(self.bands) entries
998  Zeropoint offsets to apply
999  bands : `list` [`str`]
1000  List of band names from FGCM output
1001 
1002  Returns
1003  -------
1004  formattedCat: `lsst.afw.table.SimpleCatalog`
1005  SimpleCatalog suitable for using as a reference catalog
1006  """
1007 
1008  sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
1009  minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands,
1010  addCentroid=False,
1011  addIsResolved=True,
1012  coordErrDim=0)
1013  sourceMapper.addMinimalSchema(minSchema)
1014  for band in bands:
1015  sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
1016  sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
1017  sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
1018 
1019  formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
1020  formattedCat.reserve(len(fgcmStarCat))
1021  formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
1022 
1023  # Note that we don't have to set `resolved` because the default is False
1024 
1025  for b, band in enumerate(bands):
1026  mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
1027  # We want fluxes in nJy from calibrated AB magnitudes
1028  # (after applying offset). Updated after RFC-549 and RFC-575.
1029  flux = (mag*units.ABmag).to_value(units.nJy)
1030  fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
1031 
1032  formattedCat['%s_flux' % (band)][:] = flux
1033  formattedCat['%s_fluxErr' % (band)][:] = fluxErr
1034  formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
1035  formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
1036  formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
1037 
1038  addRefCatMetadata(formattedCat)
1039 
1040  return formattedCat
1041 
1042  def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands,
1043  physicalFilterMap, returnCatalogs=True,
1044  tract=None):
1045  """Output the zeropoints in fgcm_photoCalib format.
1046 
1047  Parameters
1048  ----------
1049  camera : `lsst.afw.cameraGeom.Camera`
1050  Camera from the butler.
1051  zptCat : `lsst.afw.table.BaseCatalog`
1052  FGCM zeropoint catalog from `FgcmFitCycleTask`.
1053  visitCat : `lsst.afw.table.BaseCatalog`
1054  FGCM visitCat from `FgcmBuildStarsTask`.
1055  offsets : `numpy.array`
1056  Float array of absolute calibration offsets, one for each filter.
1057  bands : `list` [`str`]
1058  List of band names from FGCM output.
1059  physicalFilterMap : `dict`
1060  Dictionary of mappings from physical filter to FGCM band.
1061  returnCatalogs : `bool`, optional
1062  Return photoCalibs as per-visit exposure catalogs.
1063  tract: `int`, optional
1064  Tract number to output. Default is None (global calibration)
1065 
1066  Returns
1067  -------
1068  photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
1069  Generator that returns (visit, ccd, filtername, photoCalib) tuples.
1070  (returned if returnCatalogs is False).
1071  photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
1072  Generator that returns (visit, exposureCatalog) tuples.
1073  (returned if returnCatalogs is True).
1074  """
1075  # Select visit/ccds where we have a calibration
1076  # This includes ccds where we were able to interpolate from neighboring
1077  # ccds.
1078  cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
1079  selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
1080  & (zptCat['fgcmZptVar'] > 0.0))
1081 
1082  # Log warnings for any visit which has no valid zeropoints
1083  badVisits = np.unique(zptCat['visit'][~selected])
1084  goodVisits = np.unique(zptCat['visit'][selected])
1085  allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
1086  for allBadVisit in allBadVisits:
1087  self.log.warn(f'No suitable photoCalib for visit {allBadVisit}')
1088 
1089  # Get a mapping from filtername to the offsets
1090  offsetMapping = {}
1091  for f in physicalFilterMap:
1092  # Not every filter in the map will necesarily have a band.
1093  if physicalFilterMap[f] in bands:
1094  offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
1095 
1096  # Get a mapping from "ccd" to the ccd index used for the scaling
1097  ccdMapping = {}
1098  for ccdIndex, detector in enumerate(camera):
1099  ccdMapping[detector.getId()] = ccdIndex
1100 
1101  # And a mapping to get the flat-field scaling values
1102  scalingMapping = {}
1103  for rec in visitCat:
1104  scalingMapping[rec['visit']] = rec['scaling']
1105 
1106  if self.config.doComposeWcsJacobian:
1107  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
1108 
1109  # The zptCat is sorted by visit, which is useful
1110  lastVisit = -1
1111  zptCounter = 0
1112  zptVisitCatalog = None
1113  for rec in zptCat[selected]:
1114 
1115  # Retrieve overall scaling
1116  scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
1117 
1118  # The postCalibrationOffset describe any zeropoint offsets
1119  # to apply after the fgcm calibration. The first part comes
1120  # from the reference catalog match (used in testing). The
1121  # second part comes from the mean chromatic correction
1122  # (if configured).
1123  postCalibrationOffset = offsetMapping[rec['filtername']]
1124  if self.config.doApplyMeanChromaticCorrection:
1125  postCalibrationOffset += rec['fgcmDeltaChrom']
1126 
1127  fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
1128  rec['fgcmfZptChebXyMax'])
1129  # Convert from FGCM AB to nJy
1130  fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
1131  rec['fgcmfZptChebXyMax'],
1132  offset=postCalibrationOffset,
1133  scaling=scaling)
1134 
1135  if self.config.doComposeWcsJacobian:
1136 
1137  fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']],
1138  fgcmSuperStarField,
1139  fgcmZptField])
1140  else:
1141  # The photoCalib is just the product of the fgcmSuperStarField and the
1142  # fgcmZptField
1143  fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
1144 
1145  # The "mean" calibration will be set to the center of the ccd for reference
1146  calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
1147  calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
1148  photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
1149  calibrationErr=calibErr,
1150  calibration=fgcmField,
1151  isConstant=False)
1152 
1153  if not returnCatalogs:
1154  # Return individual photoCalibs
1155  yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib)
1156  else:
1157  # Return full per-visit exposure catalogs
1158  if rec['visit'] != lastVisit:
1159  # This is a new visit. If the last visit was not -1, yield
1160  # the ExposureCatalog
1161  if lastVisit > -1:
1162  yield (int(lastVisit), zptVisitCatalog)
1163  else:
1164  # We need to create a new schema
1165  zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema()
1166  zptExpCatSchema.addField('visit', type='I', doc='Visit number')
1167  zptExpCatSchema.addField('detector_id', type='I', doc='Detector number')
1168 
1169  # And start a new one
1170  zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
1171  zptVisitCatalog.resize(len(camera))
1172  zptVisitCatalog['visit'] = rec['visit']
1173  # By default all records will not resolve to a valid detector.
1174  zptVisitCatalog['detector_id'] = -1
1175 
1176  # Reset the counter
1177  zptCounter = 0
1178 
1179  lastVisit = int(rec['visit'])
1180 
1181  zptVisitCatalog[zptCounter].setPhotoCalib(photoCalib)
1182  zptVisitCatalog[zptCounter]['detector_id'] = int(rec['detector'])
1183 
1184  zptCounter += 1
1185 
1186  # Final output of last exposure catalog
1187  if returnCatalogs:
1188  yield (int(lastVisit), zptVisitCatalog)
1189 
1190  def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
1191  """
1192  Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
1193  and scaling.
1194 
1195  Parameters
1196  ----------
1197  coefficients: `numpy.array`
1198  Flattened array of chebyshev coefficients
1199  xyMax: `list` of length 2
1200  Maximum x and y of the chebyshev bounding box
1201  offset: `float`, optional
1202  Absolute calibration offset. Default is 0.0
1203  scaling: `float`, optional
1204  Flat scaling value from fgcmBuildStars. Default is 1.0
1205 
1206  Returns
1207  -------
1208  boundedField: `lsst.afw.math.ChebyshevBoundedField`
1209  """
1210 
1211  orderPlus1 = int(np.sqrt(coefficients.size))
1212  pars = np.zeros((orderPlus1, orderPlus1))
1213 
1214  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
1215  lsst.geom.Point2I(*xyMax))
1216 
1217  pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1)
1218  * (10.**(offset/-2.5))*scaling)
1219 
1220  boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
1221 
1222  return boundedField
1223 
1224  def _outputAtmospheres(self, dataRefDict, atmCat):
1225  """
1226  Output the atmospheres.
1227 
1228  Parameters
1229  ----------
1230  dataRefDict : `dict`
1231  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1232  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1233  dataRef dictionary with keys:
1234 
1235  ``"fgcmLookUpTable"``
1236  dataRef for the FGCM look-up table.
1237  atmCat : `lsst.afw.table.BaseCatalog`
1238  FGCM atmosphere parameter catalog from fgcmFitCycleTask.
1239 
1240  Returns
1241  -------
1242  atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
1243  Generator that returns (visit, transmissionCurve) tuples.
1244  """
1245  # First, we need to grab the look-up table and key info
1246  lutCat = dataRefDict['fgcmLookUpTable'].get()
1247 
1248  atmosphereTableName = lutCat[0]['tablename']
1249  elevation = lutCat[0]['elevation']
1250  atmLambda = lutCat[0]['atmLambda']
1251  lutCat = None
1252 
1253  # Make the atmosphere table if possible
1254  try:
1255  atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
1256  atmTable.loadTable()
1257  except IOError:
1258  atmTable = None
1259 
1260  if atmTable is None:
1261  # Try to use MODTRAN instead
1262  try:
1263  modGen = fgcm.ModtranGenerator(elevation)
1264  lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
1265  lambdaStep = (atmLambda[1] - atmLambda[0])/10.
1266  except (ValueError, IOError) as e:
1267  raise RuntimeError("FGCM look-up-table generated with modtran, "
1268  "but modtran not configured to run.") from e
1269 
1270  zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
1271 
1272  for i, visit in enumerate(atmCat['visit']):
1273  if atmTable is not None:
1274  # Interpolate the atmosphere table
1275  atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
1276  pwv=atmCat[i]['pwv'],
1277  o3=atmCat[i]['o3'],
1278  tau=atmCat[i]['tau'],
1279  alpha=atmCat[i]['alpha'],
1280  zenith=zenith[i],
1281  ctranslamstd=[atmCat[i]['cTrans'],
1282  atmCat[i]['lamStd']])
1283  else:
1284  # Run modtran
1285  modAtm = modGen(pmb=atmCat[i]['pmb'],
1286  pwv=atmCat[i]['pwv'],
1287  o3=atmCat[i]['o3'],
1288  tau=atmCat[i]['tau'],
1289  alpha=atmCat[i]['alpha'],
1290  zenith=zenith[i],
1291  lambdaRange=lambdaRange,
1292  lambdaStep=lambdaStep,
1293  ctranslamstd=[atmCat[i]['cTrans'],
1294  atmCat[i]['lamStd']])
1295  atmVals = modAtm['COMBINED']
1296 
1297  # Now need to create something to persist...
1298  curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
1299  wavelengths=atmLambda,
1300  throughputAtMin=atmVals[0],
1301  throughputAtMax=atmVals[-1])
1302 
1303  yield (int(visit), curve)
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:486