lsst.fgcmcal  20.0.0-4-ge48a6ca+16
fgcmOutputProducts.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Make the final fgcmcal output products.
24 
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
32 offset per band.
33 """
34 
35 import sys
36 import traceback
37 import copy
38 
39 import numpy as np
40 import healpy as hp
41 import esutil
42 from astropy import units
43 
44 import lsst.pex.config as pexConfig
45 import lsst.pipe.base as pipeBase
46 from lsst.afw.image import TransmissionCurve
47 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
48 from lsst.pipe.tasks.photoCal import PhotoCalTask
49 import lsst.geom
50 import lsst.afw.image as afwImage
51 import lsst.afw.math as afwMath
52 import lsst.afw.table as afwTable
53 from lsst.meas.algorithms import IndexerRegistry
54 from lsst.meas.algorithms import DatasetConfig
55 from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
56 
57 from .utilities import computeApproxPixelAreaFields
58 
59 import fgcm
60 
61 __all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
62 
63 
64 class FgcmOutputProductsConfig(pexConfig.Config):
65  """Config for FgcmOutputProductsTask"""
66 
67  cycleNumber = pexConfig.Field(
68  doc="Final fit cycle from FGCM fit",
69  dtype=int,
70  default=None,
71  )
72 
73  # The following fields refer to calibrating from a reference
74  # catalog, but in the future this might need to be expanded
75  doReferenceCalibration = pexConfig.Field(
76  doc=("Transfer 'absolute' calibration from reference catalog? "
77  "This afterburner step is unnecessary if reference stars "
78  "were used in the full fit in FgcmFitCycleTask."),
79  dtype=bool,
80  default=False,
81  )
82  doRefcatOutput = pexConfig.Field(
83  doc="Output standard stars in reference catalog format",
84  dtype=bool,
85  default=True,
86  )
87  doAtmosphereOutput = pexConfig.Field(
88  doc="Output atmospheres in transmission_atmosphere_fgcm format",
89  dtype=bool,
90  default=True,
91  )
92  doZeropointOutput = pexConfig.Field(
93  doc="Output zeropoints in fgcm_photoCalib format",
94  dtype=bool,
95  default=True,
96  )
97  doComposeWcsJacobian = pexConfig.Field(
98  doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
99  dtype=bool,
100  default=True,
101  )
102  refObjLoader = pexConfig.ConfigurableField(
103  target=LoadIndexedReferenceObjectsTask,
104  doc="reference object loader for 'absolute' photometric calibration",
105  )
106  photoCal = pexConfig.ConfigurableField(
107  target=PhotoCalTask,
108  doc="task to perform 'absolute' calibration",
109  )
110  referencePixelizationNside = pexConfig.Field(
111  doc="Healpix nside to pixelize catalog to compare to reference catalog",
112  dtype=int,
113  default=64,
114  )
115  referencePixelizationMinStars = pexConfig.Field(
116  doc=("Minimum number of stars per healpix pixel to select for comparison"
117  "to the specified reference catalog"),
118  dtype=int,
119  default=200,
120  )
121  referenceMinMatch = pexConfig.Field(
122  doc="Minimum number of stars matched to reference catalog to be used in statistics",
123  dtype=int,
124  default=50,
125  )
126  referencePixelizationNPixels = pexConfig.Field(
127  doc=("Number of healpix pixels to sample to do comparison. "
128  "Doing too many will take a long time and not yield any more "
129  "precise results because the final number is the median offset "
130  "(per band) from the set of pixels."),
131  dtype=int,
132  default=100,
133  )
134  datasetConfig = pexConfig.ConfigField(
135  dtype=DatasetConfig,
136  doc="Configuration for writing/reading ingested catalog",
137  )
138 
139  def setDefaults(self):
140  pexConfig.Config.setDefaults(self)
141 
142  # In order to transfer the "absolute" calibration from a reference
143  # catalog to the relatively calibrated FGCM standard stars (one number
144  # per band), we use the PhotoCalTask to match stars in a sample of healpix
145  # pixels. These basic settings ensure that only well-measured, good stars
146  # from the source and reference catalogs are used for the matching.
147 
148  # applyColorTerms needs to be False if doReferenceCalibration is False,
149  # as is the new default after DM-16702
150  self.photoCal.applyColorTerms = False
151  self.photoCal.fluxField = 'instFlux'
152  self.photoCal.magErrFloor = 0.003
153  self.photoCal.match.referenceSelection.doSignalToNoise = True
154  self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
155  self.photoCal.match.sourceSelection.doSignalToNoise = True
156  self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
157  self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
158  self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
159  self.photoCal.match.sourceSelection.doFlags = True
160  self.photoCal.match.sourceSelection.flags.good = []
161  self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
162  self.photoCal.match.sourceSelection.doUnresolved = False
163  self.datasetConfig.ref_dataset_name = 'fgcm_stars'
164  self.datasetConfig.format_version = 1
165 
166 
167 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
168  """Subclass of TaskRunner for fgcmOutputProductsTask
169 
170  fgcmOutputProductsTask.run() takes one argument, the butler, and
171  does not run on any data in the repository.
172  This runner does not use any parallelization.
173  """
174 
175  @staticmethod
176  def getTargetList(parsedCmd):
177  """
178  Return a list with one element, the butler.
179  """
180  return [parsedCmd.butler]
181 
182  def __call__(self, butler):
183  """
184  Parameters
185  ----------
186  butler: `lsst.daf.persistence.Butler`
187 
188  Returns
189  -------
190  exitStatus: `list` with `pipeBase.Struct`
191  exitStatus (0: success; 1: failure)
192  if self.doReturnResults also
193  results (`np.array` with absolute zeropoint offsets)
194  """
195  task = self.TaskClass(butler=butler, config=self.config, log=self.log)
196 
197  exitStatus = 0
198  if self.doRaise:
199  results = task.runDataRef(butler)
200  else:
201  try:
202  results = task.runDataRef(butler)
203  except Exception as e:
204  exitStatus = 1
205  task.log.fatal("Failed: %s" % e)
206  if not isinstance(e, pipeBase.TaskError):
207  traceback.print_exc(file=sys.stderr)
208 
209  task.writeMetadata(butler)
210 
211  if self.doReturnResults:
212  # The results here are the zeropoint offsets for each band
213  return [pipeBase.Struct(exitStatus=exitStatus,
214  results=results)]
215  else:
216  return [pipeBase.Struct(exitStatus=exitStatus)]
217 
218  def run(self, parsedCmd):
219  """
220  Run the task, with no multiprocessing
221 
222  Parameters
223  ----------
224  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
225  """
226 
227  resultList = []
228 
229  if self.precall(parsedCmd):
230  targetList = self.getTargetList(parsedCmd)
231  # make sure that we only get 1
232  resultList = self(targetList[0])
233 
234  return resultList
235 
236 
237 class FgcmOutputProductsTask(pipeBase.CmdLineTask):
238  """
239  Output products from FGCM global calibration.
240  """
241 
242  ConfigClass = FgcmOutputProductsConfig
243  RunnerClass = FgcmOutputProductsRunner
244  _DefaultName = "fgcmOutputProducts"
245 
246  def __init__(self, butler=None, **kwargs):
247  """
248  Instantiate an fgcmOutputProductsTask.
249 
250  Parameters
251  ----------
252  butler : `lsst.daf.persistence.Butler`
253  """
254 
255  pipeBase.CmdLineTask.__init__(self, **kwargs)
256 
257  if self.config.doReferenceCalibration:
258  # We need the ref obj loader to get the flux field
259  self.makeSubtask("refObjLoader", butler=butler)
260 
261  if self.config.doRefcatOutput:
262  self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
263  self.config.datasetConfig.indexer.active)
264 
265  # no saving of metadata for now
266  def _getMetadataName(self):
267  return None
268 
269  @pipeBase.timeMethod
270  def runDataRef(self, butler):
271  """
272  Make FGCM output products for use in the stack
273 
274  Parameters
275  ----------
276  butler: `lsst.daf.persistence.Butler`
277  cycleNumber: `int`
278  Final fit cycle number, override config.
279 
280  Returns
281  -------
282  offsets: `lsst.pipe.base.Struct`
283  A structure with array of zeropoint offsets
284 
285  Raises
286  ------
287  RuntimeError:
288  Raised if any one of the following is true:
289 
290  - butler cannot find "fgcmBuildStars_config" or
291  "fgcmBuildStarsTable_config".
292  - butler cannot find "fgcmFitCycle_config".
293  - "fgcmFitCycle_config" does not refer to
294  `self.config.cycleNumber`.
295  - butler cannot find "fgcmAtmosphereParameters" and
296  `self.config.doAtmosphereOutput` is `True`.
297  - butler cannot find "fgcmStandardStars" and
298  `self.config.doReferenceCalibration` is `True` or
299  `self.config.doRefcatOutput` is `True`.
300  - butler cannot find "fgcmZeropoints" and
301  `self.config.doZeropointOutput` is `True`.
302  """
303 
304  # Check to make sure that the fgcmBuildStars config exists, to retrieve
305  # the visit and ccd dataset tags
306  if not butler.datasetExists('fgcmBuildStarsTable_config') and \
307  not butler.datasetExists('fgcmBuildStars_config'):
308  raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
309  "which is prereq for fgcmOutputProducts")
310 
311  if butler.datasetExists('fgcmBuildStarsTable_config'):
312  fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
313  else:
314  fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
315  self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
316  self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
317  self.filterMap = fgcmBuildStarsConfig.filterMap
318 
319  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
320  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
321  "in fgcmBuildStarsTask.")
322 
323  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
324  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
325 
326  # Make sure that the fit config exists, to retrieve bands and other info
327  if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber):
328  raise RuntimeError("Cannot find fgcmFitCycle_config from cycle %d " % (self.config.cycleNumber) +
329  "which is required for fgcmOutputProducts.")
330 
331  fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber)
332  self.configBands = fitCycleConfig.bands
333 
334  if self.config.doReferenceCalibration and fitCycleConfig.doReferenceCalibration:
335  self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
336  "fitCycleConfig.doReferenceCalibration")
337 
338  # And make sure that the atmosphere was output properly
339  if (self.config.doAtmosphereOutput and
340  not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
341  raise RuntimeError("Atmosphere parameters are missing for cycle %d." %
342  (self.config.cycleNumber))
343 
344  if ((self.config.doReferenceCalibration or self.config.doRefcatOutput) and
345  (not butler.datasetExists('fgcmStandardStars',
346  fgcmcycle=self.config.cycleNumber))):
347  raise RuntimeError("Standard stars are missing for cycle %d." %
348  (self.config.cycleNumber))
349 
350  if (self.config.doZeropointOutput and
351  (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
352  raise RuntimeError("Zeropoints are missing for cycle %d." %
353  (self.config.cycleNumber))
354 
355  # And make sure this is the last cycle
356  if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1):
357  raise RuntimeError("The task fgcmOutputProducts should only be run"
358  "on the final fit cycle products")
359 
360  if self.config.doReferenceCalibration or self.config.doRefcatOutput:
361  stdCat = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
362  md = stdCat.getMetadata()
363  self.bands = md.getArray('BANDS')
364  else:
365  stdCat = None
366  self.bands = self.configBands
367 
368  if self.config.doReferenceCalibration:
369  offsets = self._computeReferenceOffsets(butler, stdCat)
370  else:
371  offsets = np.zeros(len(self.bands))
372 
373  # Output the standard stars in stack format
374  if self.config.doRefcatOutput:
375  self._outputStandardStars(butler, stdCat, offsets, self.config.datasetConfig)
376 
377  del stdCat
378 
379  # Output the gray zeropoints
380  if self.config.doZeropointOutput:
381  zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
382  visitCat = butler.get('fgcmVisitCatalog')
383 
384  self._outputZeropoints(butler, zptCat, visitCat, offsets)
385 
386  # Output the atmospheres
387  if self.config.doAtmosphereOutput:
388  atmCat = butler.get('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
389  self._outputAtmospheres(butler, atmCat)
390 
391  # We return the zp offsets
392  return pipeBase.Struct(offsets=offsets)
393 
394  def generateTractOutputProducts(self, butler, tract,
395  visitCat, zptCat, atmCat, stdCat,
396  fgcmBuildStarsConfig, fgcmFitCycleConfig):
397  """
398  Generate the output products for a given tract, as specified in the config.
399 
400  This method is here to have an alternate entry-point for
401  FgcmCalibrateTract.
402 
403  Parameters
404  ----------
405  butler: `lsst.daf.persistence.Butler`
406  tract: `int`
407  Tract number
408  visitCat: `lsst.afw.table.BaseCatalog`
409  FGCM visitCat from `FgcmBuildStarsTask`
410  zptCat: `lsst.afw.table.BaseCatalog`
411  FGCM zeropoint catalog from `FgcmFitCycleTask`
412  atmCat: `lsst.afw.table.BaseCatalog`
413  FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
414  stdCat: `lsst.afw.table.SimpleCatalog`
415  FGCM standard star catalog from `FgcmFitCycleTask`
416  fgcmBuildStarsConfig: `lsst.fgcmcal.FgcmBuildStarsConfig`
417  Configuration object from `FgcmBuildStarsTask`
418  fgcmFitCycleConfig: `lsst.fgcmcal.FgcmFitCycleConfig`
419  Configuration object from `FgcmFitCycleTask`
420  """
421 
422  self.configBands = fgcmFitCycleConfig.bands
423  self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
424  self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
425  self.filterMap = fgcmBuildStarsConfig.filterMap
426 
427  if stdCat is not None:
428  md = stdCat.getMetadata()
429  self.bands = md.getArray('BANDS')
430  else:
431  self.bands = self.configBands
432 
433  if self.config.doReferenceCalibration and fgcmFitCycleConfig.doReferenceCalibration:
434  self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
435  "fitCycleConfig.doReferenceCalibration")
436 
437  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
438  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
439  "in fgcmBuildStarsTask.")
440 
441  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
442  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
443 
444  if self.config.doReferenceCalibration:
445  offsets = self._computeReferenceOffsets(butler, stdCat)
446  else:
447  offsets = np.zeros(len(self.bands))
448 
449  if self.config.doRefcatOutput:
450  # Create a special config that has the tract number in it
451  datasetConfig = copy.copy(self.config.datasetConfig)
452  datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
453  tract)
454  self._outputStandardStars(butler, stdCat, offsets, datasetConfig)
455 
456  if self.config.doZeropointOutput:
457  self._outputZeropoints(butler, zptCat, visitCat, offsets, tract=tract)
458 
459  if self.config.doAtmosphereOutput:
460  self._outputAtmospheres(butler, atmCat, tract=tract)
461 
462  return pipeBase.Struct(offsets=offsets)
463 
464  def _computeReferenceOffsets(self, butler, stdCat):
465  """
466  Compute offsets relative to a reference catalog.
467 
468  This method splits the star catalog into healpix pixels
469  and computes the calibration transfer for a sample of
470  these pixels to approximate the 'absolute' calibration
471  values (on for each band) to apply to transfer the
472  absolute scale.
473 
474  Parameters
475  ----------
476  butler: `lsst.daf.persistence.Butler`
477  stdCat: `lsst.afw.table.SimpleCatalog`
478  FGCM standard stars
479 
480  Returns
481  -------
482  offsets: `numpy.array` of floats
483  Per band zeropoint offsets
484  """
485 
486  # Only use stars that are observed in all the bands that were actually used
487  # This will ensure that we use the same healpix pixels for the absolute
488  # calibration of each band.
489  minObs = stdCat['ngood'].min(axis=1)
490 
491  goodStars = (minObs >= 1)
492  stdCat = stdCat[goodStars]
493 
494  self.log.info("Found %d stars with at least 1 good observation in each band" %
495  (len(stdCat)))
496 
497  # We have to make a table for each pixel with flux/fluxErr
498  # This is a temporary table generated for input to the photoCal task.
499  # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
500  # have had chromatic corrections applied to get to the standard system
501  # specified by the atmosphere/instrumental parameters), nor are they
502  # in Jansky (since they don't have a proper absolute calibration: the overall
503  # zeropoint is estimated from the telescope size, etc.)
504  sourceMapper = afwTable.SchemaMapper(stdCat.schema)
505  sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
506  sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
507  doc="instrumental flux (counts)")
508  sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
509  doc="instrumental flux error (counts)")
510  badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
511  type='Flag',
512  doc="bad flag")
513 
514  # Split up the stars
515  # Note that there is an assumption here that the ra/dec coords stored
516  # on-disk are in radians, and therefore that starObs['coord_ra'] /
517  # starObs['coord_dec'] return radians when used as an array of numpy float64s.
518  theta = np.pi/2. - stdCat['coord_dec']
519  phi = stdCat['coord_ra']
520 
521  ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
522  h, rev = esutil.stat.histogram(ipring, rev=True)
523 
524  gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
525 
526  self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
527  (gdpix.size,
528  self.config.referencePixelizationNside,
529  self.config.referencePixelizationMinStars))
530 
531  if gdpix.size < self.config.referencePixelizationNPixels:
532  self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
533  (gdpix.size, self.config.referencePixelizationNPixels))
534  else:
535  # Sample out the pixels we want to use
536  gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
537 
538  results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
539  ('nstar', 'i4', len(self.bands)),
540  ('nmatch', 'i4', len(self.bands)),
541  ('zp', 'f4', len(self.bands)),
542  ('zpErr', 'f4', len(self.bands))])
543  results['hpix'] = ipring[rev[rev[gdpix]]]
544 
545  # We need a boolean index to deal with catalogs...
546  selected = np.zeros(len(stdCat), dtype=np.bool)
547 
548  refFluxFields = [None]*len(self.bands)
549 
550  for p, pix in enumerate(gdpix):
551  i1a = rev[rev[pix]: rev[pix + 1]]
552 
553  # the stdCat afwTable can only be indexed with boolean arrays,
554  # and not numpy index arrays (see DM-16497). This little trick
555  # converts the index array into a boolean array
556  selected[:] = False
557  selected[i1a] = True
558 
559  for b, band in enumerate(self.bands):
560 
561  struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
562  selected, refFluxFields)
563  results['nstar'][p, b] = len(i1a)
564  results['nmatch'][p, b] = len(struct.arrays.refMag)
565  results['zp'][p, b] = struct.zp
566  results['zpErr'][p, b] = struct.sigma
567 
568  # And compute the summary statistics
569  offsets = np.zeros(len(self.bands))
570 
571  for b, band in enumerate(self.bands):
572  # make configurable
573  ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)
574  offsets[b] = np.median(results['zp'][ok, b])
575  # use median absolute deviation to estimate Normal sigma
576  # see https://en.wikipedia.org/wiki/Median_absolute_deviation
577  madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b]))
578  self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" %
579  (band, offsets[b], madSigma))
580 
581  return offsets
582 
583  def _computeOffsetOneBand(self, sourceMapper, badStarKey,
584  b, band, stdCat, selected, refFluxFields):
585  """
586  Compute the zeropoint offset between the fgcm stdCat and the reference
587  stars for one pixel in one band
588 
589  Parameters
590  ----------
591  sourceMapper: `lsst.afw.table.SchemaMapper`
592  Mapper to go from stdCat to calibratable catalog
593  badStarKey: `lsst.afw.table.Key`
594  Key for the field with bad stars
595  b: `int`
596  Index of the band in the star catalog
597  band: `str`
598  Name of band for reference catalog
599  stdCat: `lsst.afw.table.SimpleCatalog`
600  FGCM standard stars
601  selected: `numpy.array(dtype=np.bool)`
602  Boolean array of which stars are in the pixel
603  refFluxFields: `list`
604  List of names of flux fields for reference catalog
605  """
606 
607  sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
608  sourceCat.reserve(selected.sum())
609  sourceCat.extend(stdCat[selected], mapper=sourceMapper)
610  sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5))
611  sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] *
612  sourceCat['instFlux'])
613  # Make sure we only use stars that have valid measurements
614  # (This is perhaps redundant with requirements above that the
615  # stars be observed in all bands, but it can't hurt)
616  badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0)
617  for rec in sourceCat[badStar]:
618  rec.set(badStarKey, True)
619 
620  exposure = afwImage.ExposureF()
621  exposure.setFilter(afwImage.Filter(band))
622 
623  if refFluxFields[b] is None:
624  # Need to find the flux field in the reference catalog
625  # to work around limitations of DirectMatch in PhotoCal
626  ctr = stdCat[0].getCoord()
627  rad = 0.05*lsst.geom.degrees
628  refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
629  refFluxFields[b] = refDataTest.fluxField
630 
631  # Make a copy of the config so that we can modify it
632  calConfig = copy.copy(self.config.photoCal.value)
633  calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
634  calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'
635  calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
636  config=calConfig,
637  schema=sourceCat.getSchema())
638 
639  struct = calTask.run(exposure, sourceCat)
640 
641  return struct
642 
643  def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig):
644  """
645  Output standard stars in indexed reference catalog format.
646 
647  Parameters
648  ----------
649  butler: `lsst.daf.persistence.Butler`
650  stdCat: `lsst.afw.table.SimpleCatalog`
651  FGCM standard star catalog from fgcmFitCycleTask
652  offsets: `numpy.array` of floats
653  Per band zeropoint offsets
654  datasetConfig: `lsst.meas.algorithms.DatasetConfig`
655  Config for reference dataset
656  """
657 
658  self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
659 
660  # We determine the conversion from the native units (typically radians) to
661  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
662  # numpy arrays rather than Angles, which would we approximately 600x slower.
663  # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
664  # (as Angles) for input
665  conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
666  indices = np.array(self.indexer.indexPoints(stdCat['coord_ra']*conv,
667  stdCat['coord_dec']*conv))
668 
669  formattedCat = self._formatCatalog(stdCat, offsets)
670 
671  # Write the master schema
672  dataId = self.indexer.makeDataId('master_schema',
673  datasetConfig.ref_dataset_name)
674  masterCat = afwTable.SimpleCatalog(formattedCat.schema)
675  addRefCatMetadata(masterCat)
676  butler.put(masterCat, 'ref_cat', dataId=dataId)
677 
678  # Break up the pixels using a histogram
679  h, rev = esutil.stat.histogram(indices, rev=True)
680  gd, = np.where(h > 0)
681  selected = np.zeros(len(formattedCat), dtype=np.bool)
682  for i in gd:
683  i1a = rev[rev[i]: rev[i + 1]]
684 
685  # the formattedCat afwTable can only be indexed with boolean arrays,
686  # and not numpy index arrays (see DM-16497). This little trick
687  # converts the index array into a boolean array
688  selected[:] = False
689  selected[i1a] = True
690 
691  # Write the individual pixel
692  dataId = self.indexer.makeDataId(indices[i1a[0]],
693  datasetConfig.ref_dataset_name)
694  butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
695 
696  # And save the dataset configuration
697  dataId = self.indexer.makeDataId(None, datasetConfig.ref_dataset_name)
698  butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
699 
700  self.log.info("Done outputting standard stars.")
701 
702  def _formatCatalog(self, fgcmStarCat, offsets):
703  """
704  Turn an FGCM-formatted star catalog, applying zeropoint offsets.
705 
706  Parameters
707  ----------
708  fgcmStarCat: `lsst.afw.Table.SimpleCatalog`
709  SimpleCatalog as output by fgcmcal
710  offsets: `list` with len(self.bands) entries
711  Zeropoint offsets to apply
712 
713  Returns
714  -------
715  formattedCat: `lsst.afw.table.SimpleCatalog`
716  SimpleCatalog suitable for using as a reference catalog
717  """
718 
719  sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
720  minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(self.bands,
721  addCentroid=False,
722  addIsResolved=True,
723  coordErrDim=0)
724  sourceMapper.addMinimalSchema(minSchema)
725  for band in self.bands:
726  sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
727  sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
728  sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
729 
730  formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
731  formattedCat.reserve(len(fgcmStarCat))
732  formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
733 
734  # Note that we don't have to set `resolved` because the default is False
735 
736  for b, band in enumerate(self.bands):
737  mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
738  # We want fluxes in nJy from calibrated AB magnitudes
739  # (after applying offset). Updated after RFC-549 and RFC-575.
740  flux = (mag*units.ABmag).to_value(units.nJy)
741  fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
742 
743  formattedCat['%s_flux' % (band)][:] = flux
744  formattedCat['%s_fluxErr' % (band)][:] = fluxErr
745  formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
746  formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
747  formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
748 
749  addRefCatMetadata(formattedCat)
750 
751  return formattedCat
752 
753  def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None):
754  """
755  Output the zeropoints in fgcm_photoCalib format.
756 
757  Parameters
758  ----------
759  butler: `lsst.daf.persistence.Butler`
760  zptCat: `lsst.afw.table.BaseCatalog`
761  FGCM zeropoint catalog from `FgcmFitCycleTask`
762  visitCat: `lsst.afw.table.BaseCatalog`
763  FGCM visitCat from `FgcmBuildStarsTask`
764  offsets: `numpy.array`
765  Float array of absolute calibration offsets, one for each filter
766  tract: `int`, optional
767  Tract number to output. Default is None (global calibration)
768  """
769 
770  if tract is None:
771  datasetType = 'fgcm_photoCalib'
772  else:
773  datasetType = 'fgcm_tract_photoCalib'
774 
775  self.log.info("Outputting %s objects" % (datasetType))
776 
777  # Select visit/ccds where we have a calibration
778  # This includes ccds where we were able to interpolate from neighboring
779  # ccds.
780  cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
781  too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD']
782  selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) &
783  (zptCat['fgcmZptVar'] > 0.0))
784 
785  # We also select the "best" calibrations, avoiding interpolation. These
786  # are only used for mapping filternames
787  selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) &
788  (zptCat['fgcmZptVar'] > 0.0))
789 
790  # Log warnings for any visit which has no valid zeropoints
791  badVisits = np.unique(zptCat['visit'][~selected])
792  goodVisits = np.unique(zptCat['visit'][selected])
793  allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
794  for allBadVisit in allBadVisits:
795  self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}')
796 
797  # Get the mapping from filtername to dataId filter name, empirically
798  filterMapping = {}
799  nFound = 0
800  for rec in zptCat[selected_best]:
801  if rec['filtername'] in filterMapping:
802  continue
803  dataId = {self.visitDataRefName: int(rec['visit']),
804  self.ccdDataRefName: int(rec['ccd'])}
805  dataRef = butler.dataRef('raw', dataId=dataId)
806  filterMapping[rec['filtername']] = dataRef.dataId['filter']
807  nFound += 1
808  if nFound == len(self.filterMap):
809  break
810 
811  # Get a mapping from filtername to the offsets
812  offsetMapping = {}
813  for f in self.filterMap:
814  # Not every filter in the map will necesarily have a band.
815  if self.filterMap[f] in self.bands:
816  offsetMapping[f] = offsets[self.bands.index(self.filterMap[f])]
817 
818  # Get a mapping from "ccd" to the ccd index used for the scaling
819  camera = butler.get('camera')
820  ccdMapping = {}
821  for ccdIndex, detector in enumerate(camera):
822  ccdMapping[detector.getId()] = ccdIndex
823 
824  # And a mapping to get the flat-field scaling values
825  scalingMapping = {}
826  for rec in visitCat:
827  scalingMapping[rec['visit']] = rec['scaling']
828 
829  if self.config.doComposeWcsJacobian:
830  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
831 
832  for rec in zptCat[selected]:
833 
834  # Retrieve overall scaling
835  scaling = scalingMapping[rec['visit']][ccdMapping[rec['ccd']]]
836 
837  fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
838  rec['fgcmfZptChebXyMax'])
839  # Convert from FGCM AB to nJy
840  fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
841  rec['fgcmfZptChebXyMax'],
842  offset=offsetMapping[rec['filtername']],
843  scaling=scaling)
844 
845  if self.config.doComposeWcsJacobian:
846 
847  fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['ccd']],
848  fgcmSuperStarField,
849  fgcmZptField])
850  else:
851  # The photoCalib is just the product of the fgcmSuperStarField and the
852  # fgcmZptField
853  fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
854 
855  # The "mean" calibration will be set to the center of the ccd for reference
856  calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
857  calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
858  photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
859  calibrationErr=calibErr,
860  calibration=fgcmField,
861  isConstant=False)
862 
863  if tract is None:
864  butler.put(photoCalib, datasetType,
865  dataId={self.visitDataRefName: int(rec['visit']),
866  self.ccdDataRefName: int(rec['ccd']),
867  'filter': filterMapping[rec['filtername']]})
868  else:
869  butler.put(photoCalib, datasetType,
870  dataId={self.visitDataRefName: int(rec['visit']),
871  self.ccdDataRefName: int(rec['ccd']),
872  'filter': filterMapping[rec['filtername']],
873  'tract': tract})
874 
875  self.log.info("Done outputting %s objects" % (datasetType))
876 
877  def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
878  """
879  Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
880  and scaling.
881 
882  Parameters
883  ----------
884  coefficients: `numpy.array`
885  Flattened array of chebyshev coefficients
886  xyMax: `list` of length 2
887  Maximum x and y of the chebyshev bounding box
888  offset: `float`, optional
889  Absolute calibration offset. Default is 0.0
890  scaling: `float`, optional
891  Flat scaling value from fgcmBuildStars. Default is 1.0
892 
893  Returns
894  -------
895  boundedField: `lsst.afw.math.ChebyshevBoundedField`
896  """
897 
898  orderPlus1 = int(np.sqrt(coefficients.size))
899  pars = np.zeros((orderPlus1, orderPlus1))
900 
901  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
902  lsst.geom.Point2I(*xyMax))
903 
904  pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) *
905  (10.**(offset/-2.5))*scaling)
906 
907  boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
908 
909  return boundedField
910 
911  def _outputAtmospheres(self, butler, atmCat, tract=None):
912  """
913  Output the atmospheres.
914 
915  Parameters
916  ----------
917  butler: `lsst.daf.persistence.Butler`
918  atmCat: `lsst.afw.table.BaseCatalog`
919  FGCM atmosphere parameter catalog from fgcmFitCycleTask
920  tract: `int`, optional
921  Tract number to output. Default is None (global calibration)
922  """
923 
924  self.log.info("Outputting atmosphere transmissions")
925 
926  # First, we need to grab the look-up table and key info
927  lutCat = butler.get('fgcmLookUpTable')
928 
929  atmosphereTableName = lutCat[0]['tablename']
930  elevation = lutCat[0]['elevation']
931  atmLambda = lutCat[0]['atmLambda']
932  lutCat = None
933 
934  # Make the atmosphere table if possible
935  try:
936  atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
937  atmTable.loadTable()
938  except IOError:
939  atmTable = None
940 
941  if atmTable is None:
942  # Try to use MODTRAN instead
943  try:
944  modGen = fgcm.ModtranGenerator(elevation)
945  lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
946  lambdaStep = (atmLambda[1] - atmLambda[0])/10.
947  except (ValueError, IOError) as e:
948  raise RuntimeError("FGCM look-up-table generated with modtran, "
949  "but modtran not configured to run.") from e
950 
951  zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
952 
953  for i, visit in enumerate(atmCat['visit']):
954  if atmTable is not None:
955  # Interpolate the atmosphere table
956  atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
957  pwv=atmCat[i]['pwv'],
958  o3=atmCat[i]['o3'],
959  tau=atmCat[i]['tau'],
960  alpha=atmCat[i]['alpha'],
961  zenith=zenith[i],
962  ctranslamstd=[atmCat[i]['cTrans'],
963  atmCat[i]['lamStd']])
964  else:
965  # Run modtran
966  modAtm = modGen(pmb=atmCat[i]['pmb'],
967  pwv=atmCat[i]['pwv'],
968  o3=atmCat[i]['o3'],
969  tau=atmCat[i]['tau'],
970  alpha=atmCat[i]['alpha'],
971  zenith=zenith[i],
972  lambdaRange=lambdaRange,
973  lambdaStep=lambdaStep,
974  ctranslamstd=[atmCat[i]['cTrans'],
975  atmCat[i]['lamStd']])
976  atmVals = modAtm['COMBINED']
977 
978  # Now need to create something to persist...
979  curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
980  wavelengths=atmLambda,
981  throughputAtMin=atmVals[0],
982  throughputAtMax=atmVals[-1])
983 
984  if tract is None:
985  butler.put(curve, "transmission_atmosphere_fgcm",
986  dataId={self.visitDataRefName: visit})
987  else:
988  butler.put(curve, "transmission_atmosphere_fgcm_tract",
989  dataId={self.visitDataRefName: visit,
990  'tract': tract})
991 
992  self.log.info("Done outputting atmosphere transmissions")
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig.datasetConfig
datasetConfig
Definition: fgcmOutputProducts.py:134
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.filterMap
filterMap
Definition: fgcmOutputProducts.py:317
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._computeOffsetOneBand
def _computeOffsetOneBand(self, sourceMapper, badStarKey, b, band, stdCat, selected, refFluxFields)
Definition: fgcmOutputProducts.py:583
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.bands
bands
Definition: fgcmOutputProducts.py:363
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._outputZeropoints
def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None)
Definition: fgcmOutputProducts.py:753
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.__init__
def __init__(self, butler=None, **kwargs)
Definition: fgcmOutputProducts.py:246
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.generateTractOutputProducts
def generateTractOutputProducts(self, butler, tract, visitCat, zptCat, atmCat, stdCat, fgcmBuildStarsConfig, fgcmFitCycleConfig)
Definition: fgcmOutputProducts.py:394
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._getChebyshevBoundedField
def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0)
Definition: fgcmOutputProducts.py:877
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig.photoCal
photoCal
Definition: fgcmOutputProducts.py:106
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner
Definition: fgcmOutputProducts.py:167
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig
Definition: fgcmOutputProducts.py:64
lsst.fgcmcal.utilities.computeApproxPixelAreaFields
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:472
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.visitDataRefName
visitDataRefName
Definition: fgcmOutputProducts.py:315
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._outputStandardStars
def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig)
Definition: fgcmOutputProducts.py:643
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig.setDefaults
def setDefaults(self)
Definition: fgcmOutputProducts.py:139
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
Definition: fgcmOutputProducts.py:237
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._formatCatalog
def _formatCatalog(self, fgcmStarCat, offsets)
Definition: fgcmOutputProducts.py:702
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.configBands
configBands
Definition: fgcmOutputProducts.py:332
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner.__call__
def __call__(self, butler)
Definition: fgcmOutputProducts.py:182
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.indexer
indexer
Definition: fgcmOutputProducts.py:262
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner.run
def run(self, parsedCmd)
Definition: fgcmOutputProducts.py:218
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.runDataRef
def runDataRef(self, butler)
Definition: fgcmOutputProducts.py:270
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._outputAtmospheres
def _outputAtmospheres(self, butler, atmCat, tract=None)
Definition: fgcmOutputProducts.py:911
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner.getTargetList
def getTargetList(parsedCmd)
Definition: fgcmOutputProducts.py:176
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._computeReferenceOffsets
def _computeReferenceOffsets(self, butler, stdCat)
Definition: fgcmOutputProducts.py:464
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.ccdDataRefName
ccdDataRefName
Definition: fgcmOutputProducts.py:316