lsst.fgcmcal  20.0.0-5-gfcebe35+8726ad94da
fgcmOutputProducts.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Make the final fgcmcal output products.
24 
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
32 offset per band.
33 """
34 
35 import sys
36 import traceback
37 import copy
38 
39 import numpy as np
40 import healpy as hp
41 import esutil
42 from astropy import units
43 
44 import lsst.pex.config as pexConfig
45 import lsst.pipe.base as pipeBase
46 from lsst.afw.image import TransmissionCurve
47 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
48 from lsst.pipe.tasks.photoCal import PhotoCalTask
49 import lsst.geom
50 import lsst.afw.image as afwImage
51 import lsst.afw.math as afwMath
52 import lsst.afw.table as afwTable
53 from lsst.meas.algorithms import IndexerRegistry
54 from lsst.meas.algorithms import DatasetConfig
55 from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
56 
57 from .utilities import computeApproxPixelAreaFields
58 
59 import fgcm
60 
61 __all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
62 
63 
64 class FgcmOutputProductsConfig(pexConfig.Config):
65  """Config for FgcmOutputProductsTask"""
66 
67  cycleNumber = pexConfig.Field(
68  doc="Final fit cycle from FGCM fit",
69  dtype=int,
70  default=None,
71  )
72 
73  # The following fields refer to calibrating from a reference
74  # catalog, but in the future this might need to be expanded
75  doReferenceCalibration = pexConfig.Field(
76  doc=("Transfer 'absolute' calibration from reference catalog? "
77  "This afterburner step is unnecessary if reference stars "
78  "were used in the full fit in FgcmFitCycleTask."),
79  dtype=bool,
80  default=False,
81  )
82  doRefcatOutput = pexConfig.Field(
83  doc="Output standard stars in reference catalog format",
84  dtype=bool,
85  default=True,
86  )
87  doAtmosphereOutput = pexConfig.Field(
88  doc="Output atmospheres in transmission_atmosphere_fgcm format",
89  dtype=bool,
90  default=True,
91  )
92  doZeropointOutput = pexConfig.Field(
93  doc="Output zeropoints in fgcm_photoCalib format",
94  dtype=bool,
95  default=True,
96  )
97  doComposeWcsJacobian = pexConfig.Field(
98  doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
99  dtype=bool,
100  default=True,
101  )
102  doApplyMeanChromaticCorrection = pexConfig.Field(
103  doc="Apply the mean chromatic correction to the zeropoints?",
104  dtype=bool,
105  default=True,
106  )
107  refObjLoader = pexConfig.ConfigurableField(
108  target=LoadIndexedReferenceObjectsTask,
109  doc="reference object loader for 'absolute' photometric calibration",
110  )
111  photoCal = pexConfig.ConfigurableField(
112  target=PhotoCalTask,
113  doc="task to perform 'absolute' calibration",
114  )
115  referencePixelizationNside = pexConfig.Field(
116  doc="Healpix nside to pixelize catalog to compare to reference catalog",
117  dtype=int,
118  default=64,
119  )
120  referencePixelizationMinStars = pexConfig.Field(
121  doc=("Minimum number of stars per healpix pixel to select for comparison"
122  "to the specified reference catalog"),
123  dtype=int,
124  default=200,
125  )
126  referenceMinMatch = pexConfig.Field(
127  doc="Minimum number of stars matched to reference catalog to be used in statistics",
128  dtype=int,
129  default=50,
130  )
131  referencePixelizationNPixels = pexConfig.Field(
132  doc=("Number of healpix pixels to sample to do comparison. "
133  "Doing too many will take a long time and not yield any more "
134  "precise results because the final number is the median offset "
135  "(per band) from the set of pixels."),
136  dtype=int,
137  default=100,
138  )
139  datasetConfig = pexConfig.ConfigField(
140  dtype=DatasetConfig,
141  doc="Configuration for writing/reading ingested catalog",
142  )
143 
144  def setDefaults(self):
145  pexConfig.Config.setDefaults(self)
146 
147  # In order to transfer the "absolute" calibration from a reference
148  # catalog to the relatively calibrated FGCM standard stars (one number
149  # per band), we use the PhotoCalTask to match stars in a sample of healpix
150  # pixels. These basic settings ensure that only well-measured, good stars
151  # from the source and reference catalogs are used for the matching.
152 
153  # applyColorTerms needs to be False if doReferenceCalibration is False,
154  # as is the new default after DM-16702
155  self.photoCal.applyColorTerms = False
156  self.photoCal.fluxField = 'instFlux'
157  self.photoCal.magErrFloor = 0.003
158  self.photoCal.match.referenceSelection.doSignalToNoise = True
159  self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
160  self.photoCal.match.sourceSelection.doSignalToNoise = True
161  self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
162  self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
163  self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
164  self.photoCal.match.sourceSelection.doFlags = True
165  self.photoCal.match.sourceSelection.flags.good = []
166  self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
167  self.photoCal.match.sourceSelection.doUnresolved = False
168  self.datasetConfig.ref_dataset_name = 'fgcm_stars'
169  self.datasetConfig.format_version = 1
170 
171 
172 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
173  """Subclass of TaskRunner for fgcmOutputProductsTask
174 
175  fgcmOutputProductsTask.run() takes one argument, the butler, and
176  does not run on any data in the repository.
177  This runner does not use any parallelization.
178  """
179 
180  @staticmethod
181  def getTargetList(parsedCmd):
182  """
183  Return a list with one element, the butler.
184  """
185  return [parsedCmd.butler]
186 
187  def __call__(self, butler):
188  """
189  Parameters
190  ----------
191  butler: `lsst.daf.persistence.Butler`
192 
193  Returns
194  -------
195  exitStatus: `list` with `pipeBase.Struct`
196  exitStatus (0: success; 1: failure)
197  if self.doReturnResults also
198  results (`np.array` with absolute zeropoint offsets)
199  """
200  task = self.TaskClass(butler=butler, config=self.config, log=self.log)
201 
202  exitStatus = 0
203  if self.doRaise:
204  results = task.runDataRef(butler)
205  else:
206  try:
207  results = task.runDataRef(butler)
208  except Exception as e:
209  exitStatus = 1
210  task.log.fatal("Failed: %s" % e)
211  if not isinstance(e, pipeBase.TaskError):
212  traceback.print_exc(file=sys.stderr)
213 
214  task.writeMetadata(butler)
215 
216  if self.doReturnResults:
217  # The results here are the zeropoint offsets for each band
218  return [pipeBase.Struct(exitStatus=exitStatus,
219  results=results)]
220  else:
221  return [pipeBase.Struct(exitStatus=exitStatus)]
222 
223  def run(self, parsedCmd):
224  """
225  Run the task, with no multiprocessing
226 
227  Parameters
228  ----------
229  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
230  """
231 
232  resultList = []
233 
234  if self.precall(parsedCmd):
235  targetList = self.getTargetList(parsedCmd)
236  # make sure that we only get 1
237  resultList = self(targetList[0])
238 
239  return resultList
240 
241 
242 class FgcmOutputProductsTask(pipeBase.CmdLineTask):
243  """
244  Output products from FGCM global calibration.
245  """
246 
247  ConfigClass = FgcmOutputProductsConfig
248  RunnerClass = FgcmOutputProductsRunner
249  _DefaultName = "fgcmOutputProducts"
250 
251  def __init__(self, butler=None, **kwargs):
252  """
253  Instantiate an fgcmOutputProductsTask.
254 
255  Parameters
256  ----------
257  butler : `lsst.daf.persistence.Butler`
258  """
259 
260  pipeBase.CmdLineTask.__init__(self, **kwargs)
261 
262  if self.config.doReferenceCalibration:
263  # We need the ref obj loader to get the flux field
264  self.makeSubtask("refObjLoader", butler=butler)
265 
266  if self.config.doRefcatOutput:
267  self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
268  self.config.datasetConfig.indexer.active)
269 
270  # no saving of metadata for now
271  def _getMetadataName(self):
272  return None
273 
274  @pipeBase.timeMethod
275  def runDataRef(self, butler):
276  """
277  Make FGCM output products for use in the stack
278 
279  Parameters
280  ----------
281  butler: `lsst.daf.persistence.Butler`
282  cycleNumber: `int`
283  Final fit cycle number, override config.
284 
285  Returns
286  -------
287  offsets: `lsst.pipe.base.Struct`
288  A structure with array of zeropoint offsets
289 
290  Raises
291  ------
292  RuntimeError:
293  Raised if any one of the following is true:
294 
295  - butler cannot find "fgcmBuildStars_config" or
296  "fgcmBuildStarsTable_config".
297  - butler cannot find "fgcmFitCycle_config".
298  - "fgcmFitCycle_config" does not refer to
299  `self.config.cycleNumber`.
300  - butler cannot find "fgcmAtmosphereParameters" and
301  `self.config.doAtmosphereOutput` is `True`.
302  - butler cannot find "fgcmStandardStars" and
303  `self.config.doReferenceCalibration` is `True` or
304  `self.config.doRefcatOutput` is `True`.
305  - butler cannot find "fgcmZeropoints" and
306  `self.config.doZeropointOutput` is `True`.
307  """
308 
309  # Check to make sure that the fgcmBuildStars config exists, to retrieve
310  # the visit and ccd dataset tags
311  if not butler.datasetExists('fgcmBuildStarsTable_config') and \
312  not butler.datasetExists('fgcmBuildStars_config'):
313  raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, "
314  "which is prereq for fgcmOutputProducts")
315 
316  if butler.datasetExists('fgcmBuildStarsTable_config'):
317  fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config')
318  else:
319  fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
320  self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
321  self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
322  self.filterMap = fgcmBuildStarsConfig.filterMap
323 
324  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
325  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
326  "in fgcmBuildStarsTask.")
327 
328  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
329  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
330 
331  # Make sure that the fit config exists, to retrieve bands and other info
332  if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber):
333  raise RuntimeError("Cannot find fgcmFitCycle_config from cycle %d " % (self.config.cycleNumber) +
334  "which is required for fgcmOutputProducts.")
335 
336  fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber)
337  self.configBands = fitCycleConfig.bands
338 
339  if self.config.doReferenceCalibration and fitCycleConfig.doReferenceCalibration:
340  self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
341  "fitCycleConfig.doReferenceCalibration")
342 
343  # And make sure that the atmosphere was output properly
344  if (self.config.doAtmosphereOutput and
345  not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
346  raise RuntimeError("Atmosphere parameters are missing for cycle %d." %
347  (self.config.cycleNumber))
348 
349  if ((self.config.doReferenceCalibration or self.config.doRefcatOutput) and
350  (not butler.datasetExists('fgcmStandardStars',
351  fgcmcycle=self.config.cycleNumber))):
352  raise RuntimeError("Standard stars are missing for cycle %d." %
353  (self.config.cycleNumber))
354 
355  if (self.config.doZeropointOutput and
356  (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
357  raise RuntimeError("Zeropoints are missing for cycle %d." %
358  (self.config.cycleNumber))
359 
360  # And make sure this is the last cycle
361  if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1):
362  raise RuntimeError("The task fgcmOutputProducts should only be run"
363  "on the final fit cycle products")
364 
365  if self.config.doReferenceCalibration or self.config.doRefcatOutput:
366  stdCat = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
367  md = stdCat.getMetadata()
368  self.bands = md.getArray('BANDS')
369  else:
370  stdCat = None
371  self.bands = self.configBands
372 
373  if self.config.doReferenceCalibration:
374  offsets = self._computeReferenceOffsets(butler, stdCat)
375  else:
376  offsets = np.zeros(len(self.bands))
377 
378  # Output the standard stars in stack format
379  if self.config.doRefcatOutput:
380  self._outputStandardStars(butler, stdCat, offsets, self.config.datasetConfig)
381 
382  del stdCat
383 
384  # Output the gray zeropoints
385  if self.config.doZeropointOutput:
386  zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
387  visitCat = butler.get('fgcmVisitCatalog')
388 
389  self._outputZeropoints(butler, zptCat, visitCat, offsets)
390 
391  # Output the atmospheres
392  if self.config.doAtmosphereOutput:
393  atmCat = butler.get('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
394  self._outputAtmospheres(butler, atmCat)
395 
396  # We return the zp offsets
397  return pipeBase.Struct(offsets=offsets)
398 
399  def generateTractOutputProducts(self, butler, tract,
400  visitCat, zptCat, atmCat, stdCat,
401  fgcmBuildStarsConfig, fgcmFitCycleConfig):
402  """
403  Generate the output products for a given tract, as specified in the config.
404 
405  This method is here to have an alternate entry-point for
406  FgcmCalibrateTract.
407 
408  Parameters
409  ----------
410  butler: `lsst.daf.persistence.Butler`
411  tract: `int`
412  Tract number
413  visitCat: `lsst.afw.table.BaseCatalog`
414  FGCM visitCat from `FgcmBuildStarsTask`
415  zptCat: `lsst.afw.table.BaseCatalog`
416  FGCM zeropoint catalog from `FgcmFitCycleTask`
417  atmCat: `lsst.afw.table.BaseCatalog`
418  FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
419  stdCat: `lsst.afw.table.SimpleCatalog`
420  FGCM standard star catalog from `FgcmFitCycleTask`
421  fgcmBuildStarsConfig: `lsst.fgcmcal.FgcmBuildStarsConfig`
422  Configuration object from `FgcmBuildStarsTask`
423  fgcmFitCycleConfig: `lsst.fgcmcal.FgcmFitCycleConfig`
424  Configuration object from `FgcmFitCycleTask`
425  """
426 
427  self.configBands = fgcmFitCycleConfig.bands
428  self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
429  self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
430  self.filterMap = fgcmBuildStarsConfig.filterMap
431 
432  if stdCat is not None:
433  md = stdCat.getMetadata()
434  self.bands = md.getArray('BANDS')
435  else:
436  self.bands = self.configBands
437 
438  if self.config.doReferenceCalibration and fgcmFitCycleConfig.doReferenceCalibration:
439  self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
440  "fitCycleConfig.doReferenceCalibration")
441 
442  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
443  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
444  "in fgcmBuildStarsTask.")
445 
446  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
447  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
448 
449  if self.config.doReferenceCalibration:
450  offsets = self._computeReferenceOffsets(butler, stdCat)
451  else:
452  offsets = np.zeros(len(self.bands))
453 
454  if self.config.doRefcatOutput:
455  # Create a special config that has the tract number in it
456  datasetConfig = copy.copy(self.config.datasetConfig)
457  datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
458  tract)
459  self._outputStandardStars(butler, stdCat, offsets, datasetConfig)
460 
461  if self.config.doZeropointOutput:
462  self._outputZeropoints(butler, zptCat, visitCat, offsets, tract=tract)
463 
464  if self.config.doAtmosphereOutput:
465  self._outputAtmospheres(butler, atmCat, tract=tract)
466 
467  return pipeBase.Struct(offsets=offsets)
468 
469  def _computeReferenceOffsets(self, butler, stdCat):
470  """
471  Compute offsets relative to a reference catalog.
472 
473  This method splits the star catalog into healpix pixels
474  and computes the calibration transfer for a sample of
475  these pixels to approximate the 'absolute' calibration
476  values (on for each band) to apply to transfer the
477  absolute scale.
478 
479  Parameters
480  ----------
481  butler: `lsst.daf.persistence.Butler`
482  stdCat: `lsst.afw.table.SimpleCatalog`
483  FGCM standard stars
484 
485  Returns
486  -------
487  offsets: `numpy.array` of floats
488  Per band zeropoint offsets
489  """
490 
491  # Only use stars that are observed in all the bands that were actually used
492  # This will ensure that we use the same healpix pixels for the absolute
493  # calibration of each band.
494  minObs = stdCat['ngood'].min(axis=1)
495 
496  goodStars = (minObs >= 1)
497  stdCat = stdCat[goodStars]
498 
499  self.log.info("Found %d stars with at least 1 good observation in each band" %
500  (len(stdCat)))
501 
502  # We have to make a table for each pixel with flux/fluxErr
503  # This is a temporary table generated for input to the photoCal task.
504  # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
505  # have had chromatic corrections applied to get to the standard system
506  # specified by the atmosphere/instrumental parameters), nor are they
507  # in Jansky (since they don't have a proper absolute calibration: the overall
508  # zeropoint is estimated from the telescope size, etc.)
509  sourceMapper = afwTable.SchemaMapper(stdCat.schema)
510  sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
511  sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
512  doc="instrumental flux (counts)")
513  sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
514  doc="instrumental flux error (counts)")
515  badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
516  type='Flag',
517  doc="bad flag")
518 
519  # Split up the stars
520  # Note that there is an assumption here that the ra/dec coords stored
521  # on-disk are in radians, and therefore that starObs['coord_ra'] /
522  # starObs['coord_dec'] return radians when used as an array of numpy float64s.
523  theta = np.pi/2. - stdCat['coord_dec']
524  phi = stdCat['coord_ra']
525 
526  ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
527  h, rev = esutil.stat.histogram(ipring, rev=True)
528 
529  gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
530 
531  self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
532  (gdpix.size,
533  self.config.referencePixelizationNside,
534  self.config.referencePixelizationMinStars))
535 
536  if gdpix.size < self.config.referencePixelizationNPixels:
537  self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
538  (gdpix.size, self.config.referencePixelizationNPixels))
539  else:
540  # Sample out the pixels we want to use
541  gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
542 
543  results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
544  ('nstar', 'i4', len(self.bands)),
545  ('nmatch', 'i4', len(self.bands)),
546  ('zp', 'f4', len(self.bands)),
547  ('zpErr', 'f4', len(self.bands))])
548  results['hpix'] = ipring[rev[rev[gdpix]]]
549 
550  # We need a boolean index to deal with catalogs...
551  selected = np.zeros(len(stdCat), dtype=np.bool)
552 
553  refFluxFields = [None]*len(self.bands)
554 
555  for p, pix in enumerate(gdpix):
556  i1a = rev[rev[pix]: rev[pix + 1]]
557 
558  # the stdCat afwTable can only be indexed with boolean arrays,
559  # and not numpy index arrays (see DM-16497). This little trick
560  # converts the index array into a boolean array
561  selected[:] = False
562  selected[i1a] = True
563 
564  for b, band in enumerate(self.bands):
565 
566  struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
567  selected, refFluxFields)
568  results['nstar'][p, b] = len(i1a)
569  results['nmatch'][p, b] = len(struct.arrays.refMag)
570  results['zp'][p, b] = struct.zp
571  results['zpErr'][p, b] = struct.sigma
572 
573  # And compute the summary statistics
574  offsets = np.zeros(len(self.bands))
575 
576  for b, band in enumerate(self.bands):
577  # make configurable
578  ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)
579  offsets[b] = np.median(results['zp'][ok, b])
580  # use median absolute deviation to estimate Normal sigma
581  # see https://en.wikipedia.org/wiki/Median_absolute_deviation
582  madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b]))
583  self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" %
584  (band, offsets[b], madSigma))
585 
586  return offsets
587 
588  def _computeOffsetOneBand(self, sourceMapper, badStarKey,
589  b, band, stdCat, selected, refFluxFields):
590  """
591  Compute the zeropoint offset between the fgcm stdCat and the reference
592  stars for one pixel in one band
593 
594  Parameters
595  ----------
596  sourceMapper: `lsst.afw.table.SchemaMapper`
597  Mapper to go from stdCat to calibratable catalog
598  badStarKey: `lsst.afw.table.Key`
599  Key for the field with bad stars
600  b: `int`
601  Index of the band in the star catalog
602  band: `str`
603  Name of band for reference catalog
604  stdCat: `lsst.afw.table.SimpleCatalog`
605  FGCM standard stars
606  selected: `numpy.array(dtype=np.bool)`
607  Boolean array of which stars are in the pixel
608  refFluxFields: `list`
609  List of names of flux fields for reference catalog
610  """
611 
612  sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
613  sourceCat.reserve(selected.sum())
614  sourceCat.extend(stdCat[selected], mapper=sourceMapper)
615  sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5))
616  sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] *
617  sourceCat['instFlux'])
618  # Make sure we only use stars that have valid measurements
619  # (This is perhaps redundant with requirements above that the
620  # stars be observed in all bands, but it can't hurt)
621  badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0)
622  for rec in sourceCat[badStar]:
623  rec.set(badStarKey, True)
624 
625  exposure = afwImage.ExposureF()
626  exposure.setFilter(afwImage.Filter(band))
627 
628  if refFluxFields[b] is None:
629  # Need to find the flux field in the reference catalog
630  # to work around limitations of DirectMatch in PhotoCal
631  ctr = stdCat[0].getCoord()
632  rad = 0.05*lsst.geom.degrees
633  refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
634  refFluxFields[b] = refDataTest.fluxField
635 
636  # Make a copy of the config so that we can modify it
637  calConfig = copy.copy(self.config.photoCal.value)
638  calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
639  calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'
640  calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
641  config=calConfig,
642  schema=sourceCat.getSchema())
643 
644  struct = calTask.run(exposure, sourceCat)
645 
646  return struct
647 
648  def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig):
649  """
650  Output standard stars in indexed reference catalog format.
651 
652  Parameters
653  ----------
654  butler: `lsst.daf.persistence.Butler`
655  stdCat: `lsst.afw.table.SimpleCatalog`
656  FGCM standard star catalog from fgcmFitCycleTask
657  offsets: `numpy.array` of floats
658  Per band zeropoint offsets
659  datasetConfig: `lsst.meas.algorithms.DatasetConfig`
660  Config for reference dataset
661  """
662 
663  self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
664 
665  # We determine the conversion from the native units (typically radians) to
666  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
667  # numpy arrays rather than Angles, which would we approximately 600x slower.
668  # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
669  # (as Angles) for input
670  conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
671  indices = np.array(self.indexer.indexPoints(stdCat['coord_ra']*conv,
672  stdCat['coord_dec']*conv))
673 
674  formattedCat = self._formatCatalog(stdCat, offsets)
675 
676  # Write the master schema
677  dataId = self.indexer.makeDataId('master_schema',
678  datasetConfig.ref_dataset_name)
679  masterCat = afwTable.SimpleCatalog(formattedCat.schema)
680  addRefCatMetadata(masterCat)
681  butler.put(masterCat, 'ref_cat', dataId=dataId)
682 
683  # Break up the pixels using a histogram
684  h, rev = esutil.stat.histogram(indices, rev=True)
685  gd, = np.where(h > 0)
686  selected = np.zeros(len(formattedCat), dtype=np.bool)
687  for i in gd:
688  i1a = rev[rev[i]: rev[i + 1]]
689 
690  # the formattedCat afwTable can only be indexed with boolean arrays,
691  # and not numpy index arrays (see DM-16497). This little trick
692  # converts the index array into a boolean array
693  selected[:] = False
694  selected[i1a] = True
695 
696  # Write the individual pixel
697  dataId = self.indexer.makeDataId(indices[i1a[0]],
698  datasetConfig.ref_dataset_name)
699  butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
700 
701  # And save the dataset configuration
702  dataId = self.indexer.makeDataId(None, datasetConfig.ref_dataset_name)
703  butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
704 
705  self.log.info("Done outputting standard stars.")
706 
707  def _formatCatalog(self, fgcmStarCat, offsets):
708  """
709  Turn an FGCM-formatted star catalog, applying zeropoint offsets.
710 
711  Parameters
712  ----------
713  fgcmStarCat: `lsst.afw.Table.SimpleCatalog`
714  SimpleCatalog as output by fgcmcal
715  offsets: `list` with len(self.bands) entries
716  Zeropoint offsets to apply
717 
718  Returns
719  -------
720  formattedCat: `lsst.afw.table.SimpleCatalog`
721  SimpleCatalog suitable for using as a reference catalog
722  """
723 
724  sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
725  minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(self.bands,
726  addCentroid=False,
727  addIsResolved=True,
728  coordErrDim=0)
729  sourceMapper.addMinimalSchema(minSchema)
730  for band in self.bands:
731  sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
732  sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
733  sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
734 
735  formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
736  formattedCat.reserve(len(fgcmStarCat))
737  formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
738 
739  # Note that we don't have to set `resolved` because the default is False
740 
741  for b, band in enumerate(self.bands):
742  mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
743  # We want fluxes in nJy from calibrated AB magnitudes
744  # (after applying offset). Updated after RFC-549 and RFC-575.
745  flux = (mag*units.ABmag).to_value(units.nJy)
746  fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
747 
748  formattedCat['%s_flux' % (band)][:] = flux
749  formattedCat['%s_fluxErr' % (band)][:] = fluxErr
750  formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
751  formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
752  formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
753 
754  addRefCatMetadata(formattedCat)
755 
756  return formattedCat
757 
758  def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None):
759  """
760  Output the zeropoints in fgcm_photoCalib format.
761 
762  Parameters
763  ----------
764  butler: `lsst.daf.persistence.Butler`
765  zptCat: `lsst.afw.table.BaseCatalog`
766  FGCM zeropoint catalog from `FgcmFitCycleTask`
767  visitCat: `lsst.afw.table.BaseCatalog`
768  FGCM visitCat from `FgcmBuildStarsTask`
769  offsets: `numpy.array`
770  Float array of absolute calibration offsets, one for each filter
771  tract: `int`, optional
772  Tract number to output. Default is None (global calibration)
773  """
774 
775  if tract is None:
776  datasetType = 'fgcm_photoCalib'
777  else:
778  datasetType = 'fgcm_tract_photoCalib'
779 
780  self.log.info("Outputting %s objects" % (datasetType))
781 
782  # Select visit/ccds where we have a calibration
783  # This includes ccds where we were able to interpolate from neighboring
784  # ccds.
785  cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
786  too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD']
787  selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) &
788  (zptCat['fgcmZptVar'] > 0.0))
789 
790  # We also select the "best" calibrations, avoiding interpolation. These
791  # are only used for mapping filternames
792  selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) &
793  (zptCat['fgcmZptVar'] > 0.0))
794 
795  # Log warnings for any visit which has no valid zeropoints
796  badVisits = np.unique(zptCat['visit'][~selected])
797  goodVisits = np.unique(zptCat['visit'][selected])
798  allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
799  for allBadVisit in allBadVisits:
800  self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}')
801 
802  # Get the mapping from filtername to dataId filter name, empirically
803  filterMapping = {}
804  nFound = 0
805  for rec in zptCat[selected_best]:
806  if rec['filtername'] in filterMapping:
807  continue
808  dataId = {self.visitDataRefName: int(rec['visit']),
809  self.ccdDataRefName: int(rec['ccd'])}
810  dataRef = butler.dataRef('raw', dataId=dataId)
811  filterMapping[rec['filtername']] = dataRef.dataId['filter']
812  nFound += 1
813  if nFound == len(self.filterMap):
814  break
815 
816  # Get a mapping from filtername to the offsets
817  offsetMapping = {}
818  for f in self.filterMap:
819  # Not every filter in the map will necesarily have a band.
820  if self.filterMap[f] in self.bands:
821  offsetMapping[f] = offsets[self.bands.index(self.filterMap[f])]
822 
823  # Get a mapping from "ccd" to the ccd index used for the scaling
824  camera = butler.get('camera')
825  ccdMapping = {}
826  for ccdIndex, detector in enumerate(camera):
827  ccdMapping[detector.getId()] = ccdIndex
828 
829  # And a mapping to get the flat-field scaling values
830  scalingMapping = {}
831  for rec in visitCat:
832  scalingMapping[rec['visit']] = rec['scaling']
833 
834  if self.config.doComposeWcsJacobian:
835  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
836 
837  for rec in zptCat[selected]:
838 
839  # Retrieve overall scaling
840  scaling = scalingMapping[rec['visit']][ccdMapping[rec['ccd']]]
841 
842  # The postCalibrationOffset describe any zeropoint offsets
843  # to apply after the fgcm calibration. The first part comes
844  # from the reference catalog match (used in testing). The
845  # second part comes from the mean chromatic correction
846  # (if configured).
847  postCalibrationOffset = offsetMapping[rec['filtername']]
848  if self.config.doApplyMeanChromaticCorrection:
849  postCalibrationOffset += rec['fgcmDeltaChrom']
850 
851  fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
852  rec['fgcmfZptChebXyMax'])
853  # Convert from FGCM AB to nJy
854  fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
855  rec['fgcmfZptChebXyMax'],
856  offset=postCalibrationOffset,
857  scaling=scaling)
858 
859  if self.config.doComposeWcsJacobian:
860 
861  fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['ccd']],
862  fgcmSuperStarField,
863  fgcmZptField])
864  else:
865  # The photoCalib is just the product of the fgcmSuperStarField and the
866  # fgcmZptField
867  fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
868 
869  # The "mean" calibration will be set to the center of the ccd for reference
870  calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
871  calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
872  photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
873  calibrationErr=calibErr,
874  calibration=fgcmField,
875  isConstant=False)
876 
877  if tract is None:
878  butler.put(photoCalib, datasetType,
879  dataId={self.visitDataRefName: int(rec['visit']),
880  self.ccdDataRefName: int(rec['ccd']),
881  'filter': filterMapping[rec['filtername']]})
882  else:
883  butler.put(photoCalib, datasetType,
884  dataId={self.visitDataRefName: int(rec['visit']),
885  self.ccdDataRefName: int(rec['ccd']),
886  'filter': filterMapping[rec['filtername']],
887  'tract': tract})
888 
889  self.log.info("Done outputting %s objects" % (datasetType))
890 
891  def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
892  """
893  Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
894  and scaling.
895 
896  Parameters
897  ----------
898  coefficients: `numpy.array`
899  Flattened array of chebyshev coefficients
900  xyMax: `list` of length 2
901  Maximum x and y of the chebyshev bounding box
902  offset: `float`, optional
903  Absolute calibration offset. Default is 0.0
904  scaling: `float`, optional
905  Flat scaling value from fgcmBuildStars. Default is 1.0
906 
907  Returns
908  -------
909  boundedField: `lsst.afw.math.ChebyshevBoundedField`
910  """
911 
912  orderPlus1 = int(np.sqrt(coefficients.size))
913  pars = np.zeros((orderPlus1, orderPlus1))
914 
915  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
916  lsst.geom.Point2I(*xyMax))
917 
918  pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) *
919  (10.**(offset/-2.5))*scaling)
920 
921  boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
922 
923  return boundedField
924 
925  def _outputAtmospheres(self, butler, atmCat, tract=None):
926  """
927  Output the atmospheres.
928 
929  Parameters
930  ----------
931  butler: `lsst.daf.persistence.Butler`
932  atmCat: `lsst.afw.table.BaseCatalog`
933  FGCM atmosphere parameter catalog from fgcmFitCycleTask
934  tract: `int`, optional
935  Tract number to output. Default is None (global calibration)
936  """
937 
938  self.log.info("Outputting atmosphere transmissions")
939 
940  # First, we need to grab the look-up table and key info
941  lutCat = butler.get('fgcmLookUpTable')
942 
943  atmosphereTableName = lutCat[0]['tablename']
944  elevation = lutCat[0]['elevation']
945  atmLambda = lutCat[0]['atmLambda']
946  lutCat = None
947 
948  # Make the atmosphere table if possible
949  try:
950  atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
951  atmTable.loadTable()
952  except IOError:
953  atmTable = None
954 
955  if atmTable is None:
956  # Try to use MODTRAN instead
957  try:
958  modGen = fgcm.ModtranGenerator(elevation)
959  lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
960  lambdaStep = (atmLambda[1] - atmLambda[0])/10.
961  except (ValueError, IOError) as e:
962  raise RuntimeError("FGCM look-up-table generated with modtran, "
963  "but modtran not configured to run.") from e
964 
965  zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
966 
967  for i, visit in enumerate(atmCat['visit']):
968  if atmTable is not None:
969  # Interpolate the atmosphere table
970  atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
971  pwv=atmCat[i]['pwv'],
972  o3=atmCat[i]['o3'],
973  tau=atmCat[i]['tau'],
974  alpha=atmCat[i]['alpha'],
975  zenith=zenith[i],
976  ctranslamstd=[atmCat[i]['cTrans'],
977  atmCat[i]['lamStd']])
978  else:
979  # Run modtran
980  modAtm = modGen(pmb=atmCat[i]['pmb'],
981  pwv=atmCat[i]['pwv'],
982  o3=atmCat[i]['o3'],
983  tau=atmCat[i]['tau'],
984  alpha=atmCat[i]['alpha'],
985  zenith=zenith[i],
986  lambdaRange=lambdaRange,
987  lambdaStep=lambdaStep,
988  ctranslamstd=[atmCat[i]['cTrans'],
989  atmCat[i]['lamStd']])
990  atmVals = modAtm['COMBINED']
991 
992  # Now need to create something to persist...
993  curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
994  wavelengths=atmLambda,
995  throughputAtMin=atmVals[0],
996  throughputAtMax=atmVals[-1])
997 
998  if tract is None:
999  butler.put(curve, "transmission_atmosphere_fgcm",
1000  dataId={self.visitDataRefName: visit})
1001  else:
1002  butler.put(curve, "transmission_atmosphere_fgcm_tract",
1003  dataId={self.visitDataRefName: visit,
1004  'tract': tract})
1005 
1006  self.log.info("Done outputting atmosphere transmissions")
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig.datasetConfig
datasetConfig
Definition: fgcmOutputProducts.py:139
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.filterMap
filterMap
Definition: fgcmOutputProducts.py:322
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._computeOffsetOneBand
def _computeOffsetOneBand(self, sourceMapper, badStarKey, b, band, stdCat, selected, refFluxFields)
Definition: fgcmOutputProducts.py:588
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.bands
bands
Definition: fgcmOutputProducts.py:368
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._outputZeropoints
def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None)
Definition: fgcmOutputProducts.py:758
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.__init__
def __init__(self, butler=None, **kwargs)
Definition: fgcmOutputProducts.py:251
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.generateTractOutputProducts
def generateTractOutputProducts(self, butler, tract, visitCat, zptCat, atmCat, stdCat, fgcmBuildStarsConfig, fgcmFitCycleConfig)
Definition: fgcmOutputProducts.py:399
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._getChebyshevBoundedField
def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0)
Definition: fgcmOutputProducts.py:891
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig.photoCal
photoCal
Definition: fgcmOutputProducts.py:111
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner
Definition: fgcmOutputProducts.py:172
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig
Definition: fgcmOutputProducts.py:64
lsst.fgcmcal.utilities.computeApproxPixelAreaFields
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:474
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.visitDataRefName
visitDataRefName
Definition: fgcmOutputProducts.py:320
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._outputStandardStars
def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig)
Definition: fgcmOutputProducts.py:648
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig.setDefaults
def setDefaults(self)
Definition: fgcmOutputProducts.py:144
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
Definition: fgcmOutputProducts.py:242
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._formatCatalog
def _formatCatalog(self, fgcmStarCat, offsets)
Definition: fgcmOutputProducts.py:707
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.configBands
configBands
Definition: fgcmOutputProducts.py:337
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner.__call__
def __call__(self, butler)
Definition: fgcmOutputProducts.py:187
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.indexer
indexer
Definition: fgcmOutputProducts.py:267
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner.run
def run(self, parsedCmd)
Definition: fgcmOutputProducts.py:223
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.runDataRef
def runDataRef(self, butler)
Definition: fgcmOutputProducts.py:275
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._outputAtmospheres
def _outputAtmospheres(self, butler, atmCat, tract=None)
Definition: fgcmOutputProducts.py:925
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner.getTargetList
def getTargetList(parsedCmd)
Definition: fgcmOutputProducts.py:181
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._computeReferenceOffsets
def _computeReferenceOffsets(self, butler, stdCat)
Definition: fgcmOutputProducts.py:469
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.ccdDataRefName
ccdDataRefName
Definition: fgcmOutputProducts.py:321