lsst.fgcmcal  19.0.0-20-g3336a1e+6
fgcmOutputProducts.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Make the final fgcmcal output products.
24 
25 This task takes the final output from fgcmFitCycle and produces the following
26 outputs for use in the DM stack: the FGCM standard stars in a reference
27 catalog format; the model atmospheres in "transmission_atmosphere_fgcm"
28 format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the
29 task can transfer the 'absolute' calibration from a reference catalog
30 to put the fgcm standard stars in units of Jansky. This is accomplished
31 by matching stars in a sample of healpix pixels, and applying the median
32 offset per band.
33 """
34 
35 import sys
36 import traceback
37 import copy
38 
39 import numpy as np
40 import healpy as hp
41 import esutil
42 from astropy import units
43 
44 import lsst.pex.config as pexConfig
45 import lsst.pipe.base as pipeBase
46 from lsst.afw.image import TransmissionCurve
47 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
48 from lsst.pipe.tasks.photoCal import PhotoCalTask
49 import lsst.geom
50 import lsst.afw.image as afwImage
51 import lsst.afw.math as afwMath
52 import lsst.afw.table as afwTable
53 from lsst.meas.algorithms import IndexerRegistry
54 from lsst.meas.algorithms import DatasetConfig
55 from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
56 
57 from .utilities import computeApproxPixelAreaFields
58 
59 import fgcm
60 
61 __all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']
62 
63 
64 class FgcmOutputProductsConfig(pexConfig.Config):
65  """Config for FgcmOutputProductsTask"""
66 
67  cycleNumber = pexConfig.Field(
68  doc="Final fit cycle from FGCM fit",
69  dtype=int,
70  default=None,
71  )
72 
73  # The following fields refer to calibrating from a reference
74  # catalog, but in the future this might need to be expanded
75  doReferenceCalibration = pexConfig.Field(
76  doc=("Transfer 'absolute' calibration from reference catalog? "
77  "This afterburner step is unnecessary if reference stars "
78  "were used in the full fit in FgcmFitCycleTask."),
79  dtype=bool,
80  default=False,
81  )
82  doRefcatOutput = pexConfig.Field(
83  doc="Output standard stars in reference catalog format",
84  dtype=bool,
85  default=True,
86  )
87  doAtmosphereOutput = pexConfig.Field(
88  doc="Output atmospheres in transmission_atmosphere_fgcm format",
89  dtype=bool,
90  default=True,
91  )
92  doZeropointOutput = pexConfig.Field(
93  doc="Output zeropoints in fgcm_photoCalib format",
94  dtype=bool,
95  default=True,
96  )
97  doComposeWcsJacobian = pexConfig.Field(
98  doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
99  dtype=bool,
100  default=True,
101  )
102  refObjLoader = pexConfig.ConfigurableField(
103  target=LoadIndexedReferenceObjectsTask,
104  doc="reference object loader for 'absolute' photometric calibration",
105  )
106  photoCal = pexConfig.ConfigurableField(
107  target=PhotoCalTask,
108  doc="task to perform 'absolute' calibration",
109  )
110  referencePixelizationNside = pexConfig.Field(
111  doc="Healpix nside to pixelize catalog to compare to reference catalog",
112  dtype=int,
113  default=64,
114  )
115  referencePixelizationMinStars = pexConfig.Field(
116  doc=("Minimum number of stars per healpix pixel to select for comparison"
117  "to the specified reference catalog"),
118  dtype=int,
119  default=200,
120  )
121  referenceMinMatch = pexConfig.Field(
122  doc="Minimum number of stars matched to reference catalog to be used in statistics",
123  dtype=int,
124  default=50,
125  )
126  referencePixelizationNPixels = pexConfig.Field(
127  doc=("Number of healpix pixels to sample to do comparison. "
128  "Doing too many will take a long time and not yield any more "
129  "precise results because the final number is the median offset "
130  "(per band) from the set of pixels."),
131  dtype=int,
132  default=100,
133  )
134  datasetConfig = pexConfig.ConfigField(
135  dtype=DatasetConfig,
136  doc="Configuration for writing/reading ingested catalog",
137  )
138 
139  def setDefaults(self):
140  pexConfig.Config.setDefaults(self)
141 
142  # In order to transfer the "absolute" calibration from a reference
143  # catalog to the relatively calibrated FGCM standard stars (one number
144  # per band), we use the PhotoCalTask to match stars in a sample of healpix
145  # pixels. These basic settings ensure that only well-measured, good stars
146  # from the source and reference catalogs are used for the matching.
147 
148  # applyColorTerms needs to be False if doReferenceCalibration is False,
149  # as is the new default after DM-16702
150  self.photoCal.applyColorTerms = False
151  self.photoCal.fluxField = 'instFlux'
152  self.photoCal.magErrFloor = 0.003
153  self.photoCal.match.referenceSelection.doSignalToNoise = True
154  self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
155  self.photoCal.match.sourceSelection.doSignalToNoise = True
156  self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
157  self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
158  self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
159  self.photoCal.match.sourceSelection.doFlags = True
160  self.photoCal.match.sourceSelection.flags.good = []
161  self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
162  self.photoCal.match.sourceSelection.doUnresolved = False
163  self.datasetConfig.ref_dataset_name = 'fgcm_stars'
164  self.datasetConfig.format_version = 1
165 
166 
167 class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):
168  """Subclass of TaskRunner for fgcmOutputProductsTask
169 
170  fgcmOutputProductsTask.run() takes one argument, the butler, and
171  does not run on any data in the repository.
172  This runner does not use any parallelization.
173  """
174 
175  @staticmethod
176  def getTargetList(parsedCmd):
177  """
178  Return a list with one element, the butler.
179  """
180  return [parsedCmd.butler]
181 
182  def __call__(self, butler):
183  """
184  Parameters
185  ----------
186  butler: `lsst.daf.persistence.Butler`
187 
188  Returns
189  -------
190  exitStatus: `list` with `pipeBase.Struct`
191  exitStatus (0: success; 1: failure)
192  if self.doReturnResults also
193  results (`np.array` with absolute zeropoint offsets)
194  """
195  task = self.TaskClass(butler=butler, config=self.config, log=self.log)
196 
197  exitStatus = 0
198  if self.doRaise:
199  results = task.runDataRef(butler)
200  else:
201  try:
202  results = task.runDataRef(butler)
203  except Exception as e:
204  exitStatus = 1
205  task.log.fatal("Failed: %s" % e)
206  if not isinstance(e, pipeBase.TaskError):
207  traceback.print_exc(file=sys.stderr)
208 
209  task.writeMetadata(butler)
210 
211  if self.doReturnResults:
212  # The results here are the zeropoint offsets for each band
213  return [pipeBase.Struct(exitStatus=exitStatus,
214  results=results)]
215  else:
216  return [pipeBase.Struct(exitStatus=exitStatus)]
217 
218  def run(self, parsedCmd):
219  """
220  Run the task, with no multiprocessing
221 
222  Parameters
223  ----------
224  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
225  """
226 
227  resultList = []
228 
229  if self.precall(parsedCmd):
230  targetList = self.getTargetList(parsedCmd)
231  # make sure that we only get 1
232  resultList = self(targetList[0])
233 
234  return resultList
235 
236 
237 class FgcmOutputProductsTask(pipeBase.CmdLineTask):
238  """
239  Output products from FGCM global calibration.
240  """
241 
242  ConfigClass = FgcmOutputProductsConfig
243  RunnerClass = FgcmOutputProductsRunner
244  _DefaultName = "fgcmOutputProducts"
245 
246  def __init__(self, butler=None, **kwargs):
247  """
248  Instantiate an fgcmOutputProductsTask.
249 
250  Parameters
251  ----------
252  butler : `lsst.daf.persistence.Butler`
253  """
254 
255  pipeBase.CmdLineTask.__init__(self, **kwargs)
256 
257  if self.config.doReferenceCalibration:
258  # We need the ref obj loader to get the flux field
259  self.makeSubtask("refObjLoader", butler=butler)
260 
261  if self.config.doRefcatOutput:
262  self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](
263  self.config.datasetConfig.indexer.active)
264 
265  # no saving of metadata for now
266  def _getMetadataName(self):
267  return None
268 
269  @pipeBase.timeMethod
270  def runDataRef(self, butler):
271  """
272  Make FGCM output products for use in the stack
273 
274  Parameters
275  ----------
276  butler: `lsst.daf.persistence.Butler`
277  cycleNumber: `int`
278  Final fit cycle number, override config.
279 
280  Returns
281  -------
282  offsets: `lsst.pipe.base.Struct`
283  A structure with array of zeropoint offsets
284 
285  Raises
286  ------
287  RuntimeError: Raised if butler cannot find fgcmBuildStars_config, or
288  fgcmFitCycle_config, or fgcmAtmosphereParameters (and
289  `self.config.doAtmosphereOutput` is true), or fgcmStandardStars (and
290  `self.config.doReferenceCalibration or `self.config.doRefcatOutput`
291  is true), or fgcmZeropoints (and self.config.doZeropointOutput is true).
292  Also will raise if the fgcmFitCycle_config does not refer to the
293  final fit cycle.
294  """
295 
296  # Check to make sure that the fgcmBuildStars config exists, to retrieve
297  # the visit and ccd dataset tags
298  if not butler.datasetExists('fgcmBuildStars_config'):
299  raise RuntimeError("Cannot find fgcmBuildStars_config, which is prereq for fgcmOutputProducts")
300 
301  fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')
302  self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
303  self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
304  self.filterMap = fgcmBuildStarsConfig.filterMap
305 
306  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
307  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
308  "in fgcmBuildStarsTask.")
309 
310  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
311  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
312 
313  # Make sure that the fit config exists, to retrieve bands and other info
314  if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber):
315  raise RuntimeError("Cannot find fgcmFitCycle_config from cycle %d " % (self.config.cycleNumber) +
316  "which is required for fgcmOutputProducts.")
317 
318  fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber)
319  self.configBands = fitCycleConfig.bands
320 
321  if self.config.doReferenceCalibration and fitCycleConfig.doReferenceCalibration:
322  self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
323  "fitCycleConfig.doReferenceCalibration")
324 
325  # And make sure that the atmosphere was output properly
326  if (self.config.doAtmosphereOutput and
327  not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)):
328  raise RuntimeError("Atmosphere parameters are missing for cycle %d." %
329  (self.config.cycleNumber))
330 
331  if ((self.config.doReferenceCalibration or self.config.doRefcatOutput) and
332  (not butler.datasetExists('fgcmStandardStars',
333  fgcmcycle=self.config.cycleNumber))):
334  raise RuntimeError("Standard stars are missing for cycle %d." %
335  (self.config.cycleNumber))
336 
337  if (self.config.doZeropointOutput and
338  (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))):
339  raise RuntimeError("Zeropoints are missing for cycle %d." %
340  (self.config.cycleNumber))
341 
342  # And make sure this is the last cycle
343  if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1):
344  raise RuntimeError("The task fgcmOutputProducts should only be run"
345  "on the final fit cycle products")
346 
347  if self.config.doReferenceCalibration or self.config.doRefcatOutput:
348  stdCat = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
349  md = stdCat.getMetadata()
350  self.bands = md.getArray('BANDS')
351  else:
352  stdCat = None
353  self.bands = self.configBands
354 
355  if self.config.doReferenceCalibration:
356  offsets = self._computeReferenceOffsets(butler, stdCat)
357  else:
358  offsets = np.zeros(len(self.bands))
359 
360  # Output the standard stars in stack format
361  if self.config.doRefcatOutput:
362  self._outputStandardStars(butler, stdCat, offsets, self.config.datasetConfig)
363 
364  del stdCat
365 
366  # Output the gray zeropoints
367  if self.config.doZeropointOutput:
368  zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
369  visitCat = butler.get('fgcmVisitCatalog')
370 
371  self._outputZeropoints(butler, zptCat, visitCat, offsets)
372 
373  # Output the atmospheres
374  if self.config.doAtmosphereOutput:
375  atmCat = butler.get('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
376  self._outputAtmospheres(butler, atmCat)
377 
378  # We return the zp offsets
379  return pipeBase.Struct(offsets=offsets)
380 
381  def generateTractOutputProducts(self, butler, tract,
382  visitCat, zptCat, atmCat, stdCat,
383  fgcmBuildStarsConfig, fgcmFitCycleConfig):
384  """
385  Generate the output products for a given tract, as specified in the config.
386 
387  This method is here to have an alternate entry-point for
388  FgcmCalibrateTract.
389 
390  Parameters
391  ----------
392  butler: `lsst.daf.persistence.Butler`
393  tract: `int`
394  Tract number
395  visitCat: `lsst.afw.table.BaseCatalog`
396  FGCM visitCat from `FgcmBuildStarsTask`
397  zptCat: `lsst.afw.table.BaseCatalog`
398  FGCM zeropoint catalog from `FgcmFitCycleTask`
399  atmCat: `lsst.afw.table.BaseCatalog`
400  FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
401  stdCat: `lsst.afw.table.SimpleCatalog`
402  FGCM standard star catalog from `FgcmFitCycleTask`
403  fgcmBuildStarsConfig: `lsst.fgcmcal.FgcmBuildStarsConfig`
404  Configuration object from `FgcmBuildStarsTask`
405  fgcmFitCycleConfig: `lsst.fgcmcal.FgcmFitCycleConfig`
406  Configuration object from `FgcmFitCycleTask`
407  """
408 
409  self.configBands = fgcmFitCycleConfig.bands
410  self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName
411  self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName
412  self.filterMap = fgcmBuildStarsConfig.filterMap
413 
414  if stdCat is not None:
415  md = stdCat.getMetadata()
416  self.bands = md.getArray('BANDS')
417  else:
418  self.bands = self.configBands
419 
420  if self.config.doReferenceCalibration and fgcmFitCycleConfig.doReferenceCalibration:
421  self.log.warn("doReferenceCalibration is set, and is possibly redundant with "
422  "fitCycleConfig.doReferenceCalibration")
423 
424  if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
425  raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
426  "in fgcmBuildStarsTask.")
427 
428  if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
429  self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
430 
431  if self.config.doReferenceCalibration:
432  offsets = self._computeReferenceOffsets(butler, stdCat)
433  else:
434  offsets = np.zeros(len(self.bands))
435 
436  if self.config.doRefcatOutput:
437  # Create a special config that has the tract number in it
438  datasetConfig = copy.copy(self.config.datasetConfig)
439  datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name,
440  tract)
441  self._outputStandardStars(butler, stdCat, offsets, datasetConfig)
442 
443  if self.config.doZeropointOutput:
444  self._outputZeropoints(butler, zptCat, visitCat, offsets, tract=tract)
445 
446  if self.config.doAtmosphereOutput:
447  self._outputAtmospheres(butler, atmCat, tract=tract)
448 
449  return pipeBase.Struct(offsets=offsets)
450 
451  def _computeReferenceOffsets(self, butler, stdCat):
452  """
453  Compute offsets relative to a reference catalog.
454 
455  This method splits the star catalog into healpix pixels
456  and computes the calibration transfer for a sample of
457  these pixels to approximate the 'absolute' calibration
458  values (on for each band) to apply to transfer the
459  absolute scale.
460 
461  Parameters
462  ----------
463  butler: `lsst.daf.persistence.Butler`
464  stdCat: `lsst.afw.table.SimpleCatalog`
465  FGCM standard stars
466 
467  Returns
468  -------
469  offsets: `numpy.array` of floats
470  Per band zeropoint offsets
471  """
472 
473  # Only use stars that are observed in all the bands that were actually used
474  # This will ensure that we use the same healpix pixels for the absolute
475  # calibration of each band.
476  minObs = stdCat['ngood'].min(axis=1)
477 
478  goodStars = (minObs >= 1)
479  stdCat = stdCat[goodStars]
480 
481  self.log.info("Found %d stars with at least 1 good observation in each band" %
482  (len(stdCat)))
483 
484  # We have to make a table for each pixel with flux/fluxErr
485  # This is a temporary table generated for input to the photoCal task.
486  # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
487  # have had chromatic corrections applied to get to the standard system
488  # specified by the atmosphere/instrumental parameters), nor are they
489  # in Jansky (since they don't have a proper absolute calibration: the overall
490  # zeropoint is estimated from the telescope size, etc.)
491  sourceMapper = afwTable.SchemaMapper(stdCat.schema)
492  sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
493  sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
494  doc="instrumental flux (counts)")
495  sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
496  doc="instrumental flux error (counts)")
497  badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
498  type='Flag',
499  doc="bad flag")
500 
501  # Split up the stars
502  # Note that there is an assumption here that the ra/dec coords stored
503  # on-disk are in radians, and therefore that starObs['coord_ra'] /
504  # starObs['coord_dec'] return radians when used as an array of numpy float64s.
505  theta = np.pi/2. - stdCat['coord_dec']
506  phi = stdCat['coord_ra']
507 
508  ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
509  h, rev = esutil.stat.histogram(ipring, rev=True)
510 
511  gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
512 
513  self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
514  (gdpix.size,
515  self.config.referencePixelizationNside,
516  self.config.referencePixelizationMinStars))
517 
518  if gdpix.size < self.config.referencePixelizationNPixels:
519  self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" %
520  (gdpix.size, self.config.referencePixelizationNPixels))
521  else:
522  # Sample out the pixels we want to use
523  gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
524 
525  results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
526  ('nstar', 'i4', len(self.bands)),
527  ('nmatch', 'i4', len(self.bands)),
528  ('zp', 'f4', len(self.bands)),
529  ('zpErr', 'f4', len(self.bands))])
530  results['hpix'] = ipring[rev[rev[gdpix]]]
531 
532  # We need a boolean index to deal with catalogs...
533  selected = np.zeros(len(stdCat), dtype=np.bool)
534 
535  refFluxFields = [None]*len(self.bands)
536 
537  for p, pix in enumerate(gdpix):
538  i1a = rev[rev[pix]: rev[pix + 1]]
539 
540  # the stdCat afwTable can only be indexed with boolean arrays,
541  # and not numpy index arrays (see DM-16497). This little trick
542  # converts the index array into a boolean array
543  selected[:] = False
544  selected[i1a] = True
545 
546  for b, band in enumerate(self.bands):
547 
548  struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat,
549  selected, refFluxFields)
550  results['nstar'][p, b] = len(i1a)
551  results['nmatch'][p, b] = len(struct.arrays.refMag)
552  results['zp'][p, b] = struct.zp
553  results['zpErr'][p, b] = struct.sigma
554 
555  # And compute the summary statistics
556  offsets = np.zeros(len(self.bands))
557 
558  for b, band in enumerate(self.bands):
559  # make configurable
560  ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)
561  offsets[b] = np.median(results['zp'][ok, b])
562  # use median absolute deviation to estimate Normal sigma
563  # see https://en.wikipedia.org/wiki/Median_absolute_deviation
564  madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b]))
565  self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" %
566  (band, offsets[b], madSigma))
567 
568  return offsets
569 
570  def _computeOffsetOneBand(self, sourceMapper, badStarKey,
571  b, band, stdCat, selected, refFluxFields):
572  """
573  Compute the zeropoint offset between the fgcm stdCat and the reference
574  stars for one pixel in one band
575 
576  Parameters
577  ----------
578  sourceMapper: `lsst.afw.table.SchemaMapper`
579  Mapper to go from stdCat to calibratable catalog
580  badStarKey: `lsst.afw.table.Key`
581  Key for the field with bad stars
582  b: `int`
583  Index of the band in the star catalog
584  band: `str`
585  Name of band for reference catalog
586  stdCat: `lsst.afw.table.SimpleCatalog`
587  FGCM standard stars
588  selected: `numpy.array(dtype=np.bool)`
589  Boolean array of which stars are in the pixel
590  refFluxFields: `list`
591  List of names of flux fields for reference catalog
592  """
593 
594  sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
595  sourceCat.reserve(selected.sum())
596  sourceCat.extend(stdCat[selected], mapper=sourceMapper)
597  sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5))
598  sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] *
599  sourceCat['instFlux'])
600  # Make sure we only use stars that have valid measurements
601  # (This is perhaps redundant with requirements above that the
602  # stars be observed in all bands, but it can't hurt)
603  badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0)
604  for rec in sourceCat[badStar]:
605  rec.set(badStarKey, True)
606 
607  exposure = afwImage.ExposureF()
608  exposure.setFilter(afwImage.Filter(band))
609 
610  if refFluxFields[b] is None:
611  # Need to find the flux field in the reference catalog
612  # to work around limitations of DirectMatch in PhotoCal
613  ctr = stdCat[0].getCoord()
614  rad = 0.05*lsst.geom.degrees
615  refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)
616  refFluxFields[b] = refDataTest.fluxField
617 
618  # Make a copy of the config so that we can modify it
619  calConfig = copy.copy(self.config.photoCal.value)
620  calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]
621  calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'
622  calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
623  config=calConfig,
624  schema=sourceCat.getSchema())
625 
626  struct = calTask.run(exposure, sourceCat)
627 
628  return struct
629 
630  def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig):
631  """
632  Output standard stars in indexed reference catalog format.
633 
634  Parameters
635  ----------
636  butler: `lsst.daf.persistence.Butler`
637  stdCat: `lsst.afw.table.SimpleCatalog`
638  FGCM standard star catalog from fgcmFitCycleTask
639  offsets: `numpy.array` of floats
640  Per band zeropoint offsets
641  datasetConfig: `lsst.meas.algorithms.DatasetConfig`
642  Config for reference dataset
643  """
644 
645  self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name))
646 
647  # We determine the conversion from the native units (typically radians) to
648  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
649  # numpy arrays rather than Angles, which would we approximately 600x slower.
650  # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords
651  # (as Angles) for input
652  conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra'])
653  indices = np.array(self.indexer.indexPoints(stdCat['coord_ra']*conv,
654  stdCat['coord_dec']*conv))
655 
656  formattedCat = self._formatCatalog(stdCat, offsets)
657 
658  # Write the master schema
659  dataId = self.indexer.makeDataId('master_schema',
660  datasetConfig.ref_dataset_name)
661  masterCat = afwTable.SimpleCatalog(formattedCat.schema)
662  addRefCatMetadata(masterCat)
663  butler.put(masterCat, 'ref_cat', dataId=dataId)
664 
665  # Break up the pixels using a histogram
666  h, rev = esutil.stat.histogram(indices, rev=True)
667  gd, = np.where(h > 0)
668  selected = np.zeros(len(formattedCat), dtype=np.bool)
669  for i in gd:
670  i1a = rev[rev[i]: rev[i + 1]]
671 
672  # the formattedCat afwTable can only be indexed with boolean arrays,
673  # and not numpy index arrays (see DM-16497). This little trick
674  # converts the index array into a boolean array
675  selected[:] = False
676  selected[i1a] = True
677 
678  # Write the individual pixel
679  dataId = self.indexer.makeDataId(indices[i1a[0]],
680  datasetConfig.ref_dataset_name)
681  butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)
682 
683  # And save the dataset configuration
684  dataId = self.indexer.makeDataId(None, datasetConfig.ref_dataset_name)
685  butler.put(datasetConfig, 'ref_cat_config', dataId=dataId)
686 
687  self.log.info("Done outputting standard stars.")
688 
689  def _formatCatalog(self, fgcmStarCat, offsets):
690  """
691  Turn an FGCM-formatted star catalog, applying zeropoint offsets.
692 
693  Parameters
694  ----------
695  fgcmStarCat: `lsst.afw.Table.SimpleCatalog`
696  SimpleCatalog as output by fgcmcal
697  offsets: `list` with len(self.bands) entries
698  Zeropoint offsets to apply
699 
700  Returns
701  -------
702  formattedCat: `lsst.afw.table.SimpleCatalog`
703  SimpleCatalog suitable for using as a reference catalog
704  """
705 
706  sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
707  minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(self.bands,
708  addCentroid=False,
709  addIsResolved=True,
710  coordErrDim=0)
711  sourceMapper.addMinimalSchema(minSchema)
712  for band in self.bands:
713  sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32)
714  sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32)
715  sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32)
716 
717  formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
718  formattedCat.reserve(len(fgcmStarCat))
719  formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
720 
721  # Note that we don't have to set `resolved` because the default is False
722 
723  for b, band in enumerate(self.bands):
724  mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b]
725  # We want fluxes in nJy from calibrated AB magnitudes
726  # (after applying offset). Updated after RFC-549 and RFC-575.
727  flux = (mag*units.ABmag).to_value(units.nJy)
728  fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64)
729 
730  formattedCat['%s_flux' % (band)][:] = flux
731  formattedCat['%s_fluxErr' % (band)][:] = fluxErr
732  formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
733  formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
734  formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b]
735 
736  addRefCatMetadata(formattedCat)
737 
738  return formattedCat
739 
740  def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None):
741  """
742  Output the zeropoints in fgcm_photoCalib format.
743 
744  Parameters
745  ----------
746  butler: `lsst.daf.persistence.Butler`
747  zptCat: `lsst.afw.table.BaseCatalog`
748  FGCM zeropoint catalog from `FgcmFitCycleTask`
749  visitCat: `lsst.afw.table.BaseCatalog`
750  FGCM visitCat from `FgcmBuildStarsTask`
751  offsets: `numpy.array`
752  Float array of absolute calibration offsets, one for each filter
753  tract: `int`, optional
754  Tract number to output. Default is None (global calibration)
755  """
756 
757  if tract is None:
758  datasetType = 'fgcm_photoCalib'
759  else:
760  datasetType = 'fgcm_tract_photoCalib'
761 
762  self.log.info("Outputting %s objects" % (datasetType))
763 
764  # Select visit/ccds where we have a calibration
765  # This includes ccds where we were able to interpolate from neighboring
766  # ccds.
767  cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
768  too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD']
769  selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) &
770  (zptCat['fgcmZptVar'] > 0.0))
771 
772  # We also select the "best" calibrations, avoiding interpolation. These
773  # are only used for mapping filternames
774  selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) &
775  (zptCat['fgcmZptVar'] > 0.0))
776 
777  # Log warnings for any visit which has no valid zeropoints
778  badVisits = np.unique(zptCat['visit'][~selected])
779  goodVisits = np.unique(zptCat['visit'][selected])
780  allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
781  for allBadVisit in allBadVisits:
782  self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}')
783 
784  # Get the mapping from filtername to dataId filter name, empirically
785  filterMapping = {}
786  nFound = 0
787  for rec in zptCat[selected_best]:
788  if rec['filtername'] in filterMapping:
789  continue
790  dataId = {self.visitDataRefName: int(rec['visit']),
791  self.ccdDataRefName: int(rec['ccd'])}
792  dataRef = butler.dataRef('raw', dataId=dataId)
793  filterMapping[rec['filtername']] = dataRef.dataId['filter']
794  nFound += 1
795  if nFound == len(self.filterMap):
796  break
797 
798  # Get a mapping from filtername to the offsets
799  offsetMapping = {}
800  for f in self.filterMap:
801  # Not every filter in the map will necesarily have a band.
802  if self.filterMap[f] in self.bands:
803  offsetMapping[f] = offsets[self.bands.index(self.filterMap[f])]
804 
805  # Get a mapping from "ccd" to the ccd index used for the scaling
806  camera = butler.get('camera')
807  ccdMapping = {}
808  for ccdIndex, detector in enumerate(camera):
809  ccdMapping[detector.getId()] = ccdIndex
810 
811  # And a mapping to get the flat-field scaling values
812  scalingMapping = {}
813  for rec in visitCat:
814  scalingMapping[rec['visit']] = rec['scaling']
815 
816  if self.config.doComposeWcsJacobian:
817  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
818 
819  for rec in zptCat[selected]:
820 
821  # Retrieve overall scaling
822  scaling = scalingMapping[rec['visit']][ccdMapping[rec['ccd']]]
823 
824  fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'],
825  rec['fgcmfZptChebXyMax'])
826  # Convert from FGCM AB to nJy
827  fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy),
828  rec['fgcmfZptChebXyMax'],
829  offset=offsetMapping[rec['filtername']],
830  scaling=scaling)
831 
832  if self.config.doComposeWcsJacobian:
833 
834  fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['ccd']],
835  fgcmSuperStarField,
836  fgcmZptField])
837  else:
838  # The photoCalib is just the product of the fgcmSuperStarField and the
839  # fgcmZptField
840  fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField])
841 
842  # The "mean" calibration will be set to the center of the ccd for reference
843  calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
844  calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar'])
845  photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
846  calibrationErr=calibErr,
847  calibration=fgcmField,
848  isConstant=False)
849 
850  if tract is None:
851  butler.put(photoCalib, datasetType,
852  dataId={self.visitDataRefName: int(rec['visit']),
853  self.ccdDataRefName: int(rec['ccd']),
854  'filter': filterMapping[rec['filtername']]})
855  else:
856  butler.put(photoCalib, datasetType,
857  dataId={self.visitDataRefName: int(rec['visit']),
858  self.ccdDataRefName: int(rec['ccd']),
859  'filter': filterMapping[rec['filtername']],
860  'tract': tract})
861 
862  self.log.info("Done outputting %s objects" % (datasetType))
863 
864  def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0):
865  """
866  Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
867  and scaling.
868 
869  Parameters
870  ----------
871  coefficients: `numpy.array`
872  Flattened array of chebyshev coefficients
873  xyMax: `list` of length 2
874  Maximum x and y of the chebyshev bounding box
875  offset: `float`, optional
876  Absolute calibration offset. Default is 0.0
877  scaling: `float`, optional
878  Flat scaling value from fgcmBuildStars. Default is 1.0
879 
880  Returns
881  -------
882  boundedField: `lsst.afw.math.ChebyshevBoundedField`
883  """
884 
885  orderPlus1 = int(np.sqrt(coefficients.size))
886  pars = np.zeros((orderPlus1, orderPlus1))
887 
888  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
889  lsst.geom.Point2I(*xyMax))
890 
891  pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) *
892  (10.**(offset/-2.5))*scaling)
893 
894  boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
895 
896  return boundedField
897 
898  def _outputAtmospheres(self, butler, atmCat, tract=None):
899  """
900  Output the atmospheres.
901 
902  Parameters
903  ----------
904  butler: `lsst.daf.persistence.Butler`
905  atmCat: `lsst.afw.table.BaseCatalog`
906  FGCM atmosphere parameter catalog from fgcmFitCycleTask
907  tract: `int`, optional
908  Tract number to output. Default is None (global calibration)
909  """
910 
911  self.log.info("Outputting atmosphere transmissions")
912 
913  # First, we need to grab the look-up table and key info
914  lutCat = butler.get('fgcmLookUpTable')
915 
916  atmosphereTableName = lutCat[0]['tablename']
917  elevation = lutCat[0]['elevation']
918  atmLambda = lutCat[0]['atmLambda']
919  lutCat = None
920 
921  # Make the atmosphere table if possible
922  try:
923  atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName)
924  atmTable.loadTable()
925  except IOError:
926  atmTable = None
927 
928  if atmTable is None:
929  # Try to use MODTRAN instead
930  try:
931  modGen = fgcm.ModtranGenerator(elevation)
932  lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10.
933  lambdaStep = (atmLambda[1] - atmLambda[0])/10.
934  except (ValueError, IOError) as e:
935  raise RuntimeError("FGCM look-up-table generated with modtran, "
936  "but modtran not configured to run.") from e
937 
938  zenith = np.degrees(np.arccos(1./atmCat['secZenith']))
939 
940  for i, visit in enumerate(atmCat['visit']):
941  if atmTable is not None:
942  # Interpolate the atmosphere table
943  atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'],
944  pwv=atmCat[i]['pwv'],
945  o3=atmCat[i]['o3'],
946  tau=atmCat[i]['tau'],
947  alpha=atmCat[i]['alpha'],
948  zenith=zenith[i],
949  ctranslamstd=[atmCat[i]['cTrans'],
950  atmCat[i]['lamStd']])
951  else:
952  # Run modtran
953  modAtm = modGen(pmb=atmCat[i]['pmb'],
954  pwv=atmCat[i]['pwv'],
955  o3=atmCat[i]['o3'],
956  tau=atmCat[i]['tau'],
957  alpha=atmCat[i]['alpha'],
958  zenith=zenith[i],
959  lambdaRange=lambdaRange,
960  lambdaStep=lambdaStep,
961  ctranslamstd=[atmCat[i]['cTrans'],
962  atmCat[i]['lamStd']])
963  atmVals = modAtm['COMBINED']
964 
965  # Now need to create something to persist...
966  curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals,
967  wavelengths=atmLambda,
968  throughputAtMin=atmVals[0],
969  throughputAtMax=atmVals[-1])
970 
971  if tract is None:
972  butler.put(curve, "transmission_atmosphere_fgcm",
973  dataId={self.visitDataRefName: visit})
974  else:
975  butler.put(curve, "transmission_atmosphere_fgcm_tract",
976  dataId={self.visitDataRefName: visit,
977  'tract': tract})
978 
979  self.log.info("Done outputting atmosphere transmissions")
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig.datasetConfig
datasetConfig
Definition: fgcmOutputProducts.py:134
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.filterMap
filterMap
Definition: fgcmOutputProducts.py:304
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._computeOffsetOneBand
def _computeOffsetOneBand(self, sourceMapper, badStarKey, b, band, stdCat, selected, refFluxFields)
Definition: fgcmOutputProducts.py:570
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.bands
bands
Definition: fgcmOutputProducts.py:350
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._outputZeropoints
def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None)
Definition: fgcmOutputProducts.py:740
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.__init__
def __init__(self, butler=None, **kwargs)
Definition: fgcmOutputProducts.py:246
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.generateTractOutputProducts
def generateTractOutputProducts(self, butler, tract, visitCat, zptCat, atmCat, stdCat, fgcmBuildStarsConfig, fgcmFitCycleConfig)
Definition: fgcmOutputProducts.py:381
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._getChebyshevBoundedField
def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0)
Definition: fgcmOutputProducts.py:864
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig.photoCal
photoCal
Definition: fgcmOutputProducts.py:106
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner
Definition: fgcmOutputProducts.py:167
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig
Definition: fgcmOutputProducts.py:64
lsst.fgcmcal.utilities.computeApproxPixelAreaFields
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:474
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.visitDataRefName
visitDataRefName
Definition: fgcmOutputProducts.py:302
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._outputStandardStars
def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig)
Definition: fgcmOutputProducts.py:630
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsConfig.setDefaults
def setDefaults(self)
Definition: fgcmOutputProducts.py:139
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
Definition: fgcmOutputProducts.py:237
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._formatCatalog
def _formatCatalog(self, fgcmStarCat, offsets)
Definition: fgcmOutputProducts.py:689
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.configBands
configBands
Definition: fgcmOutputProducts.py:319
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner.__call__
def __call__(self, butler)
Definition: fgcmOutputProducts.py:182
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.indexer
indexer
Definition: fgcmOutputProducts.py:262
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner.run
def run(self, parsedCmd)
Definition: fgcmOutputProducts.py:218
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.runDataRef
def runDataRef(self, butler)
Definition: fgcmOutputProducts.py:270
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._outputAtmospheres
def _outputAtmospheres(self, butler, atmCat, tract=None)
Definition: fgcmOutputProducts.py:898
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsRunner.getTargetList
def getTargetList(parsedCmd)
Definition: fgcmOutputProducts.py:176
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask._computeReferenceOffsets
def _computeReferenceOffsets(self, butler, stdCat)
Definition: fgcmOutputProducts.py:451
lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask.ccdDataRefName
ccdDataRefName
Definition: fgcmOutputProducts.py:303