lsst.fgcmcal  20.0.0-5-gfcebe35+bb8216a93e
fgcmBuildStars.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Build star observations for input to FGCM.
24 
25 This task finds all the visits and calexps in a repository (or a subset
26 based on command line parameters) and extract all the potential calibration
27 stars for input into fgcm. This task additionally uses fgcm to match
28 star observations into unique stars, and performs as much cleaning of
29 the input catalog as possible.
30 """
31 
32 import time
33 
34 import numpy as np
35 
36 import lsst.pex.config as pexConfig
37 import lsst.pipe.base as pipeBase
38 import lsst.afw.table as afwTable
39 
40 from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41 from .utilities import computeApproxPixelAreaFields
42 
43 __all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask']
44 
45 
47  """Config for FgcmBuildStarsTask"""
48 
49  referenceCCD = pexConfig.Field(
50  doc="Reference CCD for scanning visits",
51  dtype=int,
52  default=13,
53  )
54  checkAllCcds = pexConfig.Field(
55  doc=("Check repo for all CCDs for each visit specified. To be used when the "
56  "full set of ids (visit/ccd) are not specified on the command line. For "
57  "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58  "faster than the alternatives."),
59  dtype=bool,
60  default=True,
61  )
62 
63  def setDefaults(self):
64  super().setDefaults()
65 
66  sourceSelector = self.sourceSelector["science"]
67 
68  # The names here correspond to raw src catalogs, which differ
69  # from the post-transformed sourceTable_visit catalogs.
70  # Therefore, field and flag names cannot be easily
71  # derived from the base config class.
72  fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
73  sourceSelector.flags.bad = ['base_PixelFlags_flag_edge',
74  'base_PixelFlags_flag_interpolatedCenter',
75  'base_PixelFlags_flag_saturatedCenter',
76  'base_PixelFlags_flag_crCenter',
77  'base_PixelFlags_flag_bad',
78  'base_PixelFlags_flag_interpolated',
79  'base_PixelFlags_flag_saturated',
80  'slot_Centroid_flag',
81  fluxFlagName]
82 
84  localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85  sourceSelector.flags.bad.append(localBackgroundFlagName)
86 
87  sourceSelector.signalToNoise.fluxField = self.instFluxField
88  sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
89 
90 
92  """
93  Build stars for the FGCM global calibration, using src catalogs.
94  """
95  ConfigClass = FgcmBuildStarsConfig
96  RunnerClass = FgcmBuildStarsRunner
97  _DefaultName = "fgcmBuildStars"
98 
99  @classmethod
100  def _makeArgumentParser(cls):
101  """Create an argument parser"""
102  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
103  parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789")
104 
105  return parser
106 
107  def findAndGroupDataRefs(self, butler, dataRefs):
108  self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
109 
110  camera = butler.get('camera')
111 
112  ccdIds = []
113  for detector in camera:
114  ccdIds.append(detector.getId())
115 
116  # TODO: related to DM-13730, this dance of looking for source visits
117  # will be unnecessary with Gen3 Butler. This should be part of
118  # DM-13730.
119 
120  nVisits = 0
121 
122  groupedDataRefs = {}
123  for dataRef in dataRefs:
124  visit = dataRef.dataId[self.config.visitDataRefName]
125  # If we don't have the dataset, just continue
126  if not dataRef.datasetExists(datasetType='src'):
127  continue
128  # If we need to check all ccds, do it here
129  if self.config.checkAllCcds:
130  if visit in groupedDataRefs:
131  # We already have found this visit
132  continue
133  dataId = dataRef.dataId.copy()
134  # For each ccd we must check that a valid source catalog exists.
135  for ccdId in ccdIds:
136  dataId[self.config.ccdDataRefName] = ccdId
137  if butler.datasetExists('src', dataId=dataId):
138  goodDataRef = butler.dataRef('src', dataId=dataId)
139  if visit in groupedDataRefs:
140  if (goodDataRef.dataId[self.config.ccdDataRefName] not in
141  [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
142  groupedDataRefs[visit].append(goodDataRef)
143  else:
144  # This is a new visit
145  nVisits += 1
146  groupedDataRefs[visit] = [goodDataRef]
147  else:
148  # We have already confirmed that the dataset exists, so no need
149  # to check here.
150  if visit in groupedDataRefs:
151  if (dataRef.dataId[self.config.ccdDataRefName] not in
152  [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
153  groupedDataRefs[visit].append(dataRef)
154  else:
155  # This is a new visit
156  nVisits += 1
157  groupedDataRefs[visit] = [dataRef]
158 
159  if (nVisits % 100) == 0 and nVisits > 0:
160  self.log.info("Found %d unique %ss..." % (nVisits,
161  self.config.visitDataRefName))
162 
163  self.log.info("Found %d unique %ss total." % (nVisits,
164  self.config.visitDataRefName))
165 
166  # Put them in ccd order, with the reference ccd first (if available)
167  def ccdSorter(dataRef):
168  ccdId = dataRef.dataId[self.config.ccdDataRefName]
169  if ccdId == self.config.referenceCCD:
170  return -100
171  else:
172  return ccdId
173 
174  # If we did not check all ccds, put them in ccd order
175  if not self.config.checkAllCcds:
176  for visit in groupedDataRefs:
177  groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
178 
179  return groupedDataRefs
180 
181  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
182  calibFluxApertureRadius=None,
183  visitCatDataRef=None,
184  starObsDataRef=None,
185  inStarObsCat=None):
186  startTime = time.time()
187 
188  # If both dataRefs are None, then we assume the caller does not
189  # want to store checkpoint files. If both are set, we will
190  # do checkpoint files. And if only one is set, this is potentially
191  # unintentional and we will warn.
192  if (visitCatDataRef is not None and starObsDataRef is None or
193  visitCatDataRef is None and starObsDataRef is not None):
194  self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
195  "no checkpoint files will be persisted.")
196 
197  if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
198  raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
199 
200  # create our source schema. Use the first valid dataRef
201  dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
202  sourceSchema = dataRef.get('src_schema', immediate=True).schema
203 
204  # Construct a mapping from ccd number to index
205  camera = dataRef.get('camera')
206  ccdMapping = {}
207  for ccdIndex, detector in enumerate(camera):
208  ccdMapping[detector.getId()] = ccdIndex
209 
210  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
211 
212  sourceMapper = self._makeSourceMapper(sourceSchema)
213 
214  # We also have a temporary catalog that will accumulate aperture measurements
215  aperMapper = self._makeAperMapper(sourceSchema)
216 
217  outputSchema = sourceMapper.getOutputSchema()
218 
219  if inStarObsCat is not None:
220  fullCatalog = inStarObsCat
221  comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
222  comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
223  if not comp1 or not comp2:
224  raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
225  else:
226  fullCatalog = afwTable.BaseCatalog(outputSchema)
227 
228  # FGCM will provide relative calibration for the flux in config.instFluxField
229 
230  instFluxKey = sourceSchema[self.config.instFluxField].asKey()
231  instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey()
232  visitKey = outputSchema['visit'].asKey()
233  ccdKey = outputSchema['ccd'].asKey()
234  instMagKey = outputSchema['instMag'].asKey()
235  instMagErrKey = outputSchema['instMagErr'].asKey()
236  deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey()
237 
238  # Prepare local background if desired
239  if self.config.doSubtractLocalBackground:
240  localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
241  localBackgroundArea = np.pi*calibFluxApertureRadius**2.
242 
243  aperOutputSchema = aperMapper.getOutputSchema()
244 
245  instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
246  instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey()
247  instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
248  instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey()
249  instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
250  instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
251  instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
252  instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
253 
254  k = 2.5/np.log(10.)
255 
256  # loop over visits
257  for ctr, visit in enumerate(visitCat):
258  if visit['sources_read']:
259  continue
260 
261  expTime = visit['exptime']
262 
263  nStarInVisit = 0
264 
265  # Reset the aperture catalog (per visit)
266  aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
267 
268  for dataRef in groupedDataRefs[visit['visit']]:
269 
270  ccdId = dataRef.dataId[self.config.ccdDataRefName]
271 
272  sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
273  goodSrc = self.sourceSelector.selectSources(sources)
274 
275  tempCat = afwTable.BaseCatalog(fullCatalog.schema)
276  tempCat.reserve(goodSrc.selected.sum())
277  tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
278  tempCat[visitKey][:] = visit['visit']
279  tempCat[ccdKey][:] = ccdId
280 
281  # Compute "instrumental magnitude" by scaling flux with exposure time.
282  scaledInstFlux = (sources[instFluxKey][goodSrc.selected] *
283  visit['scaling'][ccdMapping[ccdId]])
284  tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
285 
286  # Compute the change in magnitude from the background offset
287  if self.config.doSubtractLocalBackground:
288  # At the moment we only adjust the flux and not the flux
289  # error by the background because the error on
290  # base_LocalBackground_instFlux is the rms error in the
291  # background annulus, not the error on the mean in the
292  # background estimate (which is much smaller, by sqrt(n)
293  # pixels used to estimate the background, which we do not
294  # have access to in this task). In the default settings,
295  # the annulus is sufficiently large such that these
296  # additional errors are are negligibly small (much less
297  # than a mmag in quadrature).
298 
299  localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
300 
301  # This is the difference between the mag with background correction
302  # and the mag without background correction.
303  tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected] -
304  localBackground[goodSrc.selected]) -
305  -2.5*np.log10(sources[instFluxKey][goodSrc.selected]))
306  else:
307  tempCat[deltaMagBkgKey][:] = 0.0
308 
309  # Compute instMagErr from instFluxErr/instFlux, any scaling
310  # will cancel out.
311 
312  tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] /
313  sources[instFluxKey][goodSrc.selected])
314 
315  # Compute the jacobian from an approximate PixelAreaBoundedField
316  tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'],
317  tempCat['y'])
318 
319  # Apply the jacobian if configured
320  if self.config.doApplyWcsJacobian:
321  tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
322 
323  fullCatalog.extend(tempCat)
324 
325  # And the aperture information
326  # This does not need the jacobian because it is all locally relative
327  tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
328  tempAperCat.reserve(goodSrc.selected.sum())
329  tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
330 
331  with np.warnings.catch_warnings():
332  # Ignore warnings, we will filter infinities and
333  # nans below.
334  np.warnings.simplefilter("ignore")
335 
336  tempAperCat[instMagInKey][:] = -2.5*np.log10(
337  sources[instFluxAperInKey][goodSrc.selected])
338  tempAperCat[instMagErrInKey][:] = k*(
339  sources[instFluxErrAperInKey][goodSrc.selected] /
340  sources[instFluxAperInKey][goodSrc.selected])
341  tempAperCat[instMagOutKey][:] = -2.5*np.log10(
342  sources[instFluxAperOutKey][goodSrc.selected])
343  tempAperCat[instMagErrOutKey][:] = k*(
344  sources[instFluxErrAperOutKey][goodSrc.selected] /
345  sources[instFluxAperOutKey][goodSrc.selected])
346 
347  aperVisitCatalog.extend(tempAperCat)
348 
349  nStarInVisit += len(tempCat)
350 
351  # Compute the median delta-aper
352  if not aperVisitCatalog.isContiguous():
353  aperVisitCatalog = aperVisitCatalog.copy(deep=True)
354 
355  instMagIn = aperVisitCatalog[instMagInKey]
356  instMagErrIn = aperVisitCatalog[instMagErrInKey]
357  instMagOut = aperVisitCatalog[instMagOutKey]
358  instMagErrOut = aperVisitCatalog[instMagErrOutKey]
359 
360  ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
361  np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
362 
363  visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
364  visit['sources_read'] = True
365 
366  self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" %
367  (nStarInVisit, visit['visit'], visit['deltaAper']))
368 
369  if ((ctr % self.config.nVisitsPerCheckpoint) == 0 and
370  starObsDataRef is not None and visitCatDataRef is not None):
371  # We need to persist both the stars and the visit catalog which gets
372  # additional metadata from each visit.
373  starObsDataRef.put(fullCatalog)
374  visitCatDataRef.put(visitCat)
375 
376  self.log.info("Found all good star observations in %.2f s" %
377  (time.time() - startTime))
378 
379  return fullCatalog
380 
381  def _makeAperMapper(self, sourceSchema):
382  """
383  Make a schema mapper for fgcm aperture measurements
384 
385  Parameters
386  ----------
387  sourceSchema: `afwTable.Schema`
388  Default source schema from the butler
389 
390  Returns
391  -------
392  aperMapper: `afwTable.schemaMapper`
393  Mapper to the FGCM aperture schema
394  """
395 
396  aperMapper = afwTable.SchemaMapper(sourceSchema)
397  aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
398  aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
399  aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64,
400  doc="Magnitude at inner aperture")
401  aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64,
402  doc="Magnitude error at inner aperture")
403  aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64,
404  doc="Magnitude at outer aperture")
405  aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64,
406  doc="Magnitude error at outer aperture")
407 
408  return aperMapper
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsConfig
Definition: fgcmBuildStars.py:46
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask
Definition: fgcmBuildStarsBase.py:260
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsConfig.setDefaults
def setDefaults(self)
Definition: fgcmBuildStars.py:63
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask
Definition: fgcmBuildStars.py:91
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask.fgcmMakeAllStarObservations
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
Definition: fgcmBuildStars.py:181
lsst.fgcmcal.utilities.computeApproxPixelAreaFields
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:474
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask._makeAperMapper
def _makeAperMapper(self, sourceSchema)
Definition: fgcmBuildStars.py:381
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask._DefaultName
string _DefaultName
Definition: fgcmBuildStars.py:97
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask.findAndGroupDataRefs
def findAndGroupDataRefs(self, butler, dataRefs)
Definition: fgcmBuildStars.py:107
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.doSubtractLocalBackground
doSubtractLocalBackground
Definition: fgcmBuildStarsBase.py:131
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.sourceSelector
sourceSelector
Definition: fgcmBuildStarsBase.py:142
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase
Definition: fgcmBuildStarsBase.py:49
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.instFluxField
instFluxField
Definition: fgcmBuildStarsBase.py:52
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeSourceMapper
def _makeSourceMapper(self, sourceSchema)
Definition: fgcmBuildStarsBase.py:555
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.localBackgroundFluxField
localBackgroundFluxField
Definition: fgcmBuildStarsBase.py:137