lsst.fgcmcal  20.0.0-4-ge48a6ca+4
fgcmBuildStars.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Build star observations for input to FGCM.
24 
25 This task finds all the visits and calexps in a repository (or a subset
26 based on command line parameters) and extract all the potential calibration
27 stars for input into fgcm. This task additionally uses fgcm to match
28 star observations into unique stars, and performs as much cleaning of
29 the input catalog as possible.
30 """
31 
32 import time
33 
34 import numpy as np
35 
36 import lsst.pex.config as pexConfig
37 import lsst.pipe.base as pipeBase
38 import lsst.afw.table as afwTable
39 
40 from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41 from .utilities import computeApproxPixelAreaFields
42 
43 __all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask']
44 
45 
47  """Config for FgcmBuildStarsTask"""
48 
49  referenceCCD = pexConfig.Field(
50  doc="Reference CCD for scanning visits",
51  dtype=int,
52  default=13,
53  )
54  checkAllCcds = pexConfig.Field(
55  doc=("Check repo for all CCDs for each visit specified. To be used when the "
56  "full set of ids (visit/ccd) are not specified on the command line. For "
57  "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58  "faster than the alternatives."),
59  dtype=bool,
60  default=True,
61  )
62 
63  def setDefaults(self):
64  super().setDefaults()
65 
66  sourceSelector = self.sourceSelector["science"]
67 
68  # The names here correspond to raw src catalogs, which differ
69  # from the post-transformed sourceTable_visit catalogs.
70  # Therefore, field and flag names cannot be easily
71  # derived from the base config class.
72  fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
73  sourceSelector.flags.bad = ['base_PixelFlags_flag_edge',
74  'base_PixelFlags_flag_interpolatedCenter',
75  'base_PixelFlags_flag_saturatedCenter',
76  'base_PixelFlags_flag_crCenter',
77  'base_PixelFlags_flag_bad',
78  'base_PixelFlags_flag_interpolated',
79  'base_PixelFlags_flag_saturated',
80  'slot_Centroid_flag',
81  fluxFlagName]
82 
84  localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85  sourceSelector.flags.bad.append(localBackgroundFlagName)
86 
87  sourceSelector.signalToNoise.fluxField = self.instFluxField
88  sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
89 
90 
92  """
93  Build stars for the FGCM global calibration, using src catalogs.
94  """
95  ConfigClass = FgcmBuildStarsConfig
96  RunnerClass = FgcmBuildStarsRunner
97  _DefaultName = "fgcmBuildStars"
98 
99  @classmethod
100  def _makeArgumentParser(cls):
101  """Create an argument parser"""
102  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
103  parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789")
104 
105  return parser
106 
107  def findAndGroupDataRefs(self, butler, dataRefs):
108  self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
109 
110  camera = butler.get('camera')
111 
112  ccdIds = []
113  for detector in camera:
114  ccdIds.append(detector.getId())
115 
116  # TODO: related to DM-13730, this dance of looking for source visits
117  # will be unnecessary with Gen3 Butler. This should be part of
118  # DM-13730.
119 
120  nVisits = 0
121 
122  groupedDataRefs = {}
123  for dataRef in dataRefs:
124  visit = dataRef.dataId[self.config.visitDataRefName]
125  # If we don't have the dataset, just continue
126  if not dataRef.datasetExists(datasetType='src'):
127  continue
128  # If we need to check all ccds, do it here
129  if self.config.checkAllCcds:
130  if visit in groupedDataRefs:
131  # We already have found this visit
132  continue
133  dataId = dataRef.dataId.copy()
134  # For each ccd we must check that a valid source catalog exists.
135  for ccdId in ccdIds:
136  dataId[self.config.ccdDataRefName] = ccdId
137  if butler.datasetExists('src', dataId=dataId):
138  goodDataRef = butler.dataRef('src', dataId=dataId)
139  if visit in groupedDataRefs:
140  if (goodDataRef.dataId[self.config.ccdDataRefName] not in
141  [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
142  groupedDataRefs[visit].append(goodDataRef)
143  else:
144  # This is a new visit
145  nVisits += 1
146  groupedDataRefs[visit] = [goodDataRef]
147  else:
148  # We have already confirmed that the dataset exists, so no need
149  # to check here.
150  if visit in groupedDataRefs:
151  if (dataRef.dataId[self.config.ccdDataRefName] not in
152  [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
153  groupedDataRefs[visit].append(dataRef)
154  else:
155  # This is a new visit
156  nVisits += 1
157  groupedDataRefs[visit] = [dataRef]
158 
159  if (nVisits % 100) == 0 and nVisits > 0:
160  self.log.info("Found %d unique %ss..." % (nVisits,
161  self.config.visitDataRefName))
162 
163  self.log.info("Found %d unique %ss total." % (nVisits,
164  self.config.visitDataRefName))
165 
166  # Put them in ccd order, with the reference ccd first (if available)
167  def ccdSorter(dataRef):
168  ccdId = dataRef.dataId[self.config.ccdDataRefName]
169  if ccdId == self.config.referenceCCD:
170  return -100
171  else:
172  return ccdId
173 
174  # If we did not check all ccds, put them in ccd order
175  if not self.config.checkAllCcds:
176  for visit in groupedDataRefs:
177  groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
178 
179  return groupedDataRefs
180 
181  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
182  calibFluxApertureRadius=None,
183  visitCatDataRef=None,
184  starObsDataRef=None,
185  inStarObsCat=None):
186  startTime = time.time()
187 
188  # If both dataRefs are None, then we assume the caller does not
189  # want to store checkpoint files. If both are set, we will
190  # do checkpoint files. And if only one is set, this is potentially
191  # unintentional and we will warn.
192  if (visitCatDataRef is not None and starObsDataRef is None or
193  visitCatDataRef is None and starObsDataRef is not None):
194  self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
195  "no checkpoint files will be persisted.")
196 
197  if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
198  raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
199 
200  # create our source schema. Use the first valid dataRef
201  dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
202  sourceSchema = dataRef.get('src_schema', immediate=True).schema
203 
204  # Construct a mapping from ccd number to index
205  camera = dataRef.get('camera')
206  ccdMapping = {}
207  for ccdIndex, detector in enumerate(camera):
208  ccdMapping[detector.getId()] = ccdIndex
209 
210  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
211 
212  sourceMapper = self._makeSourceMapper(sourceSchema)
213 
214  # We also have a temporary catalog that will accumulate aperture measurements
215  aperMapper = self._makeAperMapper(sourceSchema)
216 
217  outputSchema = sourceMapper.getOutputSchema()
218 
219  if inStarObsCat is not None:
220  fullCatalog = inStarObsCat
221  comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
222  comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
223  if not comp1 or not comp2:
224  raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
225  else:
226  fullCatalog = afwTable.BaseCatalog(outputSchema)
227 
228  # FGCM will provide relative calibration for the flux in config.instFluxField
229 
230  instFluxKey = sourceSchema[self.config.instFluxField].asKey()
231  instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey()
232  visitKey = outputSchema['visit'].asKey()
233  ccdKey = outputSchema['ccd'].asKey()
234  instMagKey = outputSchema['instMag'].asKey()
235  instMagErrKey = outputSchema['instMagErr'].asKey()
236 
237  # Prepare local background if desired
238  if self.config.doSubtractLocalBackground:
239  localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
240  localBackgroundArea = np.pi*calibFluxApertureRadius**2.
241  else:
242  localBackground = 0.0
243 
244  aperOutputSchema = aperMapper.getOutputSchema()
245 
246  instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
247  instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey()
248  instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
249  instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey()
250  instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
251  instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
252  instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
253  instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
254 
255  k = 2.5/np.log(10.)
256 
257  # loop over visits
258  for ctr, visit in enumerate(visitCat):
259  if visit['sources_read']:
260  continue
261 
262  expTime = visit['exptime']
263 
264  nStarInVisit = 0
265 
266  # Reset the aperture catalog (per visit)
267  aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
268 
269  for dataRef in groupedDataRefs[visit['visit']]:
270 
271  ccdId = dataRef.dataId[self.config.ccdDataRefName]
272 
273  sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
274 
275  # If we are subtracting the local background, then correct here
276  # before we do the s/n selection. This ensures we do not have
277  # bad stars after local background subtraction.
278 
279  if self.config.doSubtractLocalBackground:
280  # At the moment we only adjust the flux and not the flux
281  # error by the background because the error on
282  # base_LocalBackground_instFlux is the rms error in the
283  # background annulus, not the error on the mean in the
284  # background estimate (which is much smaller, by sqrt(n)
285  # pixels used to estimate the background, which we do not
286  # have access to in this task). In the default settings,
287  # the annulus is sufficiently large such that these
288  # additional errors are are negligibly small (much less
289  # than a mmag in quadrature).
290 
291  localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
292  sources[instFluxKey] -= localBackground
293 
294  goodSrc = self.sourceSelector.selectSources(sources)
295 
296  tempCat = afwTable.BaseCatalog(fullCatalog.schema)
297  tempCat.reserve(goodSrc.selected.sum())
298  tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
299  tempCat[visitKey][:] = visit['visit']
300  tempCat[ccdKey][:] = ccdId
301 
302  # Compute "instrumental magnitude" by scaling flux with exposure time.
303  scaledInstFlux = (sources[instFluxKey][goodSrc.selected] *
304  visit['scaling'][ccdMapping[ccdId]])
305  tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
306 
307  # Compute instMagErr from instFluxErr/instFlux, any scaling
308  # will cancel out.
309 
310  tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] /
311  sources[instFluxKey][goodSrc.selected])
312 
313  # Compute the jacobian from an approximate PixelAreaBoundedField
314  tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'],
315  tempCat['y'])
316 
317  # Apply the jacobian if configured
318  if self.config.doApplyWcsJacobian:
319  tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
320 
321  fullCatalog.extend(tempCat)
322 
323  # And the aperture information
324  # This does not need the jacobian because it is all locally relative
325  tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
326  tempAperCat.reserve(goodSrc.selected.sum())
327  tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
328 
329  with np.warnings.catch_warnings():
330  # Ignore warnings, we will filter infinities and
331  # nans below.
332  np.warnings.simplefilter("ignore")
333 
334  tempAperCat[instMagInKey][:] = -2.5*np.log10(
335  sources[instFluxAperInKey][goodSrc.selected])
336  tempAperCat[instMagErrInKey][:] = k*(
337  sources[instFluxErrAperInKey][goodSrc.selected] /
338  sources[instFluxAperInKey][goodSrc.selected])
339  tempAperCat[instMagOutKey][:] = -2.5*np.log10(
340  sources[instFluxAperOutKey][goodSrc.selected])
341  tempAperCat[instMagErrOutKey][:] = k*(
342  sources[instFluxErrAperOutKey][goodSrc.selected] /
343  sources[instFluxAperOutKey][goodSrc.selected])
344 
345  aperVisitCatalog.extend(tempAperCat)
346 
347  nStarInVisit += len(tempCat)
348 
349  # Compute the median delta-aper
350  if not aperVisitCatalog.isContiguous():
351  aperVisitCatalog = aperVisitCatalog.copy(deep=True)
352 
353  instMagIn = aperVisitCatalog[instMagInKey]
354  instMagErrIn = aperVisitCatalog[instMagErrInKey]
355  instMagOut = aperVisitCatalog[instMagOutKey]
356  instMagErrOut = aperVisitCatalog[instMagErrOutKey]
357 
358  ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
359  np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
360 
361  visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
362  visit['sources_read'] = True
363 
364  self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" %
365  (nStarInVisit, visit['visit'], visit['deltaAper']))
366 
367  if ((ctr % self.config.nVisitsPerCheckpoint) == 0 and
368  starObsDataRef is not None and visitCatDataRef is not None):
369  # We need to persist both the stars and the visit catalog which gets
370  # additional metadata from each visit.
371  starObsDataRef.put(fullCatalog)
372  visitCatDataRef.put(visitCat)
373 
374  self.log.info("Found all good star observations in %.2f s" %
375  (time.time() - startTime))
376 
377  return fullCatalog
378 
379  def _makeAperMapper(self, sourceSchema):
380  """
381  Make a schema mapper for fgcm aperture measurements
382 
383  Parameters
384  ----------
385  sourceSchema: `afwTable.Schema`
386  Default source schema from the butler
387 
388  Returns
389  -------
390  aperMapper: `afwTable.schemaMapper`
391  Mapper to the FGCM aperture schema
392  """
393 
394  aperMapper = afwTable.SchemaMapper(sourceSchema)
395  aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
396  aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
397  aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64,
398  doc="Magnitude at inner aperture")
399  aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64,
400  doc="Magnitude error at inner aperture")
401  aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64,
402  doc="Magnitude at outer aperture")
403  aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64,
404  doc="Magnitude error at outer aperture")
405 
406  return aperMapper
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsConfig
Definition: fgcmBuildStars.py:46
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask
Definition: fgcmBuildStarsBase.py:260
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsConfig.setDefaults
def setDefaults(self)
Definition: fgcmBuildStars.py:63
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask
Definition: fgcmBuildStars.py:91
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask.fgcmMakeAllStarObservations
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
Definition: fgcmBuildStars.py:181
lsst.fgcmcal.utilities.computeApproxPixelAreaFields
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:472
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask._makeAperMapper
def _makeAperMapper(self, sourceSchema)
Definition: fgcmBuildStars.py:379
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask._DefaultName
string _DefaultName
Definition: fgcmBuildStars.py:97
lsst.fgcmcal.fgcmBuildStars.FgcmBuildStarsTask.findAndGroupDataRefs
def findAndGroupDataRefs(self, butler, dataRefs)
Definition: fgcmBuildStars.py:107
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.doSubtractLocalBackground
doSubtractLocalBackground
Definition: fgcmBuildStarsBase.py:131
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.sourceSelector
sourceSelector
Definition: fgcmBuildStarsBase.py:142
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase
Definition: fgcmBuildStarsBase.py:49
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.instFluxField
instFluxField
Definition: fgcmBuildStarsBase.py:52
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeSourceMapper
def _makeSourceMapper(self, sourceSchema)
Definition: fgcmBuildStarsBase.py:555
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.localBackgroundFluxField
localBackgroundFluxField
Definition: fgcmBuildStarsBase.py:137