lsst.fgcmcal  20.0.0-5-gfcebe35+1741cdab20
fgcmBuildStarsTable.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Build star observations for input to FGCM using sourceTable_visit.
24 
25 This task finds all the visits and sourceTable_visits in a repository (or a
26 subset based on command line parameters) and extracts all the potential
27 calibration stars for input into fgcm. This task additionally uses fgcm to
28 match star observations into unique stars, and performs as much cleaning of the
29 input catalog as possible.
30 """
31 
32 import time
33 
34 import numpy as np
35 import collections
36 
37 import lsst.pex.config as pexConfig
38 import lsst.pipe.base as pipeBase
39 import lsst.afw.table as afwTable
40 
41 from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
42 from .utilities import computeApproxPixelAreaFields
43 
44 __all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
45 
46 
48  """Config for FgcmBuildStarsTableTask"""
49 
50  referenceCCD = pexConfig.Field(
51  doc="Reference CCD for checking PSF and background",
52  dtype=int,
53  default=40,
54  )
55 
56  def setDefaults(self):
57  super().setDefaults()
58 
59  # The names here correspond to the post-transformed
60  # sourceTable_visit catalogs, which differ from the raw src
61  # catalogs. Therefore, all field and flag names cannot
62  # be derived from the base config class.
63  self.instFluxField = 'ApFlux_12_0_instFlux'
64  self.localBackgroundFluxField = 'LocalBackground_instFlux'
65  self.apertureInnerInstFluxField = 'ApFlux_12_0_instFlux'
66  self.apertureOuterInstFluxField = 'ApFlux_17_0_instFlux'
67  self.psfCandidateName = 'Calib_psf_candidate'
68 
69  sourceSelector = self.sourceSelector["science"]
70 
71  fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
72 
73  sourceSelector.flags.bad = ['PixelFlags_edge',
74  'PixelFlags_interpolatedCenter',
75  'PixelFlags_saturatedCenter',
76  'PixelFlags_crCenter',
77  'PixelFlags_bad',
78  'PixelFlags_interpolated',
79  'PixelFlags_saturated',
80  'Centroid_flag',
81  fluxFlagName]
82 
84  localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85  sourceSelector.flags.bad.append(localBackgroundFlagName)
86 
87  sourceSelector.signalToNoise.fluxField = self.instFluxField
88  sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
89 
90  sourceSelector.isolated.parentName = 'parentSourceId'
91  sourceSelector.isolated.nChildName = 'Deblend_nChild'
92 
93  sourceSelector.unresolved.name = 'extendedness'
94 
95 
97  """
98  Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
99  """
100  ConfigClass = FgcmBuildStarsTableConfig
101  RunnerClass = FgcmBuildStarsRunner
102  _DefaultName = "fgcmBuildStarsTable"
103 
104  @classmethod
105  def _makeArgumentParser(cls):
106  """Create an argument parser"""
107  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
108  parser.add_id_argument("--id", "sourceTable_visit", help="Data ID, e.g. --id visit=6789")
109 
110  return parser
111 
112  def findAndGroupDataRefs(self, butler, dataRefs):
113  self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
114 
115  camera = butler.get('camera')
116 
117  ccdIds = []
118  for detector in camera:
119  ccdIds.append(detector.getId())
120  # Insert our preferred referenceCCD first:
121  # It is fine that this is listed twice, because we only need
122  # the first calexp that is found.
123  ccdIds.insert(0, self.config.referenceCCD)
124 
125  # The visitTable building code expects a dictionary of groupedDataRefs
126  # keyed by visit, the first element as the "primary" calexp dataRef.
127  # We then append the sourceTable_visit dataRef at the end for the
128  # code which does the data reading (fgcmMakeAllStarObservations).
129 
130  groupedDataRefs = collections.defaultdict(list)
131  for dataRef in dataRefs:
132  visit = dataRef.dataId[self.config.visitDataRefName]
133 
134  # Find an existing calexp (we need for psf and metadata)
135  # and make the relevant dataRef
136  for ccdId in ccdIds:
137  try:
138  calexpRef = butler.dataRef('calexp', dataId={self.config.visitDataRefName: visit,
139  self.config.ccdDataRefName: ccdId})
140  except RuntimeError:
141  # Not found
142  continue
143  # It was found. Add and quit out, since we only
144  # need one calexp per visit.
145  groupedDataRefs[visit].append(calexpRef)
146  break
147 
148  # And append this dataRef
149  groupedDataRefs[visit].append(dataRef)
150 
151  return groupedDataRefs
152 
153  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
154  calibFluxApertureRadius=None,
155  visitCatDataRef=None,
156  starObsDataRef=None,
157  inStarObsCat=None):
158  startTime = time.time()
159 
160  # If both dataRefs are None, then we assume the caller does not
161  # want to store checkpoint files. If both are set, we will
162  # do checkpoint files. And if only one is set, this is potentially
163  # unintentional and we will warn.
164  if (visitCatDataRef is not None and starObsDataRef is None or
165  visitCatDataRef is None and starObsDataRef is not None):
166  self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
167  "no checkpoint files will be persisted.")
168 
169  if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
170  raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
171 
172  # To get the correct output schema, we use similar code as fgcmBuildStarsTask
173  # We are not actually using this mapper, except to grab the outputSchema
174  dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
175  sourceSchema = dataRef.get('src_schema', immediate=True).schema
176  sourceMapper = self._makeSourceMapper(sourceSchema)
177  outputSchema = sourceMapper.getOutputSchema()
178 
179  # Construct mapping from ccd number to index
180  camera = dataRef.get('camera')
181  ccdMapping = {}
182  for ccdIndex, detector in enumerate(camera):
183  ccdMapping[detector.getId()] = ccdIndex
184 
185  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
186 
187  if inStarObsCat is not None:
188  fullCatalog = inStarObsCat
189  comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
190  comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
191  if not comp1 or not comp2:
192  raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
193  else:
194  fullCatalog = afwTable.BaseCatalog(outputSchema)
195 
196  visitKey = outputSchema['visit'].asKey()
197  ccdKey = outputSchema['ccd'].asKey()
198  instMagKey = outputSchema['instMag'].asKey()
199  instMagErrKey = outputSchema['instMagErr'].asKey()
200 
201  # Prepare local background if desired
202  if self.config.doSubtractLocalBackground:
203  localBackgroundArea = np.pi*calibFluxApertureRadius**2.
204 
205  # Determine which columns we need from the sourceTable_visit catalogs
206  columns = self._get_sourceTable_visit_columns()
207 
208  k = 2.5/np.log(10.)
209 
210  for counter, visit in enumerate(visitCat):
211  # Check if these sources have already been read and stored in the checkpoint file
212  if visit['sources_read']:
213  continue
214 
215  expTime = visit['exptime']
216 
217  dataRef = groupedDataRefs[visit['visit']][-1]
218  srcTable = dataRef.get()
219 
220  df = srcTable.toDataFrame(columns)
221 
222  goodSrc = self.sourceSelector.selectSources(df)
223  use, = np.where(goodSrc.selected)
224 
225  tempCat = afwTable.BaseCatalog(fullCatalog.schema)
226  tempCat.resize(use.size)
227 
228  tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
229  tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
230  tempCat['x'][:] = df['x'].values[use]
231  tempCat['y'][:] = df['y'].values[use]
232  tempCat[visitKey][:] = df[self.config.visitDataRefName].values[use]
233  tempCat[ccdKey][:] = df[self.config.ccdDataRefName].values[use]
234  tempCat['psf_candidate'] = df['Calib_psf_candidate'].values[use]
235 
236  if self.config.doSubtractLocalBackground:
237  # At the moment we only adjust the flux and not the flux
238  # error by the background because the error on
239  # base_LocalBackground_instFlux is the rms error in the
240  # background annulus, not the error on the mean in the
241  # background estimate (which is much smaller, by sqrt(n)
242  # pixels used to estimate the background, which we do not
243  # have access to in this task). In the default settings,
244  # the annulus is sufficiently large such that these
245  # additional errors are are negligibly small (much less
246  # than a mmag in quadrature).
247 
248  localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
249 
250  # This is the difference between the mag with local background correction
251  # and the mag without local background correction.
252  tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] -
253  localBackground[use]) -
254  -2.5*np.log10(df[self.config.instFluxField].values[use]))
255  else:
256  tempCat['deltaMagBkg'][:] = 0.0
257 
258  # Need to loop over ccds here
259  for detector in camera:
260  ccdId = detector.getId()
261  # used index for all observations with a given ccd
262  use2 = (tempCat[ccdKey] == ccdId)
263  tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
264  tempCat['y'][use2])
265  scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] *
266  visit['scaling'][ccdMapping[ccdId]])
267  tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
268 
269  # Compute instMagErr from instFluxErr/instFlux, any scaling
270  # will cancel out.
271  tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] /
272  df[self.config.instFluxField].values[use])
273 
274  # Apply the jacobian if configured
275  if self.config.doApplyWcsJacobian:
276  tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
277 
278  fullCatalog.extend(tempCat)
279 
280  # Now do the aperture information
281  instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
282  instMagErrIn = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] /
283  df[self.config.apertureInnerInstFluxField].values[use])
284  instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
285  instMagErrOut = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] /
286  df[self.config.apertureOuterInstFluxField].values[use])
287  ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
288  np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
289 
290  visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
291  visit['sources_read'] = True
292 
293  self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
294  use.size, visit['visit'], visit['deltaAper'])
295 
296  if ((counter % self.config.nVisitsPerCheckpoint) == 0 and
297  starObsDataRef is not None and visitCatDataRef is not None):
298  # We need to persist both the stars and the visit catalog which gets
299  # additional metadata from each visit.
300  starObsDataRef.put(fullCatalog)
301  visitCatDataRef.put(visitCat)
302 
303  self.log.info("Found all good star observations in %.2f s" %
304  (time.time() - startTime))
305 
306  return fullCatalog
307 
308  def _get_sourceTable_visit_columns(self):
309  """
310  Get the sourceTable_visit columns from the config.
311 
312  Returns
313  -------
314  columns : `list`
315  List of columns to read from sourceTable_visit
316  """
317  columns = [self.config.visitDataRefName, self.config.ccdDataRefName,
318  'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
319  self.config.instFluxField, self.config.instFluxField + 'Err',
320  self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
321  self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
322  if self.sourceSelector.config.doFlags:
323  columns.extend(self.sourceSelector.config.flags.bad)
324  if self.sourceSelector.config.doUnresolved:
325  columns.append(self.sourceSelector.config.unresolved.name)
326  if self.sourceSelector.config.doIsolated:
327  columns.append(self.sourceSelector.config.isolated.parentName)
328  columns.append(self.sourceSelector.config.isolated.nChildName)
329  if self.config.doSubtractLocalBackground:
330  columns.append(self.config.localBackgroundFluxField)
331 
332  return columns
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask
Definition: fgcmBuildStarsBase.py:260
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableConfig.setDefaults
def setDefaults(self)
Definition: fgcmBuildStarsTable.py:56
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.apertureInnerInstFluxField
apertureInnerInstFluxField
Definition: fgcmBuildStarsBase.py:146
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask._get_sourceTable_visit_columns
def _get_sourceTable_visit_columns(self)
Definition: fgcmBuildStarsTable.py:308
lsst.fgcmcal.utilities.computeApproxPixelAreaFields
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:474
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.findAndGroupDataRefs
def findAndGroupDataRefs(self, butler, dataRefs)
Definition: fgcmBuildStarsTable.py:112
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask
Definition: fgcmBuildStarsTable.py:96
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.apertureOuterInstFluxField
apertureOuterInstFluxField
Definition: fgcmBuildStarsBase.py:152
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.fgcmMakeAllStarObservations
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
Definition: fgcmBuildStarsTable.py:153
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.doSubtractLocalBackground
doSubtractLocalBackground
Definition: fgcmBuildStarsBase.py:131
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.psfCandidateName
psfCandidateName
Definition: fgcmBuildStarsBase.py:126
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask._DefaultName
string _DefaultName
Definition: fgcmBuildStarsTable.py:102
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableConfig
Definition: fgcmBuildStarsTable.py:47
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.sourceSelector
sourceSelector
Definition: fgcmBuildStarsBase.py:142
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase
Definition: fgcmBuildStarsBase.py:49
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.instFluxField
instFluxField
Definition: fgcmBuildStarsBase.py:52
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeSourceMapper
def _makeSourceMapper(self, sourceSchema)
Definition: fgcmBuildStarsBase.py:555
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.localBackgroundFluxField
localBackgroundFluxField
Definition: fgcmBuildStarsBase.py:137