lsst.fgcmcal  20.0.0-4-ge48a6ca+4
fgcmBuildStarsTable.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Build star observations for input to FGCM using sourceTable_visit.
24 
25 This task finds all the visits and sourceTable_visits in a repository (or a
26 subset based on command line parameters) and extracts all the potential
27 calibration stars for input into fgcm. This task additionally uses fgcm to
28 match star observations into unique stars, and performs as much cleaning of the
29 input catalog as possible.
30 """
31 
32 import time
33 
34 import numpy as np
35 import collections
36 
37 import lsst.pex.config as pexConfig
38 import lsst.pipe.base as pipeBase
39 import lsst.afw.table as afwTable
40 
41 from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
42 from .utilities import computeApproxPixelAreaFields
43 
44 __all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
45 
46 
48  """Config for FgcmBuildStarsTableTask"""
49 
50  referenceCCD = pexConfig.Field(
51  doc="Reference CCD for checking PSF and background",
52  dtype=int,
53  default=40,
54  )
55 
56  def setDefaults(self):
57  super().setDefaults()
58 
59  # The names here correspond to the post-transformed
60  # sourceTable_visit catalogs, which differ from the raw src
61  # catalogs. Therefore, all field and flag names cannot
62  # be derived from the base config class.
63  self.instFluxField = 'ApFlux_12_0_instFlux'
64  self.localBackgroundFluxField = 'LocalBackground_instFlux'
65  self.apertureInnerInstFluxField = 'ApFlux_12_0_instFlux'
66  self.apertureOuterInstFluxField = 'ApFlux_17_0_instFlux'
67  self.psfCandidateName = 'Calib_psf_candidate'
68 
69  sourceSelector = self.sourceSelector["science"]
70 
71  fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
72 
73  sourceSelector.flags.bad = ['PixelFlags_edge',
74  'PixelFlags_interpolatedCenter',
75  'PixelFlags_saturatedCenter',
76  'PixelFlags_crCenter',
77  'PixelFlags_bad',
78  'PixelFlags_interpolated',
79  'PixelFlags_saturated',
80  'Centroid_flag',
81  fluxFlagName]
82 
84  localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85  sourceSelector.flags.bad.append(localBackgroundFlagName)
86 
87  sourceSelector.signalToNoise.fluxField = self.instFluxField
88  sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
89 
90  sourceSelector.isolated.parentName = 'parentSourceId'
91  sourceSelector.isolated.nChildName = 'Deblend_nChild'
92 
93  sourceSelector.unresolved.name = 'extendedness'
94 
95 
97  """
98  Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
99  """
100  ConfigClass = FgcmBuildStarsTableConfig
101  RunnerClass = FgcmBuildStarsRunner
102  _DefaultName = "fgcmBuildStarsTable"
103 
104  @classmethod
105  def _makeArgumentParser(cls):
106  """Create an argument parser"""
107  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
108  parser.add_id_argument("--id", "sourceTable_visit", help="Data ID, e.g. --id visit=6789")
109 
110  return parser
111 
112  def findAndGroupDataRefs(self, butler, dataRefs):
113  self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
114 
115  camera = butler.get('camera')
116 
117  ccdIds = []
118  for detector in camera:
119  ccdIds.append(detector.getId())
120  # Insert our preferred referenceCCD first:
121  # It is fine that this is listed twice, because we only need
122  # the first calexp that is found.
123  ccdIds.insert(0, self.config.referenceCCD)
124 
125  # The visitTable building code expects a dictionary of groupedDataRefs
126  # keyed by visit, the first element as the "primary" calexp dataRef.
127  # We then append the sourceTable_visit dataRef at the end for the
128  # code which does the data reading (fgcmMakeAllStarObservations).
129 
130  groupedDataRefs = collections.defaultdict(list)
131  for dataRef in dataRefs:
132  visit = dataRef.dataId[self.config.visitDataRefName]
133 
134  # Find an existing calexp (we need for psf and metadata)
135  # and make the relevant dataRef
136  for ccdId in ccdIds:
137  try:
138  calexpRef = butler.dataRef('calexp', dataId={self.config.visitDataRefName: visit,
139  self.config.ccdDataRefName: ccdId})
140  except RuntimeError:
141  # Not found
142  continue
143  # It was found. Add and quit out, since we only
144  # need one calexp per visit.
145  groupedDataRefs[visit].append(calexpRef)
146  break
147 
148  # And append this dataRef
149  groupedDataRefs[visit].append(dataRef)
150 
151  return groupedDataRefs
152 
153  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
154  calibFluxApertureRadius=None,
155  visitCatDataRef=None,
156  starObsDataRef=None,
157  inStarObsCat=None):
158  startTime = time.time()
159 
160  # If both dataRefs are None, then we assume the caller does not
161  # want to store checkpoint files. If both are set, we will
162  # do checkpoint files. And if only one is set, this is potentially
163  # unintentional and we will warn.
164  if (visitCatDataRef is not None and starObsDataRef is None or
165  visitCatDataRef is None and starObsDataRef is not None):
166  self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
167  "no checkpoint files will be persisted.")
168 
169  if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
170  raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
171 
172  # To get the correct output schema, we use similar code as fgcmBuildStarsTask
173  # We are not actually using this mapper, except to grab the outputSchema
174  dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
175  sourceSchema = dataRef.get('src_schema', immediate=True).schema
176  sourceMapper = self._makeSourceMapper(sourceSchema)
177  outputSchema = sourceMapper.getOutputSchema()
178 
179  # Construct mapping from ccd number to index
180  camera = dataRef.get('camera')
181  ccdMapping = {}
182  for ccdIndex, detector in enumerate(camera):
183  ccdMapping[detector.getId()] = ccdIndex
184 
185  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
186 
187  if inStarObsCat is not None:
188  fullCatalog = inStarObsCat
189  comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
190  comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
191  if not comp1 or not comp2:
192  raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
193  else:
194  fullCatalog = afwTable.BaseCatalog(outputSchema)
195 
196  visitKey = outputSchema['visit'].asKey()
197  ccdKey = outputSchema['ccd'].asKey()
198  instMagKey = outputSchema['instMag'].asKey()
199  instMagErrKey = outputSchema['instMagErr'].asKey()
200 
201  # Prepare local background if desired
202  if self.config.doSubtractLocalBackground:
203  localBackgroundArea = np.pi*calibFluxApertureRadius**2.
204  else:
205  localBackground = 0.0
206 
207  # Determine which columns we need from the sourceTable_visit catalogs
208  columns = self._get_sourceTable_visit_columns()
209 
210  k = 2.5/np.log(10.)
211 
212  for counter, visit in enumerate(visitCat):
213  # Check if these sources have already been read and stored in the checkpoint file
214  if visit['sources_read']:
215  continue
216 
217  expTime = visit['exptime']
218 
219  dataRef = groupedDataRefs[visit['visit']][-1]
220  srcTable = dataRef.get()
221 
222  df = srcTable.toDataFrame(columns)
223 
224  if self.config.doSubtractLocalBackground:
225  localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
226  df[self.config.instFluxField] -= localBackground
227 
228  goodSrc = self.sourceSelector.selectSources(df)
229  use, = np.where(goodSrc.selected)
230 
231  tempCat = afwTable.BaseCatalog(fullCatalog.schema)
232  tempCat.resize(use.size)
233 
234  tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
235  tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
236  tempCat['x'][:] = df['x'].values[use]
237  tempCat['y'][:] = df['y'].values[use]
238  tempCat[visitKey][:] = df[self.config.visitDataRefName].values[use]
239  tempCat[ccdKey][:] = df[self.config.ccdDataRefName].values[use]
240  tempCat['psf_candidate'] = df['Calib_psf_candidate'].values[use]
241 
242  # Need to loop over ccds here
243  for detector in camera:
244  ccdId = detector.getId()
245  # used index for all observations with a given ccd
246  use2 = (tempCat[ccdKey] == ccdId)
247  tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
248  tempCat['y'][use2])
249  scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] *
250  visit['scaling'][ccdMapping[ccdId]])
251  tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
252 
253  # Compute instMagErr from instFluxErr/instFlux, any scaling
254  # will cancel out.
255  tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] /
256  df[self.config.instFluxField].values[use])
257 
258  # Apply the jacobian if configured
259  if self.config.doApplyWcsJacobian:
260  tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
261 
262  fullCatalog.extend(tempCat)
263 
264  # Now do the aperture information
265  instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
266  instMagErrIn = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] /
267  df[self.config.apertureInnerInstFluxField].values[use])
268  instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
269  instMagErrOut = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] /
270  df[self.config.apertureOuterInstFluxField].values[use])
271  ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
272  np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
273 
274  visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
275  visit['sources_read'] = True
276 
277  self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
278  use.size, visit['visit'], visit['deltaAper'])
279 
280  if ((counter % self.config.nVisitsPerCheckpoint) == 0 and
281  starObsDataRef is not None and visitCatDataRef is not None):
282  # We need to persist both the stars and the visit catalog which gets
283  # additional metadata from each visit.
284  starObsDataRef.put(fullCatalog)
285  visitCatDataRef.put(visitCat)
286 
287  self.log.info("Found all good star observations in %.2f s" %
288  (time.time() - startTime))
289 
290  return fullCatalog
291 
292  def _get_sourceTable_visit_columns(self):
293  """
294  Get the sourceTable_visit columns from the config.
295 
296  Returns
297  -------
298  columns : `list`
299  List of columns to read from sourceTable_visit
300  """
301  columns = [self.config.visitDataRefName, self.config.ccdDataRefName,
302  'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
303  self.config.instFluxField, self.config.instFluxField + 'Err',
304  self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
305  self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
306  if self.sourceSelector.config.doFlags:
307  columns.extend(self.sourceSelector.config.flags.bad)
308  if self.sourceSelector.config.doUnresolved:
309  columns.append(self.sourceSelector.config.unresolved.name)
310  if self.sourceSelector.config.doIsolated:
311  columns.append(self.sourceSelector.config.isolated.parentName)
312  columns.append(self.sourceSelector.config.isolated.nChildName)
313  if self.config.doSubtractLocalBackground:
314  columns.append(self.config.localBackgroundFluxField)
315 
316  return columns
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask
Definition: fgcmBuildStarsBase.py:260
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableConfig.setDefaults
def setDefaults(self)
Definition: fgcmBuildStarsTable.py:56
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.apertureInnerInstFluxField
apertureInnerInstFluxField
Definition: fgcmBuildStarsBase.py:146
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask._get_sourceTable_visit_columns
def _get_sourceTable_visit_columns(self)
Definition: fgcmBuildStarsTable.py:292
lsst.fgcmcal.utilities.computeApproxPixelAreaFields
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:472
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.findAndGroupDataRefs
def findAndGroupDataRefs(self, butler, dataRefs)
Definition: fgcmBuildStarsTable.py:112
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask
Definition: fgcmBuildStarsTable.py:96
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.apertureOuterInstFluxField
apertureOuterInstFluxField
Definition: fgcmBuildStarsBase.py:152
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask.fgcmMakeAllStarObservations
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
Definition: fgcmBuildStarsTable.py:153
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.doSubtractLocalBackground
doSubtractLocalBackground
Definition: fgcmBuildStarsBase.py:131
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.psfCandidateName
psfCandidateName
Definition: fgcmBuildStarsBase.py:126
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask._DefaultName
string _DefaultName
Definition: fgcmBuildStarsTable.py:102
lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableConfig
Definition: fgcmBuildStarsTable.py:47
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.sourceSelector
sourceSelector
Definition: fgcmBuildStarsBase.py:142
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase
Definition: fgcmBuildStarsBase.py:49
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.instFluxField
instFluxField
Definition: fgcmBuildStarsBase.py:52
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeSourceMapper
def _makeSourceMapper(self, sourceSchema)
Definition: fgcmBuildStarsBase.py:555
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.localBackgroundFluxField
localBackgroundFluxField
Definition: fgcmBuildStarsBase.py:137