lsst.fgcmcal  21.0.0-5-gb155db7+5dc39e03a1
fgcmBuildStars.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Build star observations for input to FGCM.
24 
25 This task finds all the visits and calexps in a repository (or a subset
26 based on command line parameters) and extract all the potential calibration
27 stars for input into fgcm. This task additionally uses fgcm to match
28 star observations into unique stars, and performs as much cleaning of
29 the input catalog as possible.
30 """
31 
32 import time
33 
34 import numpy as np
35 
36 import lsst.pex.config as pexConfig
37 import lsst.pipe.base as pipeBase
38 import lsst.afw.table as afwTable
39 
40 from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41 from .utilities import computeApproxPixelAreaFields
42 
43 __all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask']
44 
45 
47  """Config for FgcmBuildStarsTask"""
48 
49  referenceCCD = pexConfig.Field(
50  doc="Reference CCD for scanning visits",
51  dtype=int,
52  default=13,
53  )
54  checkAllCcds = pexConfig.Field(
55  doc=("Check repo for all CCDs for each visit specified. To be used when the "
56  "full set of ids (visit/ccd) are not specified on the command line. For "
57  "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58  "faster than the alternatives."),
59  dtype=bool,
60  default=True,
61  )
62 
63  def setDefaults(self):
64  super().setDefaults()
65 
66  sourceSelector = self.sourceSelectorsourceSelector["science"]
67 
68  # The names here correspond to raw src catalogs, which differ
69  # from the post-transformed sourceTable_visit catalogs.
70  # Therefore, field and flag names cannot be easily
71  # derived from the base config class.
72  fluxFlagName = self.instFluxFieldinstFluxField[0: -len('instFlux')] + 'flag'
73  sourceSelector.flags.bad = ['base_PixelFlags_flag_edge',
74  'base_PixelFlags_flag_interpolatedCenter',
75  'base_PixelFlags_flag_saturatedCenter',
76  'base_PixelFlags_flag_crCenter',
77  'base_PixelFlags_flag_bad',
78  'base_PixelFlags_flag_interpolated',
79  'base_PixelFlags_flag_saturated',
80  'slot_Centroid_flag',
81  fluxFlagName]
82 
83  if self.doSubtractLocalBackgrounddoSubtractLocalBackground:
84  localBackgroundFlagName = self.localBackgroundFluxFieldlocalBackgroundFluxField[0: -len('instFlux')] + 'flag'
85  sourceSelector.flags.bad.append(localBackgroundFlagName)
86 
87  sourceSelector.signalToNoise.fluxField = self.instFluxFieldinstFluxField
88  sourceSelector.signalToNoise.errField = self.instFluxFieldinstFluxField + 'Err'
89 
90 
92  """
93  Build stars for the FGCM global calibration, using src catalogs.
94  """
95  ConfigClass = FgcmBuildStarsConfig
96  RunnerClass = FgcmBuildStarsRunner
97  _DefaultName = "fgcmBuildStars"
98 
99  @classmethod
100  def _makeArgumentParser(cls):
101  """Create an argument parser"""
102  parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
103  parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789")
104 
105  return parser
106 
107  def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
108  if butler is None:
109  raise RuntimeError("Gen2 _findAndGroupDataRefs must be called with a butler.")
110  if calexpDataRefDict is not None:
111  self.log.warn("Ignoring calexpDataRefDict in gen2 _findAndGroupDataRefs")
112 
113  self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
114 
115  ccdIds = []
116  for detector in camera:
117  ccdIds.append(detector.getId())
118 
119  # TODO: related to DM-13730, this dance of looking for source visits
120  # will be unnecessary with Gen3 Butler. This should be part of
121  # DM-13730.
122 
123  nVisits = 0
124 
125  groupedDataRefs = {}
126  for dataRef in dataRefs:
127  visit = dataRef.dataId[self.config.visitDataRefName]
128  # If we don't have the dataset, just continue
129  if not dataRef.datasetExists(datasetType='src'):
130  continue
131  # If we need to check all ccds, do it here
132  if self.config.checkAllCcds:
133  if visit in groupedDataRefs:
134  # We already have found this visit
135  continue
136  dataId = dataRef.dataId.copy()
137  # For each ccd we must check that a valid source catalog exists.
138  for ccdId in ccdIds:
139  dataId[self.config.ccdDataRefName] = ccdId
140  if butler.datasetExists('src', dataId=dataId):
141  goodDataRef = butler.dataRef('src', dataId=dataId)
142  if visit in groupedDataRefs:
143  if (goodDataRef.dataId[self.config.ccdDataRefName] not in
144  [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
145  groupedDataRefs[visit].append(goodDataRef)
146  else:
147  # This is a new visit
148  nVisits += 1
149  groupedDataRefs[visit] = [goodDataRef]
150  else:
151  # We have already confirmed that the dataset exists, so no need
152  # to check here.
153  if visit in groupedDataRefs:
154  if (dataRef.dataId[self.config.ccdDataRefName] not in
155  [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
156  groupedDataRefs[visit].append(dataRef)
157  else:
158  # This is a new visit
159  nVisits += 1
160  groupedDataRefs[visit] = [dataRef]
161 
162  if (nVisits % 100) == 0 and nVisits > 0:
163  self.log.info("Found %d unique %ss..." % (nVisits,
164  self.config.visitDataRefName))
165 
166  self.log.info("Found %d unique %ss total." % (nVisits,
167  self.config.visitDataRefName))
168 
169  # Put them in ccd order, with the reference ccd first (if available)
170  def ccdSorter(dataRef):
171  ccdId = dataRef.dataId[self.config.ccdDataRefName]
172  if ccdId == self.config.referenceCCD:
173  return -100
174  else:
175  return ccdId
176 
177  # If we did not check all ccds, put them in ccd order
178  if not self.config.checkAllCcds:
179  for visit in groupedDataRefs:
180  groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
181 
182  # This should be sorted by visit (the key)
183  return dict(sorted(groupedDataRefs.items()))
184 
185  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
186  srcSchemaDataRef,
187  camera,
188  calibFluxApertureRadius=None,
189  visitCatDataRef=None,
190  starObsDataRef=None,
191  inStarObsCat=None):
192  startTime = time.time()
193 
194  # If both dataRefs are None, then we assume the caller does not
195  # want to store checkpoint files. If both are set, we will
196  # do checkpoint files. And if only one is set, this is potentially
197  # unintentional and we will warn.
198  if (visitCatDataRef is not None and starObsDataRef is None
199  or visitCatDataRef is None and starObsDataRef is not None):
200  self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
201  "no checkpoint files will be persisted.")
202 
203  if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
204  raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
205 
206  # create our source schema. Use the first valid dataRef
207  dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
208  sourceSchema = dataRef.get('src_schema', immediate=True).schema
209 
210  # Construct a mapping from ccd number to index
211  ccdMapping = {}
212  for ccdIndex, detector in enumerate(camera):
213  ccdMapping[detector.getId()] = ccdIndex
214 
215  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
216 
217  sourceMapper = self._makeSourceMapper_makeSourceMapper(sourceSchema)
218 
219  # We also have a temporary catalog that will accumulate aperture measurements
220  aperMapper = self._makeAperMapper_makeAperMapper(sourceSchema)
221 
222  outputSchema = sourceMapper.getOutputSchema()
223 
224  if inStarObsCat is not None:
225  fullCatalog = inStarObsCat
226  comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
227  comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
228  if not comp1 or not comp2:
229  raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
230  else:
231  fullCatalog = afwTable.BaseCatalog(outputSchema)
232 
233  # FGCM will provide relative calibration for the flux in config.instFluxField
234 
235  instFluxKey = sourceSchema[self.config.instFluxField].asKey()
236  instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey()
237  visitKey = outputSchema['visit'].asKey()
238  ccdKey = outputSchema['ccd'].asKey()
239  instMagKey = outputSchema['instMag'].asKey()
240  instMagErrKey = outputSchema['instMagErr'].asKey()
241  deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey()
242 
243  # Prepare local background if desired
244  if self.config.doSubtractLocalBackground:
245  localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
246  localBackgroundArea = np.pi*calibFluxApertureRadius**2.
247 
248  aperOutputSchema = aperMapper.getOutputSchema()
249 
250  instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
251  instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey()
252  instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
253  instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey()
254  instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
255  instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
256  instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
257  instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
258 
259  k = 2.5/np.log(10.)
260 
261  # loop over visits
262  for ctr, visit in enumerate(visitCat):
263  if visit['sources_read']:
264  continue
265 
266  expTime = visit['exptime']
267 
268  nStarInVisit = 0
269 
270  # Reset the aperture catalog (per visit)
271  aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
272 
273  for dataRef in groupedDataRefs[visit['visit']]:
274 
275  ccdId = dataRef.dataId[self.config.ccdDataRefName]
276 
277  sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
278  goodSrc = self.sourceSelector.selectSources(sources)
279 
280  # Need to add a selection based on the local background correction
281  # if necessary
282  if self.config.doSubtractLocalBackground:
283  localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
284 
285  bad, = np.where((sources[instFluxKey] - localBackground) <= 0.0)
286  goodSrc.selected[bad] = False
287 
288  tempCat = afwTable.BaseCatalog(fullCatalog.schema)
289  tempCat.reserve(goodSrc.selected.sum())
290  tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
291  tempCat[visitKey][:] = visit['visit']
292  tempCat[ccdKey][:] = ccdId
293 
294  # Compute "instrumental magnitude" by scaling flux with exposure time.
295  scaledInstFlux = (sources[instFluxKey][goodSrc.selected]
296  * visit['scaling'][ccdMapping[ccdId]])
297  tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
298 
299  # Compute the change in magnitude from the background offset
300  if self.config.doSubtractLocalBackground:
301  # At the moment we only adjust the flux and not the flux
302  # error by the background because the error on
303  # base_LocalBackground_instFlux is the rms error in the
304  # background annulus, not the error on the mean in the
305  # background estimate (which is much smaller, by sqrt(n)
306  # pixels used to estimate the background, which we do not
307  # have access to in this task). In the default settings,
308  # the annulus is sufficiently large such that these
309  # additional errors are are negligibly small (much less
310  # than a mmag in quadrature).
311 
312  # This is the difference between the mag with background correction
313  # and the mag without background correction.
314  tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected]
315  - localBackground[goodSrc.selected]) -
316  -2.5*np.log10(sources[instFluxKey][goodSrc.selected]))
317  else:
318  tempCat[deltaMagBkgKey][:] = 0.0
319 
320  # Compute instMagErr from instFluxErr/instFlux, any scaling
321  # will cancel out.
322 
323  tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected]
324  / sources[instFluxKey][goodSrc.selected])
325 
326  # Compute the jacobian from an approximate PixelAreaBoundedField
327  tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'],
328  tempCat['y'])
329 
330  # Apply the jacobian if configured
331  if self.config.doApplyWcsJacobian:
332  tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
333 
334  fullCatalog.extend(tempCat)
335 
336  # And the aperture information
337  # This does not need the jacobian because it is all locally relative
338  tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
339  tempAperCat.reserve(goodSrc.selected.sum())
340  tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
341 
342  with np.warnings.catch_warnings():
343  # Ignore warnings, we will filter infinities and
344  # nans below.
345  np.warnings.simplefilter("ignore")
346 
347  tempAperCat[instMagInKey][:] = -2.5*np.log10(
348  sources[instFluxAperInKey][goodSrc.selected])
349  tempAperCat[instMagErrInKey][:] = k*(
350  sources[instFluxErrAperInKey][goodSrc.selected]
351  / sources[instFluxAperInKey][goodSrc.selected])
352  tempAperCat[instMagOutKey][:] = -2.5*np.log10(
353  sources[instFluxAperOutKey][goodSrc.selected])
354  tempAperCat[instMagErrOutKey][:] = k*(
355  sources[instFluxErrAperOutKey][goodSrc.selected]
356  / sources[instFluxAperOutKey][goodSrc.selected])
357 
358  aperVisitCatalog.extend(tempAperCat)
359 
360  nStarInVisit += len(tempCat)
361 
362  # Compute the median delta-aper
363  if not aperVisitCatalog.isContiguous():
364  aperVisitCatalog = aperVisitCatalog.copy(deep=True)
365 
366  instMagIn = aperVisitCatalog[instMagInKey]
367  instMagErrIn = aperVisitCatalog[instMagErrInKey]
368  instMagOut = aperVisitCatalog[instMagOutKey]
369  instMagErrOut = aperVisitCatalog[instMagErrOutKey]
370 
371  ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn)
372  & np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
373 
374  visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
375  visit['sources_read'] = True
376 
377  self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" %
378  (nStarInVisit, visit['visit'], visit['deltaAper']))
379 
380  if ((ctr % self.config.nVisitsPerCheckpoint) == 0
381  and starObsDataRef is not None and visitCatDataRef is not None):
382  # We need to persist both the stars and the visit catalog which gets
383  # additional metadata from each visit.
384  starObsDataRef.put(fullCatalog)
385  visitCatDataRef.put(visitCat)
386 
387  self.log.info("Found all good star observations in %.2f s" %
388  (time.time() - startTime))
389 
390  return fullCatalog
391 
392  def _makeAperMapper(self, sourceSchema):
393  """
394  Make a schema mapper for fgcm aperture measurements
395 
396  Parameters
397  ----------
398  sourceSchema: `afwTable.Schema`
399  Default source schema from the butler
400 
401  Returns
402  -------
403  aperMapper: `afwTable.schemaMapper`
404  Mapper to the FGCM aperture schema
405  """
406 
407  aperMapper = afwTable.SchemaMapper(sourceSchema)
408  aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
409  aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
410  aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64,
411  doc="Magnitude at inner aperture")
412  aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64,
413  doc="Magnitude error at inner aperture")
414  aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64,
415  doc="Magnitude at outer aperture")
416  aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64,
417  doc="Magnitude error at outer aperture")
418 
419  return aperMapper
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, srcSchemaDataRef, camera, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:486