lsst.pipe.tasks  16.0-13-g1e751bcc+12
propagateVisitFlags.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2014-2015 LSST/AURA
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 import numpy
24 from lsst.pex.config import Config, Field, DictField
25 from lsst.pipe.base import Task
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.table as afwTable
28 
29 
31  """!Configuration for propagating flags to coadd"""
32  flags = DictField(keytype=str, itemtype=float,
33  default={"calib_psf_candidate": 0.2, "calib_psf_used": 0.2, },
34  doc="Source catalog flags to propagate, with the threshold of relative occurrence.")
35  matchRadius = Field(dtype=float, default=0.2, doc="Source matching radius (arcsec)")
36 
37 
38 
44 
46  """!Task to propagate flags from single-frame measurements to coadd measurements
47 
48 \anchor PropagateVisitFlagsTask_
49 
50 \brief Propagate flags from individual visit measurements to coadd measurements
51 
52 \section pipe_tasks_propagateVisitFlagsTask_Contents Contents
53 
54  - \ref pipe_tasks_propagateVisitFlagsTask_Description
55  - \ref pipe_tasks_propagateVisitFlagsTask_Initialization
56  - \ref pipe_tasks_propagateVisitFlagsTask_Config
57  - \ref pipe_tasks_propagateVisitFlagsTask_Use
58  - \ref pipe_tasks_propagateVisitFlagsTask_Example
59 
60 \section pipe_tasks_propagateVisitFlagsTask_Description Description
61 
62 \copybrief PropagateVisitFlagsTask
63 
64 We want to be able to set a flag for sources on the coadds using flags
65 that were determined from the individual visits. A common example is sources
66 that were used for PSF determination, since we do not do any PSF determination
67 on the coadd but use the individual visits. This requires matching the coadd
68 source catalog to each of the catalogs from the inputs (see
69 PropagateVisitFlagsConfig.matchRadius), and thresholding on the number of
70 times a source is flagged on the input catalog.
71 
72 An important consideration in this is that the flagging of sources in the
73 individual visits can be somewhat stochastic, e.g., the same stars may not
74 always be used for PSF determination because the field of view moves slightly
75 between visits, or the seeing changed. We there threshold on the relative
76 occurrence of the flag in the visits (see PropagateVisitFlagsConfig.flags).
77 Flagging a source that is always flagged in inputs corresponds to a threshold
78 of 1, while flagging a source that is flagged in any of the input corresponds
79 to a threshold of 0. But neither of these extrema are really useful in
80 practise.
81 
82 Setting the threshold too high means that sources that are not consistently
83 flagged (e.g., due to chip gaps) will not have the flag propagated. Setting
84 that threshold too low means that random sources which are falsely flagged in
85 the inputs will start to dominate. If in doubt, we suggest making this
86 threshold relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more
87 confidence in the quality of the flagging, the lower the threshold can be.
88 
89 The relative occurrence accounts for the edge of the field-of-view of the
90 camera, but does not include chip gaps, bad or saturated pixels, etc.
91 
92 \section pipe_tasks_propagateVisitFlagsTask_Initialization Initialization
93 
94 Beyond the usual Task initialization, PropagateVisitFlagsTask also requires
95 a schema for the catalog that is being constructed.
96 
97 \section pipe_tasks_propagateVisitFlagsTask_Config Configuration parameters
98 
99 See \ref PropagateVisitFlagsConfig
100 
101 \section pipe_tasks_propagateVisitFlagsTask_Use Use
102 
103 The 'run' method (described below) is the entry-point for operations. The
104 'getCcdInputs' staticmethod is provided as a convenience for retrieving the
105 'ccdInputs' (CCD inputs table) from an Exposure.
106 
107 \copydoc run
108 
109 \section pipe_tasks_propagateVisitFlagsTask_Example Example
110 
111 \code{.py}
112 # Requires:
113 # * butler: data butler, for retrieving the CCD catalogs
114 # * coaddCatalog: catalog of source measurements on the coadd (lsst.afw.table.SourceCatalog)
115 # * coaddExposure: coadd (lsst.afw.image.Exposure)
116 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask, PropagateVisitFlagsConfig
117 config = PropagateVisitFlagsConfig()
118 config.flags["calib_psf_used"] = 0.3 # Relative threshold for this flag
119 config.matchRadius = 0.5 # Matching radius in arcsec
120 task = PropagateVisitFlagsTask(coaddCatalog.schema, config=config)
121 ccdInputs = task.getCcdInputs(coaddExposure)
122 task.run(butler, coaddCatalog, ccdInputs, coaddExposure.getWcs())
123 \endcode
124 """
125  ConfigClass = PropagateVisitFlagsConfig
126 
127  def __init__(self, schema, **kwargs):
128  Task.__init__(self, **kwargs)
129  self.schema = schema
130  self._keys = dict((f, self.schema.addField(f, type="Flag", doc="Propagated from visits")) for
131  f in self.config.flags)
132 
133  @staticmethod
134  def getCcdInputs(coaddExposure):
135  """!Convenience method to retrieve the CCD inputs table from a coadd exposure"""
136  return coaddExposure.getInfo().getCoaddInputs().ccds
137 
138  def run(self, butler, coaddSources, ccdInputs, coaddWcs):
139  """!Propagate flags from individual visit measurements to coadd
140 
141  This requires matching the coadd source catalog to each of the catalogs
142  from the inputs, and thresholding on the number of times a source is
143  flagged on the input catalog. The threshold is made on the relative
144  occurrence of the flag in each source. Flagging a source that is always
145  flagged in inputs corresponds to a threshold of 1, while flagging a
146  source that is flagged in any of the input corresponds to a threshold of
147  0. But neither of these extrema are really useful in practise.
148 
149  Setting the threshold too high means that sources that are not consistently
150  flagged (e.g., due to chip gaps) will not have the flag propagated. Setting
151  that threshold too low means that random sources which are falsely flagged in
152  the inputs will start to dominate. If in doubt, we suggest making this threshold
153  relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more confidence in
154  the quality of the flagging, the lower the threshold can be.
155 
156  The relative occurrence accounts for the edge of the field-of-view of
157  the camera, but does not include chip gaps, bad or saturated pixels, etc.
158 
159  @param[in] butler Data butler, for retrieving the input source catalogs
160  @param[in,out] coaddSources Source catalog from the coadd
161  @param[in] ccdInputs Table of CCDs that contribute to the coadd
162  @param[in] coaddWcs Wcs for coadd
163  """
164  if len(self.config.flags) == 0:
165  return
166 
167  flags = self._keys.keys()
168  visitKey = ccdInputs.schema.find("visit").key
169  ccdKey = ccdInputs.schema.find("ccd").key
170  radius = self.config.matchRadius*afwGeom.arcseconds
171 
172  self.log.info("Propagating flags %s from inputs" % (flags,))
173 
174  counts = dict((f, numpy.zeros(len(coaddSources), dtype=int)) for f in flags)
175  indices = numpy.array([s.getId() for s in coaddSources]) # Allowing for non-contiguous data
176 
177  # Accumulate counts of flags being set
178  for ccdRecord in ccdInputs:
179  v = ccdRecord.get(visitKey)
180  c = ccdRecord.get(ccdKey)
181  ccdSources = butler.get("src", visit=int(v), ccd=int(c), immediate=True)
182  for sourceRecord in ccdSources:
183  sourceRecord.updateCoord(ccdRecord.getWcs())
184  for flag in flags:
185  # We assume that the flags will be relatively rare, so it is more efficient to match
186  # against a subset of the input catalog for each flag than it is to match once against
187  # the entire catalog. It would be best to have built a kd-tree on coaddSources and
188  # keep reusing that for the matching, but we don't have a suitable implementation.
189  mc = afwTable.MatchControl()
190  mc.findOnlyClosest = False
191  matches = afwTable.matchRaDec(coaddSources, ccdSources[ccdSources.get(flag)], radius, mc)
192  for m in matches:
193  index = (numpy.where(indices == m.first.getId()))[0][0]
194  counts[flag][index] += 1
195 
196  # Apply threshold
197  for f in flags:
198  key = self._keys[f]
199  for s, num in zip(coaddSources, counts[f]):
200  numOverlaps = len(ccdInputs.subsetContaining(s.getCentroid(), coaddWcs, True))
201  s.setFlag(key, bool(num > numOverlaps*self.config.flags[f]))
202  self.log.info("Propagated %d sources with flag %s" % (sum(s.get(key) for s in coaddSources), f))
Task to propagate flags from single-frame measurements to coadd measurements.
def getCcdInputs(coaddExposure)
Convenience method to retrieve the CCD inputs table from a coadd exposure.
def run(self, butler, coaddSources, ccdInputs, coaddWcs)
Propagate flags from individual visit measurements to coadd.