lsst.pipe.tasks  16.0-39-gcc1e6200
propagateVisitFlags.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2014-2015 LSST/AURA
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 import numpy
24 from lsst.pex.config import Config, Field, DictField
25 from lsst.pipe.base import Task
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.table as afwTable
28 
29 
31  """!Configuration for propagating flags to coadd"""
32  flags = DictField(keytype=str, itemtype=float,
33  default={"calib_psf_candidate": 0.2, "calib_psf_used": 0.2, "calib_psf_reserved": 0.2,
34  "calib_astrometry_used": 0.2, "calib_photometry_used": 0.2,
35  "calib_photometry_reserved": 0.2, },
36  doc=("Source catalog flags to propagate, with the threshold of relative occurrence "
37  "(valid range: [0-1], default is 0.2). Coadd object will have flag set if the "
38  "fraction of input visits in which it is flagged is greater than the threshold."))
39  matchRadius = Field(dtype=float, default=0.2, doc="Source matching radius (arcsec)")
40  ccdName = Field(dtype=str, default='ccd', doc="Name of ccd to give to butler")
41 
42 
43 
49 
51  r"""!Task to propagate flags from single-frame measurements to coadd measurements
52 
53 \anchor PropagateVisitFlagsTask_
54 
55 \brief Propagate flags from individual visit measurements to coadd measurements
56 
57 \section pipe_tasks_propagateVisitFlagsTask_Contents Contents
58 
59  - \ref pipe_tasks_propagateVisitFlagsTask_Description
60  - \ref pipe_tasks_propagateVisitFlagsTask_Initialization
61  - \ref pipe_tasks_propagateVisitFlagsTask_Config
62  - \ref pipe_tasks_propagateVisitFlagsTask_Use
63  - \ref pipe_tasks_propagateVisitFlagsTask_Example
64 
65 \section pipe_tasks_propagateVisitFlagsTask_Description Description
66 
67 \copybrief PropagateVisitFlagsTask
68 
69 We want to be able to set a flag for sources on the coadds using flags
70 that were determined from the individual visits. A common example is sources
71 that were used for PSF determination, since we do not do any PSF determination
72 on the coadd but use the individual visits. This requires matching the coadd
73 source catalog to each of the catalogs from the inputs (see
74 PropagateVisitFlagsConfig.matchRadius), and thresholding on the number of
75 times a source is flagged on the input catalog.
76 
77 An important consideration in this is that the flagging of sources in the
78 individual visits can be somewhat stochastic, e.g., the same stars may not
79 always be used for PSF determination because the field of view moves slightly
80 between visits, or the seeing changed. We there threshold on the relative
81 occurrence of the flag in the visits (see PropagateVisitFlagsConfig.flags).
82 Flagging a source that is always flagged in inputs corresponds to a threshold
83 of 1, while flagging a source that is flagged in any of the input corresponds
84 to a threshold of 0. But neither of these extrema are really useful in
85 practise.
86 
87 Setting the threshold too high means that sources that are not consistently
88 flagged (e.g., due to chip gaps) will not have the flag propagated. Setting
89 that threshold too low means that random sources which are falsely flagged in
90 the inputs will start to dominate. If in doubt, we suggest making this
91 threshold relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more
92 confidence in the quality of the flagging, the lower the threshold can be.
93 
94 The relative occurrence accounts for the edge of the field-of-view of the
95 camera, but does not include chip gaps, bad or saturated pixels, etc.
96 
97 \section pipe_tasks_propagateVisitFlagsTask_Initialization Initialization
98 
99 Beyond the usual Task initialization, PropagateVisitFlagsTask also requires
100 a schema for the catalog that is being constructed.
101 
102 \section pipe_tasks_propagateVisitFlagsTask_Config Configuration parameters
103 
104 See \ref PropagateVisitFlagsConfig
105 
106 \section pipe_tasks_propagateVisitFlagsTask_Use Use
107 
108 The 'run' method (described below) is the entry-point for operations. The
109 'getCcdInputs' staticmethod is provided as a convenience for retrieving the
110 'ccdInputs' (CCD inputs table) from an Exposure.
111 
112 \copydoc run
113 
114 \section pipe_tasks_propagateVisitFlagsTask_Example Example
115 
116 \code{.py}
117 # Requires:
118 # * butler: data butler, for retrieving the CCD catalogs
119 # * coaddCatalog: catalog of source measurements on the coadd (lsst.afw.table.SourceCatalog)
120 # * coaddExposure: coadd (lsst.afw.image.Exposure)
121 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask, PropagateVisitFlagsConfig
122 config = PropagateVisitFlagsConfig()
123 config.flags["calib_psf_used"] = 0.3 # Relative threshold for this flag
124 config.matchRadius = 0.5 # Matching radius in arcsec
125 task = PropagateVisitFlagsTask(coaddCatalog.schema, config=config)
126 ccdInputs = task.getCcdInputs(coaddExposure)
127 task.run(butler, coaddCatalog, ccdInputs, coaddExposure.getWcs())
128 \endcode
129 """
130  ConfigClass = PropagateVisitFlagsConfig
131 
132  def __init__(self, schema, **kwargs):
133  Task.__init__(self, **kwargs)
134  self.schema = schema
135  self._keys = dict((f, self.schema.addField(f, type="Flag", doc="Propagated from visits")) for
136  f in self.config.flags)
137 
138  @staticmethod
139  def getCcdInputs(coaddExposure):
140  """!Convenience method to retrieve the CCD inputs table from a coadd exposure"""
141  return coaddExposure.getInfo().getCoaddInputs().ccds
142 
143  def run(self, butler, coaddSources, ccdInputs, coaddWcs):
144  """!Propagate flags from individual visit measurements to coadd
145 
146  This requires matching the coadd source catalog to each of the catalogs
147  from the inputs, and thresholding on the number of times a source is
148  flagged on the input catalog. The threshold is made on the relative
149  occurrence of the flag in each source. Flagging a source that is always
150  flagged in inputs corresponds to a threshold of 1, while flagging a
151  source that is flagged in any of the input corresponds to a threshold of
152  0. But neither of these extrema are really useful in practise.
153 
154  Setting the threshold too high means that sources that are not consistently
155  flagged (e.g., due to chip gaps) will not have the flag propagated. Setting
156  that threshold too low means that random sources which are falsely flagged in
157  the inputs will start to dominate. If in doubt, we suggest making this threshold
158  relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more confidence in
159  the quality of the flagging, the lower the threshold can be.
160 
161  The relative occurrence accounts for the edge of the field-of-view of
162  the camera, but does not include chip gaps, bad or saturated pixels, etc.
163 
164  @param[in] butler Data butler, for retrieving the input source catalogs
165  @param[in,out] coaddSources Source catalog from the coadd
166  @param[in] ccdInputs Table of CCDs that contribute to the coadd
167  @param[in] coaddWcs Wcs for coadd
168  """
169  if len(self.config.flags) == 0:
170  return
171 
172  flags = self._keys.keys()
173  visitKey = ccdInputs.schema.find("visit").key
174  ccdKey = ccdInputs.schema.find("ccd").key
175  radius = self.config.matchRadius*afwGeom.arcseconds
176 
177  self.log.info("Propagating flags %s from inputs" % (flags,))
178 
179  counts = dict((f, numpy.zeros(len(coaddSources), dtype=int)) for f in flags)
180  indices = numpy.array([s.getId() for s in coaddSources]) # Allowing for non-contiguous data
181 
182  # Accumulate counts of flags being set
183  for ccdRecord in ccdInputs:
184  v = ccdRecord.get(visitKey)
185  c = ccdRecord.get(ccdKey)
186  dataId = {"visit": int(v), self.config.ccdName: int(c)}
187  ccdSources = butler.get("src", dataId=dataId, immediate=True)
188  for sourceRecord in ccdSources:
189  sourceRecord.updateCoord(ccdRecord.getWcs())
190  for flag in flags:
191  # We assume that the flags will be relatively rare, so it is more efficient to match
192  # against a subset of the input catalog for each flag than it is to match once against
193  # the entire catalog. It would be best to have built a kd-tree on coaddSources and
194  # keep reusing that for the matching, but we don't have a suitable implementation.
195  mc = afwTable.MatchControl()
196  mc.findOnlyClosest = False
197  matches = afwTable.matchRaDec(coaddSources, ccdSources[ccdSources.get(flag)], radius, mc)
198  for m in matches:
199  index = (numpy.where(indices == m.first.getId()))[0][0]
200  counts[flag][index] += 1
201 
202  # Apply threshold
203  for f in flags:
204  key = self._keys[f]
205  for s, num in zip(coaddSources, counts[f]):
206  numOverlaps = len(ccdInputs.subsetContaining(s.getCentroid(), coaddWcs, True))
207  s.setFlag(key, bool(num > numOverlaps*self.config.flags[f]))
208  self.log.info("Propagated %d sources with flag %s" % (sum(s.get(key) for s in coaddSources), f))
Task to propagate flags from single-frame measurements to coadd measurements.
def getCcdInputs(coaddExposure)
Convenience method to retrieve the CCD inputs table from a coadd exposure.
Configuration for propagating flags to coadd.
def run(self, butler, coaddSources, ccdInputs, coaddWcs)
Propagate flags from individual visit measurements to coadd.