lsst.pipe.tasks  13.0-66-gfbf2f2ce+5
propagateVisitFlags.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2014-2015 LSST/AURA
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 from __future__ import absolute_import, division, print_function
25 from builtins import zip
26 import numpy
27 from lsst.pex.config import Config, Field, DictField
28 from lsst.pipe.base import Task
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.table as afwTable
31 
32 
34  """!Configuration for propagating flags to coadd"""
35  flags = DictField(keytype=str, itemtype=float,
36  default={"calib_psfCandidate": 0.2, "calib_psfUsed": 0.2, },
37  doc="Source catalog flags to propagate, with the threshold of relative occurrence.")
38  matchRadius = Field(dtype=float, default=0.2, doc="Source matching radius (arcsec)")
39 
40 
41 
47 
49  """!Task to propagate flags from single-frame measurements to coadd measurements
50 
51 \anchor PropagateVisitFlagsTask_
52 
53 \brief Propagate flags from individual visit measurements to coadd measurements
54 
55 \section pipe_tasks_propagateVisitFlagsTask_Contents Contents
56 
57  - \ref pipe_tasks_propagateVisitFlagsTask_Description
58  - \ref pipe_tasks_propagateVisitFlagsTask_Initialization
59  - \ref pipe_tasks_propagateVisitFlagsTask_Config
60  - \ref pipe_tasks_propagateVisitFlagsTask_Use
61  - \ref pipe_tasks_propagateVisitFlagsTask_Example
62 
63 \section pipe_tasks_propagateVisitFlagsTask_Description Description
64 
65 \copybrief PropagateVisitFlagsTask
66 
67 We want to be able to set a flag for sources on the coadds using flags
68 that were determined from the individual visits. A common example is sources
69 that were used for PSF determination, since we do not do any PSF determination
70 on the coadd but use the individual visits. This requires matching the coadd
71 source catalog to each of the catalogs from the inputs (see
72 PropagateVisitFlagsConfig.matchRadius), and thresholding on the number of
73 times a source is flagged on the input catalog.
74 
75 An important consideration in this is that the flagging of sources in the
76 individual visits can be somewhat stochastic, e.g., the same stars may not
77 always be used for PSF determination because the field of view moves slightly
78 between visits, or the seeing changed. We there threshold on the relative
79 occurrence of the flag in the visits (see PropagateVisitFlagsConfig.flags).
80 Flagging a source that is always flagged in inputs corresponds to a threshold
81 of 1, while flagging a source that is flagged in any of the input corresponds
82 to a threshold of 0. But neither of these extrema are really useful in
83 practise.
84 
85 Setting the threshold too high means that sources that are not consistently
86 flagged (e.g., due to chip gaps) will not have the flag propagated. Setting
87 that threshold too low means that random sources which are falsely flagged in
88 the inputs will start to dominate. If in doubt, we suggest making this
89 threshold relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more
90 confidence in the quality of the flagging, the lower the threshold can be.
91 
92 The relative occurrence accounts for the edge of the field-of-view of the
93 camera, but does not include chip gaps, bad or saturated pixels, etc.
94 
95 \section pipe_tasks_propagateVisitFlagsTask_Initialization Initialization
96 
97 Beyond the usual Task initialization, PropagateVisitFlagsTask also requires
98 a schema for the catalog that is being constructed.
99 
100 \section pipe_tasks_propagateVisitFlagsTask_Config Configuration parameters
101 
102 See \ref PropagateVisitFlagsConfig
103 
104 \section pipe_tasks_propagateVisitFlagsTask_Use Use
105 
106 The 'run' method (described below) is the entry-point for operations. The
107 'getCcdInputs' staticmethod is provided as a convenience for retrieving the
108 'ccdInputs' (CCD inputs table) from an Exposure.
109 
110 \copydoc run
111 
112 \section pipe_tasks_propagateVisitFlagsTask_Example Example
113 
114 \code{.py}
115 # Requires:
116 # * butler: data butler, for retrieving the CCD catalogs
117 # * coaddCatalog: catalog of source measurements on the coadd (lsst.afw.table.SourceCatalog)
118 # * coaddExposure: coadd (lsst.afw.image.Exposure)
119 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask, PropagateVisitFlagsConfig
120 config = PropagateVisitFlagsConfig()
121 config.flags["calib.psf.used"] = 0.3 # Relative threshold for this flag
122 config.matchRadius = 0.5 # Matching radius in arcsec
123 task = PropagateVisitFlagsTask(coaddCatalog.schema, config=config)
124 ccdInputs = task.getCcdInputs(coaddExposure)
125 task.run(butler, coaddCatalog, ccdInputs, coaddExposure.getWcs())
126 \endcode
127 """
128  ConfigClass = PropagateVisitFlagsConfig
129 
130  def __init__(self, schema, **kwargs):
131  Task.__init__(self, **kwargs)
132  self.schema = schema
133  self._keys = dict((f, self.schema.addField(f, type="Flag", doc="Propagated from visits")) for
134  f in self.config.flags)
135 
136  @staticmethod
137  def getCcdInputs(coaddExposure):
138  """!Convenience method to retrieve the CCD inputs table from a coadd exposure"""
139  return coaddExposure.getInfo().getCoaddInputs().ccds
140 
141  def run(self, butler, coaddSources, ccdInputs, coaddWcs):
142  """!Propagate flags from individual visit measurements to coadd
143 
144  This requires matching the coadd source catalog to each of the catalogs
145  from the inputs, and thresholding on the number of times a source is
146  flagged on the input catalog. The threshold is made on the relative
147  occurrence of the flag in each source. Flagging a source that is always
148  flagged in inputs corresponds to a threshold of 1, while flagging a
149  source that is flagged in any of the input corresponds to a threshold of
150  0. But neither of these extrema are really useful in practise.
151 
152  Setting the threshold too high means that sources that are not consistently
153  flagged (e.g., due to chip gaps) will not have the flag propagated. Setting
154  that threshold too low means that random sources which are falsely flagged in
155  the inputs will start to dominate. If in doubt, we suggest making this threshold
156  relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more confidence in
157  the quality of the flagging, the lower the threshold can be.
158 
159  The relative occurrence accounts for the edge of the field-of-view of
160  the camera, but does not include chip gaps, bad or saturated pixels, etc.
161 
162  @param[in] butler Data butler, for retrieving the input source catalogs
163  @param[in,out] coaddSources Source catalog from the coadd
164  @param[in] ccdInputs Table of CCDs that contribute to the coadd
165  @param[in] coaddWcs Wcs for coadd
166  """
167  if len(self.config.flags) == 0:
168  return
169 
170  flags = self._keys.keys()
171  visitKey = ccdInputs.schema.find("visit").key
172  ccdKey = ccdInputs.schema.find("ccd").key
173  radius = self.config.matchRadius*afwGeom.arcseconds
174 
175  self.log.info("Propagating flags %s from inputs" % (flags,))
176 
177  counts = dict((f, numpy.zeros(len(coaddSources), dtype=int)) for f in flags)
178  indices = numpy.array([s.getId() for s in coaddSources]) # Allowing for non-contiguous data
179 
180  # Accumulate counts of flags being set
181  for ccdRecord in ccdInputs:
182  v = ccdRecord.get(visitKey)
183  c = ccdRecord.get(ccdKey)
184  ccdSources = butler.get("src", visit=int(v), ccd=int(c), immediate=True)
185  for sourceRecord in ccdSources:
186  sourceRecord.updateCoord(ccdRecord.getWcs())
187  for flag in flags:
188  # We assume that the flags will be relatively rare, so it is more efficient to match
189  # against a subset of the input catalog for each flag than it is to match once against
190  # the entire catalog. It would be best to have built a kd-tree on coaddSources and
191  # keep reusing that for the matching, but we don't have a suitable implementation.
192  mc = afwTable.MatchControl()
193  mc.findOnlyClosest = False
194  matches = afwTable.matchRaDec(coaddSources, ccdSources[ccdSources.get(flag)], radius, mc)
195  for m in matches:
196  index = (numpy.where(indices == m.first.getId()))[0][0]
197  counts[flag][index] += 1
198 
199  # Apply threshold
200  for f in flags:
201  key = self._keys[f]
202  for s, num in zip(coaddSources, counts[f]):
203  numOverlaps = len(ccdInputs.subsetContaining(s.getCentroid(), coaddWcs, True))
204  s.setFlag(key, bool(num > numOverlaps*self.config.flags[f]))
205  self.log.info("Propagated %d sources with flag %s" % (sum(s.get(key) for s in coaddSources), f))
Task to propagate flags from single-frame measurements to coadd measurements.
def getCcdInputs(coaddExposure)
Convenience method to retrieve the CCD inputs table from a coadd exposure.
def run(self, butler, coaddSources, ccdInputs, coaddWcs)
Propagate flags from individual visit measurements to coadd.