lsst.meas.base  14.0-17-g4f4ea82+1
sfm.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 LSST Corporation.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 """Base classes for single-frame measurement plugins and the driver task for these.
24 
25 In single-frame measurement, we assume that detection and probably deblending have already been run on
26 the same frame, so a SourceCatalog has already been created with Footprints (which may be HeavyFootprints).
27 Measurements are generally recorded in the coordinate system of the image being measured (and all
28 slot-eligible fields must be), but non-slot fields may be recorded in other coordinate systems if necessary
29 to avoid information loss (this should, of course, be indicated in the field documentation).
30 """
31 
32 import lsst.pipe.base as pipeBase
33 
34 from .pluginRegistry import PluginRegistry
35 from .baseMeasurement import (BaseMeasurementPluginConfig, BaseMeasurementPlugin,
36  BaseMeasurementConfig, BaseMeasurementTask)
37 from .noiseReplacer import NoiseReplacer, DummyNoiseReplacer
38 
39 __all__ = ("SingleFramePluginConfig", "SingleFramePlugin",
40  "SingleFrameMeasurementConfig", "SingleFrameMeasurementTask")
41 
42 
44  """!
45  Base class for configs of single-frame plugin algorithms.
46  """
47  pass
48 
49 
51  """!
52  Base class for single-frame plugin algorithms.
53 
54  New Plugins can be created in Python by inheriting directly from this class
55  and implementing measure(), fail() (from BasePlugin), and optionally __init__
56  and measureN(). Plugins can also be defined in C++ via the WrappedSingleFramePlugin
57  class.
58  """
59 
60  # All subclasses of SingleFramePlugin should be registered here
61  registry = PluginRegistry(SingleFramePluginConfig)
62  ConfigClass = SingleFramePluginConfig
63 
64  def __init__(self, config, name, schema, metadata, logName=None, **kwds):
65  """!
66  Initialize the measurement object.
67 
68  @param[in] config An instance of this class's ConfigClass.
69  @param[in] name The string the plugin was registered with.
70  @param[in,out] schema The Source schema. New fields should be added here to
71  hold measurements produced by this plugin.
72  @param[in] metadata Plugin metadata that will be attached to the output catalog
73  @param[in] logName May include logName to use for this plugin
74  """
75  BaseMeasurementPlugin.__init__(self, config, name, logName=logName)
76 
77  def measure(self, measRecord, exposure):
78  """!
79  Measure the properties of a source on a single image (single-epoch image or coadd).
80 
81  @param[in,out] measRecord lsst.afw.table.SourceRecord to be filled with outputs,
82  and from which previously-measured quantities can be
83  retreived.
84 
85  @param[in] exposure lsst.afw.image.ExposureF, containing the pixel data to
86  be measured and the associated Psf, Wcs, etc. All
87  other sources in the image will have been replaced by
88  noise according to deblender outputs.
89 
90  """
91  raise NotImplementedError()
92 
93  def measureN(self, measCat, exposure):
94  """!
95  Measure the properties of a group of blended sources on a single image
96  (single-epoch image or coadd).
97 
98  @param[in,out] measCat lsst.afw.table.SourceCatalog to be filled with outputs,
99  and from which previously-measured quantities can be
100  retrieved, containing only the sources that should be
101  measured together in this call.
102 
103  @param[in] exposure lsst.afw.image.ExposureF, containing the pixel data to
104  be measured and the associated Psf, Wcs, etc. Sources
105  not in the blended hierarchy to be measured will have
106  been replaced with noise using deblender outputs.
107 
108  Derived classes that do not implement measureN() should just inherit this
109  disabled version. Derived classes that do implement measureN() should additionally
110  add a bool doMeasureN config field to their config class to signal that measureN-mode
111  is available.
112  """
113  raise NotImplementedError()
114 
115 
117  """!
118  Config class for single frame measurement driver task.
119  """
120 
121  plugins = SingleFramePlugin.registry.makeField(
122  multi=True,
123  default=["base_PixelFlags",
124  "base_SdssCentroid",
125  "base_NaiveCentroid",
126  "base_SdssShape",
127  "base_GaussianFlux",
128  "base_PsfFlux",
129  "base_CircularApertureFlux",
130  "base_SkyCoord",
131  "base_Variance",
132  "base_Blendedness",
133  ],
134  doc="Plugins to be run and their configuration"
135  )
136  algorithms = property(lambda self: self.plugins, doc="backwards-compatibility alias for plugins")
137  undeblended = SingleFramePlugin.registry.makeField(
138  multi=True,
139  default=[],
140  doc="Plugins to run on undeblended image"
141  )
142 
143 
149 
150 
152  """!
153  \anchor SingleFrameMeasurementTask_
154 
155  \brief A subtask for measuring the properties of sources on a single exposure.
156 
157  The task is configured with a list of "plugins": each plugin defines the values it
158  measures (i.e. the columns in a table it will fill) and conducts that measurement
159  on each detected source (see SingleFramePlugin). The job of the
160  measurement task is to initialize the set of plugins (which includes setting up the
161  catalog schema) from their configuration, and then invoke each plugin on each
162  source.
163 
164  When run after the deblender (see lsst.meas.deblender.SourceDeblendTask),
165  SingleFrameMeasurementTask also replaces each source's neighbors with noise before
166  measuring each source, utilizing the HeavyFootprints created by the deblender (see
167  NoiseReplacer).
168 
169  SingleFrameMeasurementTask has only two methods: __init__() and run(). For configuration
170  options, see SingleFrameMeasurementConfig.
171 
172  @section meas_base_sfm_Example A complete example of using SingleFrameMeasurementTask
173 
174  The code below is in examples/runSingleFrameTask.py
175 
176  @dontinclude runSingleFrameTask.py
177 
178  See meas_algorithms_detection_Example for more information on SourceDetectionTask.
179 
180  First, import the required tasks (there are some other standard imports;
181  read the file if you're confused):
182 
183  @skip SourceDetectionTask
184  @until SingleFrameMeasurementTask
185 
186  We need to create our tasks before processing any data as the task constructors
187  can add extra columns to the schema. The most important argument we pass these to these
188  is an lsst.afw.table.Schema object, which contains information about the fields (i.e. columns) of the
189  measurement catalog we'll create, including names, types, and additional documentation.
190  Tasks that operate on a catalog are typically passed a Schema upon construction, to which
191  they add the fields they'll fill later when run. We construct a mostly empty Schema that
192  contains just the fields required for a SourceCatalog like this:
193 
194  @skipline schema
195 
196  Now we can configure and create the SourceDetectionTask:
197 
198  @until detectionTask
199 
200  We then move on to configuring the measurement task:
201 
202  @until config
203 
204  While a reasonable set of plugins is configured by default, we'll customize the list.
205  We also need to unset one of the slots at the same time, because we're
206  not running the algorithm that it's set to by default, and that would cause problems later:
207 
208  @until psfFlux
209 
210  Now, finally, we can construct the measurement task:
211 
212  @skipline measureTask
213 
214  After constructing all the tasks, we can inspect the Schema we've created:
215 
216  @skipline print schema
217 
218  All of the fields in the
219  schema can be accessed via the get() method on a record object. See afwTable for more
220  information.
221 
222  We're now ready to process the data (we could loop over multiple exposures/catalogs using the same
223  task objects). First create the output table and process the image to find sources:
224 
225  @skipline afwTable
226  @skip result
227  @until sources
228 
229  Then measure them:
230 
231  @skipline measure
232 
233  We then might plot the results (@em e.g. if you set `--ds9` on the command line)
234 
235  @skip display
236  @until RED
237 
238  and end up with something like
239 
240  @image html runSingleFrameTask-ds9.png
241  """
242 
243  ConfigClass = SingleFrameMeasurementConfig
244 
245  NOISE_SEED_MULTIPLIER = "NOISE_SEED_MULTIPLIER"
246  NOISE_SOURCE = "NOISE_SOURCE"
247  NOISE_OFFSET = "NOISE_OFFSET"
248  NOISE_EXPOSURE_ID = "NOISE_EXPOSURE_ID"
249 
250  def __init__(self, schema, algMetadata=None, **kwds):
251  """!
252  Initialize the task. Set up the execution order of the plugins and initialize
253  the plugins, giving each plugin an opportunity to add its measurement fields to
254  the output schema and to record information in the task metadata.
255 
256  @param[in,out] schema lsst.afw.table.Schema, to be initialized to include the
257  measurement fields from the plugins already
258  @param[in,out] algMetadata lsst.daf.base.PropertyList used to record information about
259  each algorithm. An empty PropertyList will be created if None.
260  @param[in] **kwds Keyword arguments forwarded to lsst.pipe.base.Task.__init__
261  """
262 
263  super(SingleFrameMeasurementTask, self).__init__(algMetadata=algMetadata, **kwds)
264  self.schema = schema
265  self.config.slots.setupSchema(self.schema)
266  self.initializePlugins(schema=self.schema)
267 
268  # Check to see if blendedness is one of the plugins
269  if 'base_Blendedness' in self.plugins:
270  self.doBlendedness = True
271  self.blendPlugin = self.plugins['base_Blendedness']
272  else:
273  self.doBlendedness = False
274 
275  @pipeBase.timeMethod
276  def run(self, measCat, exposure, noiseImage=None, exposureId=None, beginOrder=None, endOrder=None):
277  """!
278  Run single frame measurement over an exposure and source catalog
279 
280  @param[in,out] measCat lsst.afw.table.SourceCatalog to be filled with outputs. Must
281  contain all the SourceRecords to be measured (with Footprints
282  attached), and have a schema that is a superset of self.schema.
283 
284  @param[in] exposure lsst.afw.image.ExposureF, containing the pixel data to
285  be measured and the associated Psf, Wcs, etc.
286  @param[in] noiseImage optional lsst.afw.image.ImageF for test which need to control
287  noiseReplacement
288  @param[in] exposureId optional unique exposureId used to calculate random number
289  generator seed in the NoiseReplacer.
290  @param[in] beginOrder beginning execution order (inclusive): measurements with
291  executionOrder < beginOrder are not executed. None for no limit.
292  @param[in] endOrder ending execution order (exclusive): measurements with
293  executionOrder >= endOrder are not executed. None for no limit.
294  """
295  assert measCat.getSchema().contains(self.schema)
296  footprints = {measRecord.getId(): (measRecord.getParent(), measRecord.getFootprint())
297  for measRecord in measCat}
298 
299  # noiseReplacer is used to fill the footprints with noise and save heavy footprints
300  # of the source pixels so that they can be restored one at a time for measurement.
301  # After the NoiseReplacer is constructed, all pixels in the exposure.getMaskedImage()
302  # which belong to objects in measCat will be replaced with noise
303  if self.config.doReplaceWithNoise:
304  noiseReplacer = NoiseReplacer(self.config.noiseReplacer, exposure, footprints,
305  noiseImage=noiseImage, log=self.log, exposureId=exposureId)
306  algMetadata = measCat.getMetadata()
307  if algMetadata is not None:
308  algMetadata.addInt(self.NOISE_SEED_MULTIPLIER, self.config.noiseReplacer.noiseSeedMultiplier)
309  algMetadata.addString(self.NOISE_SOURCE, self.config.noiseReplacer.noiseSource)
310  algMetadata.addDouble(self.NOISE_OFFSET, self.config.noiseReplacer.noiseOffset)
311  if exposureId is not None:
312  algMetadata.addLong(self.NOISE_EXPOSURE_ID, exposureId)
313  else:
314  noiseReplacer = DummyNoiseReplacer()
315 
316  self.runPlugins(noiseReplacer, measCat, exposure, beginOrder, endOrder)
317 
318  def runPlugins(self, noiseReplacer, measCat, exposure, beginOrder=None, endOrder=None):
319  """Function which calls the defined measument plugins on an exposure
320 
321  Parameters
322  ----------
323  noiseReplacer : lsst.meas.base.NoiseReplacer
324  noiseReplacer to fill sources not being measured with noise.
325 
326  measCat : lsst.afw.table.SourceCatalog
327  SourceCatalog to be filled with outputs. Must contain all the SourceRecords to be measured (with
328  Footprints attached), and have a schema that is a superset of self.schema.
329 
330  exposure : lsst.afw.image.ExposureF
331  Exposure contaning the pixel data to be measured and the associated PSF, WCS, etc.
332 
333  beginOrder : float
334  beginning execution order (inclusive): measurements with executionOrder < beginOrder are not
335  executed. None for no limit.
336 
337  endOrder : float
338  ending execution order (exclusive): measurements with executionOrder >= endOrder are not
339  executed. None for no limit.
340  """
341  # First, create a catalog of all parentless sources
342  # Loop through all the parent sources, first processing the children, then the parent
343  measParentCat = measCat.getChildren(0)
344 
345  nMeasCat = len(measCat)
346  nMeasParentCat = len(measParentCat)
347  self.log.info("Measuring %d source%s (%d parent%s, %d child%s) ",
348  nMeasCat, ("" if nMeasCat == 1 else "s"),
349  nMeasParentCat, ("" if nMeasParentCat == 1 else "s"),
350  nMeasCat - nMeasParentCat, ("" if nMeasCat - nMeasParentCat == 1 else "ren"))
351 
352  for parentIdx, measParentRecord in enumerate(measParentCat):
353  # first get all the children of this parent, insert footprint in turn, and measure
354  measChildCat = measCat.getChildren(measParentRecord.getId())
355  # TODO: skip this loop if there are no plugins configured for single-object mode
356  for measChildRecord in measChildCat:
357  noiseReplacer.insertSource(measChildRecord.getId())
358  self.callMeasure(measChildRecord, exposure, beginOrder=beginOrder, endOrder=endOrder)
359 
360  if self.doBlendedness:
361  self.blendPlugin.cpp.measureChildPixels(exposure.getMaskedImage(), measChildRecord)
362 
363  noiseReplacer.removeSource(measChildRecord.getId())
364 
365  # Then insert the parent footprint, and measure that
366  noiseReplacer.insertSource(measParentRecord.getId())
367  self.callMeasure(measParentRecord, exposure, beginOrder=beginOrder, endOrder=endOrder)
368 
369  if self.doBlendedness:
370  self.blendPlugin.cpp.measureChildPixels(exposure.getMaskedImage(), measParentRecord)
371 
372  # Finally, process both the parent and the child set through measureN
373  self.callMeasureN(measParentCat[parentIdx:parentIdx+1], exposure,
374  beginOrder=beginOrder, endOrder=endOrder)
375  self.callMeasureN(measChildCat, exposure, beginOrder=beginOrder, endOrder=endOrder)
376  noiseReplacer.removeSource(measParentRecord.getId())
377  # when done, restore the exposure to its original state
378  noiseReplacer.end()
379 
380  # Undeblended plugins only fire if we're running everything
381  if endOrder is None:
382  for source in measCat:
383  for plugin in self.undeblendedPlugins.iter():
384  self.doMeasurement(plugin, source, exposure)
385 
386  # Now we loop over all of the sources one more time to compute the blendedness metrics
387  if self.doBlendedness:
388  for source in measCat:
389  self.blendPlugin.cpp.measureParentPixels(exposure.getMaskedImage(), source)
390 
391  def measure(self, measCat, exposure):
392  """!
393  Backwards-compatibility alias for run()
394  """
395  self.run(measCat, exposure)
Base config class for all measurement plugins.
def callMeasure(self, measRecord, args, kwds)
Call the measure() method on all plugins, handling exceptions in a consistent way.
def measure(self, measRecord, exposure)
Measure the properties of a source on a single image (single-epoch image or coadd).
Definition: sfm.py:77
A subtask for measuring the properties of sources on a single exposure.
Definition: sfm.py:151
def measureN(self, measCat, exposure)
Measure the properties of a group of blended sources on a single image (single-epoch image or coadd)...
Definition: sfm.py:93
def measure(self, measCat, exposure)
Backwards-compatibility alias for run()
Definition: sfm.py:391
def runPlugins(self, noiseReplacer, measCat, exposure, beginOrder=None, endOrder=None)
Definition: sfm.py:318
def callMeasureN(self, measCat, args, kwds)
Call the measureN() method on all plugins, handling exceptions in a consistent way.
Base class for configs of single-frame plugin algorithms.
Definition: sfm.py:43
Base class for single-frame plugin algorithms.
Definition: sfm.py:50
def __init__(self, config, name, schema, metadata, logName=None, kwds)
Initialize the measurement object.
Definition: sfm.py:64
def run(self, measCat, exposure, noiseImage=None, exposureId=None, beginOrder=None, endOrder=None)
Run single frame measurement over an exposure and source catalog.
Definition: sfm.py:276
Config class for single frame measurement driver task.
Definition: sfm.py:116
Base class for plugin registries.
A do-nothing standin for NoiseReplacer, used when we want to disable NoiseReplacer.
def __init__(self, schema, algMetadata=None, kwds)
Initialize the task.
Definition: sfm.py:250
Class that handles replacing sources with noise during measurement.
Ultimate base class for all measurement tasks.
Base config class for all measurement driver tasks.
def doMeasurement(self, plugin, measRecord, args, kwds)
Call the measure() method on the nominated plugin, handling exceptions in a consistent way...