lsst.meas.base  14.0-9-g876143c
sfm.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 LSST Corporation.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 """Base classes for single-frame measurement plugins and the driver task for these.
24 
25 In single-frame measurement, we assume that detection and probably deblending have already been run on
26 the same frame, so a SourceCatalog has already been created with Footprints (which may be HeavyFootprints).
27 Measurements are generally recorded in the coordinate system of the image being measured (and all
28 slot-eligible fields must be), but non-slot fields may be recorded in other coordinate systems if necessary
29 to avoid information loss (this should, of course, be indicated in the field documentation).
30 """
31 
32 import lsst.pipe.base as pipeBase
33 
34 from .pluginRegistry import PluginRegistry
35 from .baseMeasurement import (BaseMeasurementPluginConfig, BaseMeasurementPlugin,
36  BaseMeasurementConfig, BaseMeasurementTask)
37 from .noiseReplacer import NoiseReplacer, DummyNoiseReplacer
38 
39 __all__ = ("SingleFramePluginConfig", "SingleFramePlugin",
40  "SingleFrameMeasurementConfig", "SingleFrameMeasurementTask")
41 
42 
44  """!
45  Base class for configs of single-frame plugin algorithms.
46  """
47  pass
48 
49 
51  """!
52  Base class for single-frame plugin algorithms.
53 
54  New Plugins can be created in Python by inheriting directly from this class
55  and implementing measure(), fail() (from BasePlugin), and optionally __init__
56  and measureN(). Plugins can also be defined in C++ via the WrappedSingleFramePlugin
57  class.
58  """
59 
60  # All subclasses of SingleFramePlugin should be registered here
61  registry = PluginRegistry(SingleFramePluginConfig)
62  ConfigClass = SingleFramePluginConfig
63 
64  def __init__(self, config, name, schema, metadata, logName=None, **kwds):
65  """!
66  Initialize the measurement object.
67 
68  @param[in] config An instance of this class's ConfigClass.
69  @param[in] name The string the plugin was registered with.
70  @param[in,out] schema The Source schema. New fields should be added here to
71  hold measurements produced by this plugin.
72  @param[in] metadata Plugin metadata that will be attached to the output catalog
73  @param[in] logName May include logName to use for this plugin
74  """
75  BaseMeasurementPlugin.__init__(self, config, name, logName=logName)
76 
77  def measure(self, measRecord, exposure):
78  """!
79  Measure the properties of a source on a single image (single-epoch image or coadd).
80 
81  @param[in,out] measRecord lsst.afw.table.SourceRecord to be filled with outputs,
82  and from which previously-measured quantities can be
83  retreived.
84 
85  @param[in] exposure lsst.afw.image.ExposureF, containing the pixel data to
86  be measured and the associated Psf, Wcs, etc. All
87  other sources in the image will have been replaced by
88  noise according to deblender outputs.
89 
90  """
91  raise NotImplementedError()
92 
93  def measureN(self, measCat, exposure):
94  """!
95  Measure the properties of a group of blended sources on a single image
96  (single-epoch image or coadd).
97 
98  @param[in,out] measCat lsst.afw.table.SourceCatalog to be filled with outputs,
99  and from which previously-measured quantities can be
100  retrieved, containing only the sources that should be
101  measured together in this call.
102 
103  @param[in] exposure lsst.afw.image.ExposureF, containing the pixel data to
104  be measured and the associated Psf, Wcs, etc. Sources
105  not in the blended hierarchy to be measured will have
106  been replaced with noise using deblender outputs.
107 
108  Derived classes that do not implement measureN() should just inherit this
109  disabled version. Derived classes that do implement measureN() should additionally
110  add a bool doMeasureN config field to their config class to signal that measureN-mode
111  is available.
112  """
113  raise NotImplementedError()
114 
115 
117  """!
118  Config class for single frame measurement driver task.
119  """
120 
121  plugins = SingleFramePlugin.registry.makeField(
122  multi=True,
123  default=["base_PixelFlags",
124  "base_SdssCentroid",
125  "base_GaussianCentroid",
126  "base_NaiveCentroid",
127  "base_SdssShape",
128  "base_GaussianFlux",
129  "base_PsfFlux",
130  "base_CircularApertureFlux",
131  "base_SkyCoord",
132  "base_Variance",
133  "base_Blendedness",
134  ],
135  doc="Plugins to be run and their configuration"
136  )
137  algorithms = property(lambda self: self.plugins, doc="backwards-compatibility alias for plugins")
138  undeblended = SingleFramePlugin.registry.makeField(
139  multi=True,
140  default=[],
141  doc="Plugins to run on undeblended image"
142  )
143 
144 
150 
151 
153  """!
154  \anchor SingleFrameMeasurementTask_
155 
156  \brief A subtask for measuring the properties of sources on a single exposure.
157 
158  The task is configured with a list of "plugins": each plugin defines the values it
159  measures (i.e. the columns in a table it will fill) and conducts that measurement
160  on each detected source (see SingleFramePlugin). The job of the
161  measurement task is to initialize the set of plugins (which includes setting up the
162  catalog schema) from their configuration, and then invoke each plugin on each
163  source.
164 
165  When run after the deblender (see lsst.meas.deblender.SourceDeblendTask),
166  SingleFrameMeasurementTask also replaces each source's neighbors with noise before
167  measuring each source, utilizing the HeavyFootprints created by the deblender (see
168  NoiseReplacer).
169 
170  SingleFrameMeasurementTask has only two methods: __init__() and run(). For configuration
171  options, see SingleFrameMeasurementConfig.
172 
173  @section meas_base_sfm_Example A complete example of using SingleFrameMeasurementTask
174 
175  The code below is in examples/runSingleFrameTask.py
176 
177  @dontinclude runSingleFrameTask.py
178 
179  See meas_algorithms_detection_Example for more information on SourceDetectionTask.
180 
181  First, import the required tasks (there are some other standard imports;
182  read the file if you're confused):
183 
184  @skip SourceDetectionTask
185  @until SingleFrameMeasurementTask
186 
187  We need to create our tasks before processing any data as the task constructors
188  can add extra columns to the schema. The most important argument we pass these to these
189  is an lsst.afw.table.Schema object, which contains information about the fields (i.e. columns) of the
190  measurement catalog we'll create, including names, types, and additional documentation.
191  Tasks that operate on a catalog are typically passed a Schema upon construction, to which
192  they add the fields they'll fill later when run. We construct a mostly empty Schema that
193  contains just the fields required for a SourceCatalog like this:
194 
195  @skipline schema
196 
197  Now we can configure and create the SourceDetectionTask:
198 
199  @until detectionTask
200 
201  We then move on to configuring the measurement task:
202 
203  @until config
204 
205  While a reasonable set of plugins is configured by default, we'll customize the list.
206  We also need to unset one of the slots at the same time, because we're
207  not running the algorithm that it's set to by default, and that would cause problems later:
208 
209  @until psfFlux
210 
211  Now, finally, we can construct the measurement task:
212 
213  @skipline measureTask
214 
215  After constructing all the tasks, we can inspect the Schema we've created:
216 
217  @skipline print schema
218 
219  All of the fields in the
220  schema can be accessed via the get() method on a record object. See afwTable for more
221  information.
222 
223  We're now ready to process the data (we could loop over multiple exposures/catalogs using the same
224  task objects). First create the output table and process the image to find sources:
225 
226  @skipline afwTable
227  @skip result
228  @until sources
229 
230  Then measure them:
231 
232  @skipline measure
233 
234  We then might plot the results (@em e.g. if you set `--ds9` on the command line)
235 
236  @skip display
237  @until RED
238 
239  and end up with something like
240 
241  @image html runSingleFrameTask-ds9.png
242  """
243 
244  ConfigClass = SingleFrameMeasurementConfig
245 
246  NOISE_SEED_MULTIPLIER = "NOISE_SEED_MULTIPLIER"
247  NOISE_SOURCE = "NOISE_SOURCE"
248  NOISE_OFFSET = "NOISE_OFFSET"
249  NOISE_EXPOSURE_ID = "NOISE_EXPOSURE_ID"
250 
251  def __init__(self, schema, algMetadata=None, **kwds):
252  """!
253  Initialize the task. Set up the execution order of the plugins and initialize
254  the plugins, giving each plugin an opportunity to add its measurement fields to
255  the output schema and to record information in the task metadata.
256 
257  @param[in,out] schema lsst.afw.table.Schema, to be initialized to include the
258  measurement fields from the plugins already
259  @param[in,out] algMetadata lsst.daf.base.PropertyList used to record information about
260  each algorithm. An empty PropertyList will be created if None.
261  @param[in] **kwds Keyword arguments forwarded to lsst.pipe.base.Task.__init__
262  """
263 
264  super(SingleFrameMeasurementTask, self).__init__(algMetadata=algMetadata, **kwds)
265  self.schema = schema
266  self.config.slots.setupSchema(self.schema)
267  self.initializePlugins(schema=self.schema)
268 
269  # Check to see if blendedness is one of the plugins
270  if 'base_Blendedness' in self.plugins:
271  self.doBlendedness = True
272  self.blendPlugin = self.plugins['base_Blendedness']
273  else:
274  self.doBlendedness = False
275 
276  @pipeBase.timeMethod
277  def run(self, measCat, exposure, noiseImage=None, exposureId=None, beginOrder=None, endOrder=None):
278  """!
279  Run single frame measurement over an exposure and source catalog
280 
281  @param[in,out] measCat lsst.afw.table.SourceCatalog to be filled with outputs. Must
282  contain all the SourceRecords to be measured (with Footprints
283  attached), and have a schema that is a superset of self.schema.
284 
285  @param[in] exposure lsst.afw.image.ExposureF, containing the pixel data to
286  be measured and the associated Psf, Wcs, etc.
287  @param[in] noiseImage optional lsst.afw.image.ImageF for test which need to control
288  noiseReplacement
289  @param[in] exposureId optional unique exposureId used to calculate random number
290  generator seed in the NoiseReplacer.
291  @param[in] beginOrder beginning execution order (inclusive): measurements with
292  executionOrder < beginOrder are not executed. None for no limit.
293  @param[in] endOrder ending execution order (exclusive): measurements with
294  executionOrder >= endOrder are not executed. None for no limit.
295  """
296  assert measCat.getSchema().contains(self.schema)
297  footprints = {measRecord.getId(): (measRecord.getParent(), measRecord.getFootprint())
298  for measRecord in measCat}
299 
300  # noiseReplacer is used to fill the footprints with noise and save heavy footprints
301  # of the source pixels so that they can be restored one at a time for measurement.
302  # After the NoiseReplacer is constructed, all pixels in the exposure.getMaskedImage()
303  # which belong to objects in measCat will be replaced with noise
304  if self.config.doReplaceWithNoise:
305  noiseReplacer = NoiseReplacer(self.config.noiseReplacer, exposure, footprints,
306  noiseImage=noiseImage, log=self.log, exposureId=exposureId)
307  algMetadata = measCat.getMetadata()
308  if algMetadata is not None:
309  algMetadata.addInt(self.NOISE_SEED_MULTIPLIER, self.config.noiseReplacer.noiseSeedMultiplier)
310  algMetadata.addString(self.NOISE_SOURCE, self.config.noiseReplacer.noiseSource)
311  algMetadata.addDouble(self.NOISE_OFFSET, self.config.noiseReplacer.noiseOffset)
312  if exposureId is not None:
313  algMetadata.addLong(self.NOISE_EXPOSURE_ID, exposureId)
314  else:
315  noiseReplacer = DummyNoiseReplacer()
316 
317  self.runPlugins(noiseReplacer, measCat, exposure, beginOrder, endOrder)
318 
319  def runPlugins(self, noiseReplacer, measCat, exposure, beginOrder=None, endOrder=None):
320  """Function which calls the defined measument plugins on an exposure
321 
322  Parameters
323  ----------
324  noiseReplacer : lsst.meas.base.NoiseReplacer
325  noiseReplacer to fill sources not being measured with noise.
326 
327  measCat : lsst.afw.table.SourceCatalog
328  SourceCatalog to be filled with outputs. Must contain all the SourceRecords to be measured (with
329  Footprints attached), and have a schema that is a superset of self.schema.
330 
331  exposure : lsst.afw.image.ExposureF
332  Exposure contaning the pixel data to be measured and the associated PSF, WCS, etc.
333 
334  beginOrder : float
335  beginning execution order (inclusive): measurements with executionOrder < beginOrder are not
336  executed. None for no limit.
337 
338  endOrder : float
339  ending execution order (exclusive): measurements with executionOrder >= endOrder are not
340  executed. None for no limit.
341  """
342  # First, create a catalog of all parentless sources
343  # Loop through all the parent sources, first processing the children, then the parent
344  measParentCat = measCat.getChildren(0)
345 
346  nMeasCat = len(measCat)
347  nMeasParentCat = len(measParentCat)
348  self.log.info("Measuring %d source%s (%d parent%s, %d child%s) ",
349  nMeasCat, ("" if nMeasCat == 1 else "s"),
350  nMeasParentCat, ("" if nMeasParentCat == 1 else "s"),
351  nMeasCat - nMeasParentCat, ("" if nMeasCat - nMeasParentCat == 1 else "ren"))
352 
353  for parentIdx, measParentRecord in enumerate(measParentCat):
354  # first get all the children of this parent, insert footprint in turn, and measure
355  measChildCat = measCat.getChildren(measParentRecord.getId())
356  # TODO: skip this loop if there are no plugins configured for single-object mode
357  for measChildRecord in measChildCat:
358  noiseReplacer.insertSource(measChildRecord.getId())
359  self.callMeasure(measChildRecord, exposure, beginOrder=beginOrder, endOrder=endOrder)
360 
361  if self.doBlendedness:
362  self.blendPlugin.cpp.measureChildPixels(exposure.getMaskedImage(), measChildRecord)
363 
364  noiseReplacer.removeSource(measChildRecord.getId())
365 
366  # Then insert the parent footprint, and measure that
367  noiseReplacer.insertSource(measParentRecord.getId())
368  self.callMeasure(measParentRecord, exposure, beginOrder=beginOrder, endOrder=endOrder)
369 
370  if self.doBlendedness:
371  self.blendPlugin.cpp.measureChildPixels(exposure.getMaskedImage(), measParentRecord)
372 
373  # Finally, process both the parent and the child set through measureN
374  self.callMeasureN(measParentCat[parentIdx:parentIdx+1], exposure,
375  beginOrder=beginOrder, endOrder=endOrder)
376  self.callMeasureN(measChildCat, exposure, beginOrder=beginOrder, endOrder=endOrder)
377  noiseReplacer.removeSource(measParentRecord.getId())
378  # when done, restore the exposure to its original state
379  noiseReplacer.end()
380 
381  # Undeblended plugins only fire if we're running everything
382  if endOrder is None:
383  for source in measCat:
384  for plugin in self.undeblendedPlugins.iter():
385  self.doMeasurement(plugin, source, exposure)
386 
387  # Now we loop over all of the sources one more time to compute the blendedness metrics
388  if self.doBlendedness:
389  for source in measCat:
390  self.blendPlugin.cpp.measureParentPixels(exposure.getMaskedImage(), source)
391 
392  def measure(self, measCat, exposure):
393  """!
394  Backwards-compatibility alias for run()
395  """
396  self.run(measCat, exposure)
int iter
Base config class for all measurement plugins.
def callMeasure(self, measRecord, args, kwds)
Call the measure() method on all plugins, handling exceptions in a consistent way.
def measure(self, measRecord, exposure)
Measure the properties of a source on a single image (single-epoch image or coadd).
Definition: sfm.py:77
A subtask for measuring the properties of sources on a single exposure.
Definition: sfm.py:152
def measureN(self, measCat, exposure)
Measure the properties of a group of blended sources on a single image (single-epoch image or coadd)...
Definition: sfm.py:93
def measure(self, measCat, exposure)
Backwards-compatibility alias for run()
Definition: sfm.py:392
def runPlugins(self, noiseReplacer, measCat, exposure, beginOrder=None, endOrder=None)
Definition: sfm.py:319
def callMeasureN(self, measCat, args, kwds)
Call the measureN() method on all plugins, handling exceptions in a consistent way.
Base class for configs of single-frame plugin algorithms.
Definition: sfm.py:43
Base class for single-frame plugin algorithms.
Definition: sfm.py:50
def __init__(self, config, name, schema, metadata, logName=None, kwds)
Initialize the measurement object.
Definition: sfm.py:64
def run(self, measCat, exposure, noiseImage=None, exposureId=None, beginOrder=None, endOrder=None)
Run single frame measurement over an exposure and source catalog.
Definition: sfm.py:277
Config class for single frame measurement driver task.
Definition: sfm.py:116
Base class for plugin registries.
A do-nothing standin for NoiseReplacer, used when we want to disable NoiseReplacer.
def __init__(self, schema, algMetadata=None, kwds)
Initialize the task.
Definition: sfm.py:251
Class that handles replacing sources with noise during measurement.
Ultimate base class for all measurement tasks.
Base config class for all measurement driver tasks.
def doMeasurement(self, plugin, measRecord, args, kwds)
Call the measure() method on the nominated plugin, handling exceptions in a consistent way...