lsst.meas.algorithms  15.0-8-g306a5613+1
dynamicDetection.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 __all__ = ["DynamicDetectionConfig", "DynamicDetectionTask"]
4 
5 import numpy as np
6 
7 from lsst.pex.config import Field, ConfigurableField
8 from lsst.pipe.base import Task, Struct
9 
10 from .detection import SourceDetectionConfig, SourceDetectionTask
11 from .skyObjects import SkyObjectsTask
12 
13 from lsst.afw.detection import FootprintSet
14 from lsst.afw.table import SourceCatalog, SourceTable, IdFactory
15 from lsst.meas.base import ForcedMeasurementTask
16 
17 import lsst.afw.image
18 import lsst.afw.math
19 
20 
21 class DynamicDetectionConfig(SourceDetectionConfig):
22  """Configuration for DynamicDetectionTask"""
23  prelimThresholdFactor = Field(dtype=float, default=0.5,
24  doc="Fraction of the threshold to use for first pass (to find sky objects)")
25  skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects")
26  doBackgroundTweak = Field(dtype=bool, default=True,
27  doc="Tweak background level so median PSF flux of sky objects is zero?")
28  minNumSources = Field(dtype=int, default=10,
29  doc="Minimum number of sky sources in statistical sample; "
30  "if below this number, we refuse to modify the threshold.")
31 
32  def setDefaults(self):
33  SourceDetectionConfig.setDefaults(self)
34  self.skyObjects.nSources = 1000 # For good statistics
35 
36 
37 class DynamicDetectionTask(SourceDetectionTask):
38  """Detection of sources on an image with a dynamic threshold
39 
40  We first detect sources using a lower threshold than normal (see config
41  parameter ``prelimThresholdFactor``) in order to identify good sky regions
42  (configurable ``skyObjects``). Then we perform forced PSF photometry on
43  those sky regions. Using those PSF flux measurements and estimated errors,
44  we set the threshold so that the stdev of the measurements matches the
45  median estimated error.
46  """
47  ConfigClass = DynamicDetectionConfig
48  _DefaultName = "dynamicDetection"
49 
50  def __init__(self, *args, **kwargs):
51  """Constructor
52 
53  Besides the usual initialisation of configurables, we also set up
54  the forced measurement which is deliberately not represented in
55  this Task's configuration parameters because we're using it as part
56  of the algorithm and we don't want to allow it to be modified.
57  """
58  SourceDetectionTask.__init__(self, *args, **kwargs)
59  self.makeSubtask("skyObjects")
60 
61  # Set up forced measurement.
62  config = ForcedMeasurementTask.ConfigClass()
63  config.plugins.names = ['base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground']
64  # We'll need the "centroid" and "psfFlux" slots
65  for slot in ("shape", "psfShape", "apFlux", "modelFlux", "instFlux", "calibFlux"):
66  setattr(config.slots, slot, None)
67  config.copyColumns = {}
68  self.skySchema = SourceTable.makeMinimalSchema()
69  self.skyMeasurement = ForcedMeasurementTask(config=config, name="skyMeasurement", parentTask=self,
70  refSchema=self.skySchema)
71 
72  def calculateThreshold(self, exposure, seed, sigma=None):
73  """Calculate new threshold
74 
75  This is the main functional addition to the vanilla
76  `SourceDetectionTask`.
77 
78  We identify sky objects and perform forced PSF photometry on
79  them. Using those PSF flux measurements and estimated errors,
80  we set the threshold so that the stdev of the measurements
81  matches the median estimated error.
82 
83  Parameters
84  ----------
85  exposure : `lsst.afw.image.Exposure`
86  Exposure on which we're detecting sources.
87  seed : `int`
88  RNG seed to use for finding sky objects.
89  sigma : `float`, optional
90  Gaussian sigma of smoothing kernel; if not provided,
91  will be deduced from the exposure's PSF.
92 
93  Returns
94  -------
95  result : `lsst.pipe.base.Struct`
96  Result struct with components:
97 
98  - ``multiplicative``: multiplicative factor to be applied to the
99  configured detection threshold (`float`).
100  - ``additive``: additive factor to be applied to the background
101  level (`float`).
102  """
103  # Make a catalog of sky objects
104  fp = self.skyObjects.run(exposure.maskedImage.mask, seed)
105  skyFootprints = FootprintSet(exposure.getBBox())
106  skyFootprints.setFootprints(fp)
107  table = SourceTable.make(self.skyMeasurement.schema)
108  catalog = SourceCatalog(table)
109  table.preallocate(len(skyFootprints.getFootprints()))
110  skyFootprints.makeSources(catalog)
111  key = catalog.getCentroidKey()
112  for source in catalog:
113  peaks = source.getFootprint().getPeaks()
114  assert len(peaks) == 1
115  source.set(key, peaks[0].getF())
116  source.updateCoord(exposure.getWcs())
117 
118  # Forced photometry on sky objects
119  self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs())
120 
121  # Calculate new threshold
122  fluxes = catalog["base_PsfFlux_flux"]
123  area = catalog["base_PsfFlux_area"]
124  bg = catalog["base_LocalBackground_flux"]
125 
126  good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] &
127  np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg))
128 
129  if good.sum() < self.config.minNumSources:
130  self.log.warn("Insufficient good flux measurements (%d < %d) for dynamic threshold calculation",
131  good.sum(), self.config.minNumSources)
132  return Struct(multiplicative=1.0, additive=0.0)
133 
134  bgMedian = np.median((fluxes/area)[good])
135 
136  lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0])
137  stdevMeas = 0.741*(uq - lq)
138  medianError = np.median(catalog["base_PsfFlux_fluxSigma"][good])
139  return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian)
140 
141  def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None):
142  """Detect footprints with a dynamic threshold
143 
144  This varies from the vanilla ``detectFootprints`` method because we
145  do detection twice: one with a low threshold so that we can find
146  sky uncontaminated by objects, then one more with the new calculated
147  threshold.
148 
149  Parameters
150  ----------
151  exposure : `lsst.afw.image.Exposure`
152  Exposure to process; DETECTED{,_NEGATIVE} mask plane will be
153  set in-place.
154  doSmooth : `bool`, optional
155  If True, smooth the image before detection using a Gaussian
156  of width ``sigma``.
157  sigma : `float`, optional
158  Gaussian Sigma of PSF (pixels); used for smoothing and to grow
159  detections; if `None` then measure the sigma of the PSF of the
160  ``exposure``.
161  clearMask : `bool`, optional
162  Clear both DETECTED and DETECTED_NEGATIVE planes before running
163  detection.
164  expId : `int`, optional
165  Exposure identifier, used as a seed for the random number
166  generator. If absent, the seed will be the sum of the image.
167 
168  Return Struct contents
169  ----------------------
170  positive : `lsst.afw.detection.FootprintSet`
171  Positive polarity footprints (may be `None`)
172  negative : `lsst.afw.detection.FootprintSet`
173  Negative polarity footprints (may be `None`)
174  numPos : `int`
175  Number of footprints in positive or 0 if detection polarity was
176  negative.
177  numNeg : `int`
178  Number of footprints in negative or 0 if detection polarity was
179  positive.
180  background : `lsst.afw.math.BackgroundList`
181  Re-estimated background. `None` if
182  ``reEstimateBackground==False``.
183  factor : `float`
184  Multiplication factor applied to the configured detection
185  threshold.
186  prelim : `lsst.pipe.base.Struct`
187  Results from preliminary detection pass.
188  """
189  maskedImage = exposure.maskedImage
190 
191  if clearMask:
192  self.clearMask(maskedImage.mask)
193  else:
194  oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(["DETECTED",
195  "DETECTED_NEGATIVE"])
196 
197  with self.tempWideBackgroundContext(exposure):
198  # Could potentially smooth with a wider kernel than the PSF in order to better pick up the
199  # wings of stars and galaxies, but for now sticking with the PSF as that's more simple.
200  psf = self.getPsf(exposure, sigma=sigma)
201  convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth)
202  middle = convolveResults.middle
203  sigma = convolveResults.sigma
204  prelim = self.applyThreshold(middle, maskedImage.getBBox(), self.config.prelimThresholdFactor)
205  self.finalizeFootprints(maskedImage.mask, prelim, sigma, self.config.prelimThresholdFactor)
206 
207  # Calculate the proper threshold
208  # seed needs to fit in a C++ 'int' so pybind doesn't choke on it
209  seed = (expId if expId is not None else int(maskedImage.image.array.sum())) % (2**31 - 1)
210  threshResults = self.calculateThreshold(exposure, seed, sigma=sigma)
211  factor = threshResults.multiplicative
212  self.log.info("Modifying configured detection threshold by factor %f to %f",
213  factor, factor*self.config.thresholdValue)
214  if self.config.doBackgroundTweak:
215  self.tweakBackground(exposure, threshResults.additive)
216 
217  # Blow away preliminary (low threshold) detection mask
218  self.clearMask(maskedImage.mask)
219  if not clearMask:
220  maskedImage.mask.array |= oldDetected
221 
222  # Rinse and repeat thresholding with new calculated threshold
223  results = self.applyThreshold(middle, maskedImage.getBBox(), factor)
224  results.prelim = prelim
225  results.background = lsst.afw.math.BackgroundList()
226  if self.config.doTempLocalBackground:
227  self.applyTempLocalBackground(exposure, middle, results)
228  self.finalizeFootprints(maskedImage.mask, results, sigma, factor)
229 
230  self.clearUnwantedResults(maskedImage.mask, results)
231 
232  if self.config.reEstimateBackground:
233  self.reEstimateBackground(maskedImage, results.background)
234 
235  self.display(exposure, results, middle)
236 
237  if self.config.doBackgroundTweak:
238  # Re-do the background tweak after any temporary backgrounds have been restored
239  #
240  # But we want to keep any large-scale background (e.g., scattered light from bright stars)
241  # from being selected for sky objects in the calculation, so do another detection pass without
242  # either the local or wide temporary background subtraction; the DETECTED pixels will mark
243  # the area to ignore.
244  originalMask = maskedImage.mask.array.copy()
245  try:
246  self.clearMask(exposure.mask)
247  convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth)
248  tweakDetResults = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor)
249  self.finalizeFootprints(maskedImage.mask, tweakDetResults, sigma, factor)
250  bgLevel = self.calculateThreshold(exposure, seed, sigma=sigma).additive
251  finally:
252  maskedImage.mask.array[:] = originalMask
253  self.tweakBackground(exposure, bgLevel, results.background)
254 
255  return results
256 
257  def tweakBackground(self, exposure, bgLevel, bgList=None):
258  """Modify the background by a constant value
259 
260  Parameters
261  ----------
262  exposure : `lsst.afw.image.Exposure`
263  Exposure for which to tweak background.
264  bgLevel : `float`
265  Background level to remove
266  bgList : `lsst.afw.math.BackgroundList`, optional
267  List of backgrounds to append to.
268 
269  Returns
270  -------
271  bg : `lsst.afw.math.BackgroundMI`
272  Constant background model.
273  """
274  self.log.info("Tweaking background by %f to match sky photometry", bgLevel)
275  exposure.image -= bgLevel
276  bgStats = lsst.afw.image.MaskedImageF(1, 1)
277  bgStats.set(0, 0, (bgLevel, 0, bgLevel))
278  bg = lsst.afw.math.BackgroundMI(exposure.getBBox(), bgStats)
279  bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER,
280  lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0, False)
281  if bgList is not None:
282  bgList.append(bgData)
283  return bg
def tweakBackground(self, exposure, bgLevel, bgList=None)
def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None)
def calculateThreshold(self, exposure, seed, sigma=None)