lsst.meas.algorithms ge25b0cbbcb+9c410686f1
Loading...
Searching...
No Matches
dynamicDetection.py
Go to the documentation of this file.
2__all__ = ["DynamicDetectionConfig", "DynamicDetectionTask"]
3
4import numpy as np
5
6from lsst.pex.config import Field, ConfigurableField
7from lsst.pipe.base import Struct, NoWorkFound
8
9from .detection import SourceDetectionConfig, SourceDetectionTask
10from .skyObjects import SkyObjectsTask
11
12from lsst.afw.detection import FootprintSet
13from lsst.afw.geom import makeCdMatrix, makeSkyWcs
14from lsst.afw.table import SourceCatalog, SourceTable
15from lsst.meas.base import ForcedMeasurementTask
16
17import lsst.afw.image
18import lsst.afw.math
19import lsst.geom as geom
20
21
23 """Configuration for DynamicDetectionTask
24 """
25 prelimThresholdFactor = Field(dtype=float, default=0.5,
26 doc="Factor by which to multiply the main detection threshold "
27 "(thresholdValue) to use for first pass (to find sky objects).")
28 prelimNegMultiplier = Field(dtype=float, default=2.5,
29 doc="Multiplier for the negative (relative to positive) polarity "
30 "detections threshold to use for first pass (to find sky objects).")
31 skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects.")
32 doBackgroundTweak = Field(dtype=bool, default=True,
33 doc="Tweak background level so median PSF flux of sky objects is zero?")
34 minFractionSources = Field(dtype=float, default=0.02,
35 doc="Minimum fraction of the requested number of sky sources for dynamic "
36 "detection to be considered a success. If the number of good sky sources "
37 "identified falls below this threshold, a NoWorkFound error is raised so "
38 "that this dataId is no longer considered in downstream processing.")
39 doBrightPrelimDetection = Field(dtype=bool, default=True,
40 doc="Do initial bright detection pass where footprints are grown "
41 "by brightGrowFactor?")
42 brightMultiplier = Field(dtype=float, default=2000.0,
43 doc="Multiplier to apply to the prelimThresholdFactor for the "
44 "\"bright\" detections stage (want this to be large to only "
45 "detect the brightest sources).")
46 brightNegFactor = Field(dtype=float, default=2.2,
47 doc="Factor by which to multiply the threshold for the negative polatiry "
48 "detections for the \"bright\" detections stage (this needs to be fairly "
49 "low given the nature of the negative polarity detections in the very "
50 "large positive polarity threshold).")
51 brightGrowFactor = Field(dtype=int, default=40,
52 doc="Factor by which to grow the footprints of sources detected in the "
53 "\"bright\" detections stage (want this to be large to mask wings of "
54 "bright sources).")
55 brightMaskFractionMax = Field(dtype=float, default=0.95,
56 doc="Maximum allowed fraction of masked pixes from the \"bright\" "
57 "detection stage (to mask regions unsuitable for sky sourcess). "
58 "If this fraction is exeeded, the detection threshold for this stage "
59 "will be increased by bisectFactor until the fraction of masked "
60 "pixels drops below this threshold.")
61 bisectFactor = Field(dtype=float, default=1.2,
62 doc="Factor by which to increase thresholds in brightMaskFractionMax loop.")
63
64 def setDefaults(self):
65 SourceDetectionConfig.setDefaults(self)
66 self.skyObjects.nSources = 1000 # For good statistics
67 for maskStr in ["INTRP", "SAT"]:
68 if maskStr not in self.skyObjects.avoidMask:
69 self.skyObjects.avoidMask.append(maskStr)
70
71
73 """Detection of sources on an image with a dynamic threshold
74
75 We first detect sources using a lower threshold than normal (see config
76 parameter ``prelimThresholdFactor``) in order to identify good sky regions
77 (configurable ``skyObjects``). Then we perform forced PSF photometry on
78 those sky regions. Using those PSF flux measurements and estimated errors,
79 we set the threshold so that the stdev of the measurements matches the
80 median estimated error.
81
82 Besides the usual initialisation of configurables, we also set up
83 the forced measurement which is deliberately not represented in
84 this Task's configuration parameters because we're using it as
85 part of the algorithm and we don't want to allow it to be modified.
86 """
87 ConfigClass = DynamicDetectionConfig
88 _DefaultName = "dynamicDetection"
89
90 def __init__(self, *args, **kwargs):
91
92 SourceDetectionTask.__init__(self, *args, **kwargs)
93 self.makeSubtask("skyObjects")
94
95 # Set up forced measurement.
96 config = ForcedMeasurementTask.ConfigClass()
97 config.plugins.names = ['base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground']
98 # We'll need the "centroid" and "psfFlux" slots
99 for slot in ("shape", "psfShape", "apFlux", "modelFlux", "gaussianFlux", "calibFlux"):
100 setattr(config.slots, slot, None)
101 config.copyColumns = {}
102 self.skySchema = SourceTable.makeMinimalSchema()
103 self.skyMeasurement = ForcedMeasurementTask(config=config, name="skyMeasurement", parentTask=self,
104 refSchema=self.skySchema)
105
106 def calculateThreshold(self, exposure, seed, sigma=None):
107 """Calculate new threshold
108
109 This is the main functional addition to the vanilla
110 `SourceDetectionTask`.
111
112 We identify sky objects and perform forced PSF photometry on
113 them. Using those PSF flux measurements and estimated errors,
114 we set the threshold so that the stdev of the measurements
115 matches the median estimated error.
116
117 Parameters
118 ----------
119 exposureOrig : `lsst.afw.image.Exposure`
120 Exposure on which we're detecting sources.
121 seed : `int`
122 RNG seed to use for finding sky objects.
123 sigma : `float`, optional
124 Gaussian sigma of smoothing kernel; if not provided,
125 will be deduced from the exposure's PSF.
126
127 Returns
128 -------
129 result : `lsst.pipe.base.Struct`
130 Result struct with components:
131
132 ``multiplicative``
133 Multiplicative factor to be applied to the
134 configured detection threshold (`float`).
135 ``additive``
136 Additive factor to be applied to the background
137 level (`float`).
138
139 Raises
140 ------
141 NoWorkFound
142 Raised if the number of good sky sources found is less than the
143 minimum fraction (``self.config.minFractionSources``) of the number
144 requested (``self.skyObjects.config.nSources``).
145 """
146 wcsIsNone = exposure.getWcs() is None
147 if wcsIsNone: # create a dummy WCS as needed by ForcedMeasurementTask
148 self.log.info("WCS for exposure is None. Setting a dummy WCS for dynamic detection.")
149 exposure.setWcs(makeSkyWcs(crpix=geom.Point2D(0, 0),
150 crval=geom.SpherePoint(0, 0, geom.degrees),
151 cdMatrix=makeCdMatrix(scale=1e-5*geom.degrees)))
152 fp = self.skyObjects.run(exposure.maskedImage.mask, seed)
153 skyFootprints = FootprintSet(exposure.getBBox())
154 skyFootprints.setFootprints(fp)
155 table = SourceTable.make(self.skyMeasurement.schema)
156 catalog = SourceCatalog(table)
157 catalog.reserve(len(skyFootprints.getFootprints()))
158 skyFootprints.makeSources(catalog)
159 key = catalog.getCentroidSlot().getMeasKey()
160 for source in catalog:
161 peaks = source.getFootprint().getPeaks()
162 assert len(peaks) == 1
163 source.set(key, peaks[0].getF())
164 source.updateCoord(exposure.getWcs())
165
166 # Forced photometry on sky objects
167 self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs())
168
169 # Calculate new threshold
170 fluxes = catalog["base_PsfFlux_instFlux"]
171 area = catalog["base_PsfFlux_area"]
172 bg = catalog["base_LocalBackground_instFlux"]
173
174 good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"]
175 & np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg))
176
177 minNumSources = int(self.config.minFractionSources*self.skyObjects.config.nSources)
178 if good.sum() < minNumSources:
179 raise NoWorkFound(f"Insufficient good sky source flux measurements ({good.sum()} < "
180 f"{minNumSources}) for dynamic threshold calculation.")
181
182 self.log.info("Number of good sky sources used for dynamic detection: %d (of %d requested).",
183 good.sum(), self.skyObjects.config.nSources)
184 bgMedian = np.median((fluxes/area)[good])
185
186 lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0])
187 stdevMeas = 0.741*(uq - lq)
188 medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good])
189 if wcsIsNone:
190 exposure.setWcs(None)
191 return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian)
192
193 def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None):
194 """Detect footprints with a dynamic threshold
195
196 This varies from the vanilla ``detectFootprints`` method because we
197 do detection three times: first with a high threshold to detect
198 "bright" (both positive and negative, the latter to identify very
199 over-subtracted regions) sources for which we grow the DETECTED and
200 DETECTED_NEGATIVE masks significantly to account for wings. Second,
201 with a low threshold to mask all non-empty regions of the image. These
202 two masks are combined and used to identify regions of sky
203 uncontaminated by objects. A final round of detection is then done
204 with the new calculated threshold.
205
206 Parameters
207 ----------
208 exposure : `lsst.afw.image.Exposure`
209 Exposure to process; DETECTED{,_NEGATIVE} mask plane will be
210 set in-place.
211 doSmooth : `bool`, optional
212 If True, smooth the image before detection using a Gaussian
213 of width ``sigma``.
214 sigma : `float`, optional
215 Gaussian Sigma of PSF (pixels); used for smoothing and to grow
216 detections; if `None` then measure the sigma of the PSF of the
217 ``exposure``.
218 clearMask : `bool`, optional
219 Clear both DETECTED and DETECTED_NEGATIVE planes before running
220 detection.
221 expId : `int`, optional
222 Exposure identifier, used as a seed for the random number
223 generator. If absent, the seed will be the sum of the image.
224
225 Returns
226 -------
227 resutls : `lsst.pipe.base.Struct`
228 The results `~lsst.pipe.base.Struct` contains:
229
230 ``positive``
231 Positive polarity footprints.
233 ``negative``
234 Negative polarity footprints.
236 ``numPos``
237 Number of footprints in positive or 0 if detection polarity was
238 negative. (`int`)
239 ``numNeg``
240 Number of footprints in negative or 0 if detection polarity was
241 positive. (`int`)
242 ``background``
243 Re-estimated background. `None` if
244 ``reEstimateBackground==False``.
245 (`lsst.afw.math.BackgroundList`)
246 ``factor``
247 Multiplication factor applied to the configured detection
248 threshold. (`float`)
249 ``prelim``
250 Results from preliminary detection pass.
251 (`lsst.pipe.base.Struct`)
252 """
253 maskedImage = exposure.maskedImage
254
255 if clearMask:
256 self.clearMask(maskedImage.mask)
257 else:
258 oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(["DETECTED",
259 "DETECTED_NEGATIVE"])
260
261 with self.tempWideBackgroundContext(exposure):
262 # Could potentially smooth with a wider kernel than the PSF in
263 # order to better pick up the wings of stars and galaxies, but for
264 # now sticking with the PSF as that's more simple.
265 psf = self.getPsf(exposure, sigma=sigma)
266 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth)
267
268 if self.config.doBrightPrelimDetection:
269 brightDetectedMask = self._computeBrightDetectionMask(maskedImage, convolveResults)
270
271 middle = convolveResults.middle
272 sigma = convolveResults.sigma
273 prelim = self.applyThreshold(
274 middle, maskedImage.getBBox(), factor=self.config.prelimThresholdFactor,
275 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor
276 )
278 maskedImage.mask, prelim, sigma, factor=self.config.prelimThresholdFactor,
279 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor
280 )
281 if self.config.doBrightPrelimDetection:
282 # Combine prelim and bright detection masks for multiplier
283 # determination.
284 maskedImage.mask.array |= brightDetectedMask
285
286 # Calculate the proper threshold
287 # seed needs to fit in a C++ 'int' so pybind doesn't choke on it
288 seed = (expId if expId is not None else int(maskedImage.image.array.sum())) % (2**31 - 1)
289 threshResults = self.calculateThreshold(exposure, seed, sigma=sigma)
290 factor = threshResults.multiplicative
291 self.log.info("Modifying configured detection threshold by factor %f to %f",
292 factor, factor*self.config.thresholdValue)
293
294 # Blow away preliminary (low threshold) detection mask
295 self.clearMask(maskedImage.mask)
296 if not clearMask:
297 maskedImage.mask.array |= oldDetected
298
299 # Rinse and repeat thresholding with new calculated threshold
300 results = self.applyThreshold(middle, maskedImage.getBBox(), factor)
301 results.prelim = prelim
302 results.background = lsst.afw.math.BackgroundList()
303 if self.config.doTempLocalBackground:
304 self.applyTempLocalBackground(exposure, middle, results)
305 self.finalizeFootprints(maskedImage.mask, results, sigma, factor=factor)
306
307 self.clearUnwantedResults(maskedImage.mask, results)
308
309 if self.config.reEstimateBackground:
310 self.reEstimateBackground(maskedImage, results.background)
311
312 self.display(exposure, results, middle)
313
314 if self.config.doBackgroundTweak:
315 # Re-do the background tweak after any temporary backgrounds have
316 # been restored.
317 #
318 # But we want to keep any large-scale background (e.g., scattered
319 # light from bright stars) from being selected for sky objects in
320 # the calculation, so do another detection pass without either the
321 # local or wide temporary background subtraction; the DETECTED
322 # pixels will mark the area to ignore.
323 originalMask = maskedImage.mask.array.copy()
324 try:
325 self.clearMask(exposure.mask)
326 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth)
327 tweakDetResults = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor)
328 self.finalizeFootprints(maskedImage.mask, tweakDetResults, sigma, factor=factor)
329 bgLevel = self.calculateThreshold(exposure, seed, sigma=sigma).additive
330 finally:
331 maskedImage.mask.array[:] = originalMask
332 self.tweakBackground(exposure, bgLevel, results.background)
333
334 return results
335
336 def tweakBackground(self, exposure, bgLevel, bgList=None):
337 """Modify the background by a constant value
338
339 Parameters
340 ----------
341 exposure : `lsst.afw.image.Exposure`
342 Exposure for which to tweak background.
343 bgLevel : `float`
344 Background level to remove
345 bgList : `lsst.afw.math.BackgroundList`, optional
346 List of backgrounds to append to.
347
348 Returns
349 -------
351 Constant background model.
352 """
353 self.log.info("Tweaking background by %f to match sky photometry", bgLevel)
354 exposure.image -= bgLevel
355 bgStats = lsst.afw.image.MaskedImageF(1, 1)
356 bgStats.set(bgLevel, 0, bgLevel)
357 bg = lsst.afw.math.BackgroundMI(exposure.getBBox(), bgStats)
358 bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER,
359 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0, False)
360 if bgList is not None:
361 bgList.append(bgData)
362 return bg
363
364 def _computeBrightDetectionMask(self, maskedImage, convolveResults):
365 """Perform an initial bright source detection pass.
366
367 Perform an initial bright object detection pass using a high detection
368 threshold. The footprints in this pass are grown significantly more
369 than is typical to account for wings around bright sources. The
370 negative polarity detections in this pass help in masking severely
371 over-subtracted regions.
372
373 A maximum fraction of masked pixel from this pass is ensured via
374 the config ``brightMaskFractionMax``. If the masked pixel fraction is
375 above this value, the detection thresholds here are increased by
376 ``bisectFactor`` in a while loop until the detected masked fraction
377 falls below this value.
378
379 Parameters
380 ----------
381 maskedImage : `lsst.afw.image.MaskedImage`
382 Masked image on which to run the detection.
383 convolveResults : `lsst.pipe.base.Struct`
384 The results of the self.convolveImage function with attributes:
385
386 ``middle``
387 Convolved image, without the edges
389 ``sigma``
390 Gaussian sigma used for the convolution (`float`).
391
392 Returns
393 -------
394 brightDetectedMask : `numpy.ndarray`
395 Boolean array representing the union of the bright detection pass
396 DETECTED and DETECTED_NEGATIVE masks.
397 """
398 # Initialize some parameters.
399 brightPosFactor = (
400 self.config.prelimThresholdFactor*self.config.brightMultiplier/self.config.bisectFactor
401 )
402 brightNegFactor = self.config.brightNegFactor/self.config.bisectFactor
403 nPix = 1
404 nPixDet = 1
405 nPixDetNeg = 1
406 brightMaskFractionMax = self.config.brightMaskFractionMax
407
408 # Loop until masked fraction is smaller than
409 # brightMaskFractionMax, increasing the thresholds by
410 # config.bisectFactor on each iteration (rarely necessary
411 # for current defaults).
412 while nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax:
413 self.clearMask(maskedImage.mask)
414 brightPosFactor *= self.config.bisectFactor
415 brightNegFactor *= self.config.bisectFactor
416 prelimBright = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(),
417 factor=brightPosFactor, factorNeg=brightNegFactor)
419 maskedImage.mask, prelimBright, convolveResults.sigma*self.config.brightGrowFactor,
420 factor=brightPosFactor, factorNeg=brightNegFactor
421 )
422 # Check that not too many pixels got masked.
423 nPix = maskedImage.mask.array.size
424 nPixDet = countMaskedPixels(maskedImage, "DETECTED")
425 self.log.info("Number (%) of bright DETECTED pix: {} ({:.1f}%)".
426 format(nPixDet, 100*nPixDet/nPix))
427 nPixDetNeg = countMaskedPixels(maskedImage, "DETECTED_NEGATIVE")
428 self.log.info("Number (%) of bright DETECTED_NEGATIVE pix: {} ({:.1f}%)".
429 format(nPixDetNeg, 100*nPixDetNeg/nPix))
430 if nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax:
431 self.log.warn("Too high a fraction (%.1f > %.1f) of pixels were masked with current "
432 "\"bright\" detection round thresholds. Increasing by a factor of %f "
433 "and trying again.", max(nPixDetNeg, nPixDet)/nPix,
434 brightMaskFractionMax, self.config.bisectFactor)
435
436 # Save the mask planes from the "bright" detection round, then
437 # clear them before moving on to the "prelim" detection phase.
438 brightDetectedMask = (maskedImage.mask.array
439 & maskedImage.mask.getPlaneBitMask(["DETECTED", "DETECTED_NEGATIVE"]))
440 self.clearMask(maskedImage.mask)
441 return brightDetectedMask
442
443
444def countMaskedPixels(maskedIm, maskPlane):
445 """Count the number of pixels in a given mask plane.
446
447 Parameters
448 ----------
449 maskedIm : `lsst.afw.image.MaskedImage`
450 Masked image to examine.
451 maskPlane : `str`
452 Name of the mask plane to examine.
453
454 Returns
455 -------
456 nPixMasked : `int`
457 Number of pixels with ``maskPlane`` bit set.
458 """
459 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
460 nPixMasked = np.sum(np.bitwise_and(maskedIm.mask.array, maskBit))/maskBit
461 return nPixMasked
def getPsf(self, exposure, sigma=None)
Definition: detection.py:420
def run(self, table, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None)
Definition: detection.py:215
def convolveImage(self, maskedImage, psf, doSmooth=True)
Definition: detection.py:454
def applyTempLocalBackground(self, exposure, middle, results)
Definition: detection.py:351
def reEstimateBackground(self, maskedImage, backgrounds)
Definition: detection.py:652
def applyThreshold(self, middle, bbox, factor=1.0, factorNeg=None)
Definition: detection.py:516
def display(self, exposure, results, convolvedImage=None)
Definition: detection.py:294
def finalizeFootprints(self, mask, results, sigma, factor=1.0, factorNeg=None)
Definition: detection.py:582
def tweakBackground(self, exposure, bgLevel, bgList=None)
def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None)
def calculateThreshold(self, exposure, seed, sigma=None)
def _computeBrightDetectionMask(self, maskedImage, convolveResults)
def countMaskedPixels(maskedIm, maskPlane)