2__all__ = [
"DynamicDetectionConfig",
"DynamicDetectionTask"]
7from lsst.pipe.base
import Struct, NoWorkFound
9from .detection
import SourceDetectionConfig, SourceDetectionTask
10from .skyObjects
import SkyObjectsTask
21 """Configuration for DynamicDetectionTask
23 prelimThresholdFactor = Field(dtype=float, default=0.5,
24 doc="Factor by which to multiply the main detection threshold "
25 "(thresholdValue) to use for first pass (to find sky objects).")
26 prelimNegMultiplier = Field(dtype=float, default=2.5,
27 doc=
"Multiplier for the negative (relative to positive) polarity "
28 "detections threshold to use for first pass (to find sky objects).")
29 skyObjects = ConfigurableField(target=SkyObjectsTask, doc=
"Generate sky objects.")
30 doBackgroundTweak = Field(dtype=bool, default=
True,
31 doc=
"Tweak background level so median PSF flux of sky objects is zero?")
32 minFractionSources = Field(dtype=float, default=0.02,
33 doc=
"Minimum fraction of the requested number of sky sources for dynamic "
34 "detection to be considered a success. If the number of good sky sources "
35 "identified falls below this threshold, a NoWorkFound error is raised so "
36 "that this dataId is no longer considered in downstream processing.")
37 doBrightPrelimDetection = Field(dtype=bool, default=
True,
38 doc=
"Do initial bright detection pass where footprints are grown "
39 "by brightGrowFactor?")
40 brightMultiplier = Field(dtype=float, default=2000.0,
41 doc=
"Multiplier to apply to the prelimThresholdFactor for the "
42 "\"bright\" detections stage (want this to be large to only "
43 "detect the brightest sources).")
44 brightNegFactor = Field(dtype=float, default=2.2,
45 doc=
"Factor by which to multiply the threshold for the negative polatiry "
46 "detections for the \"bright\" detections stage (this needs to be fairly "
47 "low given the nature of the negative polarity detections in the very "
48 "large positive polarity threshold).")
49 brightGrowFactor = Field(dtype=int, default=40,
50 doc=
"Factor by which to grow the footprints of sources detected in the "
51 "\"bright\" detections stage (want this to be large to mask wings of "
53 brightMaskFractionMax = Field(dtype=float, default=0.95,
54 doc=
"Maximum allowed fraction of masked pixes from the \"bright\" "
55 "detection stage (to mask regions unsuitable for sky sourcess). "
56 "If this fraction is exeeded, the detection threshold for this stage "
57 "will be increased by bisectFactor until the fraction of masked "
58 "pixels drops below this threshold.")
59 bisectFactor = Field(dtype=float, default=1.2,
60 doc=
"Factor by which to increase thresholds in brightMaskFractionMax loop.")
63 SourceDetectionConfig.setDefaults(self)
65 for maskStr
in [
"SAT"]:
71 """Detection of sources on an image with a dynamic threshold
73 We first detect sources using a lower threshold than normal (see config
74 parameter ``prelimThresholdFactor``) in order to identify good sky regions
75 (configurable ``skyObjects``). Then we perform forced PSF photometry on
76 those sky regions. Using those PSF flux measurements
and estimated errors,
77 we set the threshold so that the stdev of the measurements matches the
78 median estimated error.
80 Besides the usual initialisation of configurables, we also set up
81 the forced measurement which
is deliberately
not represented
in
82 this Task
's configuration parameters because we're using it
as
83 part of the algorithm
and we don
't want to allow it to be modified.
85 ConfigClass = DynamicDetectionConfig
86 _DefaultName = "dynamicDetection"
90 SourceDetectionTask.__init__(self, *args, **kwargs)
91 self.makeSubtask(
"skyObjects")
94 config = ForcedMeasurementTask.ConfigClass()
95 config.plugins.names = [
'base_TransformedCentroid',
'base_PsfFlux',
'base_LocalBackground']
97 for slot
in (
"shape",
"psfShape",
"apFlux",
"modelFlux",
"gaussianFlux",
"calibFlux"):
98 setattr(config.slots, slot,
None)
99 config.copyColumns = {}
101 self.
skyMeasurement = ForcedMeasurementTask(config=config, name=
"skyMeasurement", parentTask=self,
104 def calculateThreshold(self, exposure, seed, sigma=None, minFractionSourcesFactor=1.0, isBgTweak=False):
105 """Calculate new threshold
107 This is the main functional addition to the vanilla
108 `SourceDetectionTask`.
110 We identify sky objects
and perform forced PSF photometry on
111 them. Using those PSF flux measurements
and estimated errors,
112 we set the threshold so that the stdev of the measurements
113 matches the median estimated error.
118 Exposure on which we
're detecting sources.
120 RNG seed to use for finding sky objects.
121 sigma : `float`, optional
122 Gaussian sigma of smoothing kernel;
if not provided,
123 will be deduced
from the exposure
's PSF.
124 minFractionSourcesFactor : `float`
125 Change the fraction of required sky sources from that set
in
126 ``self.config.minFractionSources`` by this factor. NOTE: this
127 is intended
for use
in the background tweak
pass (the detection
128 threshold
is much lower there, so many more pixels end up marked
129 as DETECTED
or DETECTED_NEGATIVE, leaving less room
for sky
132 Set to ``
True``
for the background tweak
pass (
for more helpful
137 result : `lsst.pipe.base.Struct`
138 Result struct
with components:
140 - ``multiplicative``: multiplicative factor to be applied to the
141 configured detection threshold (`float`).
142 - ``additive``: additive factor to be applied to the background
148 Raised
if the number of good sky sources found
is less than the
150 (``self.config.minFractionSources``*``minFractionSourcesFactor``)
151 of the number requested (``self.skyObjects.config.nSources``).
154 fp = self.skyObjects.
run(exposure.maskedImage.mask, seed)
156 skyFootprints.setFootprints(fp)
158 catalog = SourceCatalog(table)
159 catalog.reserve(len(skyFootprints.getFootprints()))
160 skyFootprints.makeSources(catalog)
161 key = catalog.getCentroidSlot().getMeasKey()
162 for source
in catalog:
163 peaks = source.getFootprint().getPeaks()
164 assert len(peaks) == 1
165 source.set(key, peaks[0].getF())
166 source.updateCoord(exposure.getWcs())
172 fluxes = catalog[
"base_PsfFlux_instFlux"]
173 area = catalog[
"base_PsfFlux_area"]
174 bg = catalog[
"base_LocalBackground_instFlux"]
176 good = (~catalog[
"base_PsfFlux_flag"] & ~catalog[
"base_LocalBackground_flag"]
177 & np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg))
179 minNumSources = int(self.config.minFractionSources*self.skyObjects.config.nSources)
182 if minFractionSourcesFactor != 1.0:
183 minNumSources = max(3, int(minNumSources*minFractionSourcesFactor))
184 if good.sum() < minNumSources:
186 msg = (f
"Insufficient good sky source flux measurements ({good.sum()} < "
187 f
"{minNumSources}) for dynamic threshold calculation.")
189 msg = (f
"Insufficient good sky source flux measurements ({good.sum()} < "
190 f
"{minNumSources}) for background tweak calculation.")
192 nPix = exposure.mask.array.size
194 nGoodPix = np.sum(exposure.mask.array & badPixelMask == 0)
195 if nGoodPix/nPix > 0.2:
197 nDetectedPix = np.sum(exposure.mask.array & detectedPixelMask != 0)
198 msg += (f
" However, {nGoodPix}/{nPix} pixels are not marked NO_DATA or BAD, "
199 "so there should be sufficient area to locate suitable sky sources. "
200 f
"Note that {nDetectedPix} of {nGoodPix} \"good\" pixels were marked "
201 "as DETECTED or DETECTED_NEGATIVE.")
202 raise RuntimeError(msg)
203 raise NoWorkFound(msg)
206 self.log.info(
"Number of good sky sources used for dynamic detection: %d (of %d requested).",
207 good.sum(), self.skyObjects.config.nSources)
209 self.log.info(
"Number of good sky sources used for dynamic detection background tweak:"
210 " %d (of %d requested).", good.sum(), self.skyObjects.config.nSources)
211 bgMedian = np.median((fluxes/area)[good])
213 lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0])
214 stdevMeas = 0.741*(uq - lq)
215 medianError = np.median(catalog[
"base_PsfFlux_instFluxErr"][good])
216 return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian)
218 def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None):
219 """Detect footprints with a dynamic threshold
221 This varies from the vanilla ``detectFootprints`` method because we
222 do detection three times: first
with a high threshold to detect
223 "bright" (both positive
and negative, the latter to identify very
224 over-subtracted regions) sources
for which we grow the DETECTED
and
225 DETECTED_NEGATIVE masks significantly to account
for wings. Second,
226 with a low threshold to mask all non-empty regions of the image. These
227 two masks are combined
and used to identify regions of sky
228 uncontaminated by objects. A final round of detection
is then done
229 with the new calculated threshold.
234 Exposure to process; DETECTED{,_NEGATIVE} mask plane will be
236 doSmooth : `bool`, optional
237 If
True, smooth the image before detection using a Gaussian
239 sigma : `float`, optional
240 Gaussian Sigma of PSF (pixels); used
for smoothing
and to grow
241 detections;
if `
None` then measure the sigma of the PSF of the
243 clearMask : `bool`, optional
244 Clear both DETECTED
and DETECTED_NEGATIVE planes before running
246 expId : `int`, optional
247 Exposure identifier, used
as a seed
for the random number
248 generator. If absent, the seed will be the sum of the image.
250 Return Struct contents
251 ----------------------
253 Positive polarity footprints (may be `
None`)
255 Negative polarity footprints (may be `
None`)
257 Number of footprints
in positive
or 0
if detection polarity was
260 Number of footprints
in negative
or 0
if detection polarity was
262 background : `lsst.afw.math.BackgroundList`
263 Re-estimated background. `
None`
if
264 ``reEstimateBackground==
False``.
266 Multiplication factor applied to the configured detection
268 prelim : `lsst.pipe.base.Struct`
269 Results
from preliminary detection
pass.
271 maskedImage = exposure.maskedImage
276 oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask([
"DETECTED",
277 "DETECTED_NEGATIVE"])
278 nPix = maskedImage.mask.array.size
280 nGoodPix = np.sum(maskedImage.mask.array & badPixelMask == 0)
281 self.log.info(
"Number of good data pixels (i.e. not NO_DATA or BAD): {} ({:.1f}% of total)".
282 format(nGoodPix, 100*nGoodPix/nPix))
287 psf = self.
getPsf(exposure, sigma=sigma)
288 convolveResults = self.
convolveImage(maskedImage, psf, doSmooth=doSmooth)
290 if self.config.doBrightPrelimDetection:
293 middle = convolveResults.middle
294 sigma = convolveResults.sigma
296 middle, maskedImage.getBBox(), factor=self.config.prelimThresholdFactor,
297 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor
300 maskedImage.mask, prelim, sigma, factor=self.config.prelimThresholdFactor,
301 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor
303 if self.config.doBrightPrelimDetection:
306 maskedImage.mask.array |= brightDetectedMask
310 seed = (expId
if expId
is not None else int(maskedImage.image.array.sum())) % (2**31 - 1)
312 factor = threshResults.multiplicative
313 self.log.info(
"Modifying configured detection threshold by factor %f to %f",
314 factor, factor*self.config.thresholdValue)
319 maskedImage.mask.array |= oldDetected
322 results = self.
applyThreshold(middle, maskedImage.getBBox(), factor)
323 results.prelim = prelim
324 results.background = lsst.afw.math.BackgroundList()
325 if self.config.doTempLocalBackground:
331 if self.config.reEstimateBackground:
334 self.
display(exposure, results, middle)
336 if self.config.doBackgroundTweak:
343 originalMask = maskedImage.mask.array.copy()
346 convolveResults = self.
convolveImage(maskedImage, psf, doSmooth=doSmooth)
347 tweakDetResults = self.
applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor)
349 bgLevel = self.
calculateThreshold(exposure, seed, sigma=sigma, minFractionSourcesFactor=0.5,
350 isBgTweak=
True).additive
352 maskedImage.mask.array[:] = originalMask
358 """Modify the background by a constant value
363 Exposure for which to tweak background.
365 Background level to remove
366 bgList : `lsst.afw.math.BackgroundList`, optional
367 List of backgrounds to append to.
372 Constant background model.
374 self.log.info("Tweaking background by %f to match sky photometry", bgLevel)
375 exposure.image -= bgLevel
376 bgStats = lsst.afw.image.MaskedImageF(1, 1)
377 bgStats.set(bgLevel, 0, bgLevel)
379 bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER,
380 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0,
False)
381 if bgList
is not None:
382 bgList.append(bgData)
385 def _computeBrightDetectionMask(self, maskedImage, convolveResults):
386 """Perform an initial bright source detection pass.
388 Perform an initial bright object detection pass using a high detection
389 threshold. The footprints
in this
pass are grown significantly more
390 than
is typical to account
for wings around bright sources. The
391 negative polarity detections
in this
pass help
in masking severely
392 over-subtracted regions.
394 A maximum fraction of masked pixel
from this
pass is ensured via
395 the config ``brightMaskFractionMax``. If the masked pixel fraction
is
396 above this value, the detection thresholds here are increased by
397 ``bisectFactor``
in a
while loop until the detected masked fraction
398 falls below this value.
403 Masked image on which to run the detection.
404 convolveResults : `lsst.pipe.base.Struct`
405 The results of the self.
convolveImage function
with attributes:
408 Convolved image, without the edges
411 Gaussian sigma used
for the convolution (`float`).
415 brightDetectedMask : `numpy.ndarray`
416 Boolean array representing the union of the bright detection
pass
417 DETECTED
and DETECTED_NEGATIVE masks.
421 self.config.prelimThresholdFactor*self.config.brightMultiplier/self.config.bisectFactor
423 brightNegFactor = self.config.brightNegFactor/self.config.bisectFactor
427 brightMaskFractionMax = self.config.brightMaskFractionMax
433 while nPixDetNeg/nPix > brightMaskFractionMax
or nPixDet/nPix > brightMaskFractionMax:
435 brightPosFactor *= self.config.bisectFactor
436 brightNegFactor *= self.config.bisectFactor
437 prelimBright = self.
applyThreshold(convolveResults.middle, maskedImage.getBBox(),
438 factor=brightPosFactor, factorNeg=brightNegFactor)
440 maskedImage.mask, prelimBright, convolveResults.sigma*self.config.brightGrowFactor,
441 factor=brightPosFactor, factorNeg=brightNegFactor
444 nPix = maskedImage.mask.array.size
446 self.log.info(
"Number (%) of bright DETECTED pix: {} ({:.1f}%)".
447 format(nPixDet, 100*nPixDet/nPix))
449 self.log.info(
"Number (%) of bright DETECTED_NEGATIVE pix: {} ({:.1f}%)".
450 format(nPixDetNeg, 100*nPixDetNeg/nPix))
451 if nPixDetNeg/nPix > brightMaskFractionMax
or nPixDet/nPix > brightMaskFractionMax:
452 self.log.warn(
"Too high a fraction (%.1f > %.1f) of pixels were masked with current "
453 "\"bright\" detection round thresholds. Increasing by a factor of %f "
454 "and trying again.", max(nPixDetNeg, nPixDet)/nPix,
455 brightMaskFractionMax, self.config.bisectFactor)
459 brightDetectedMask = (maskedImage.mask.array
460 & maskedImage.mask.getPlaneBitMask([
"DETECTED",
"DETECTED_NEGATIVE"]))
462 return brightDetectedMask
466 """Count the number of pixels in a given mask plane.
471 Masked image to examine.
473 Name of the mask plane to examine.
478 Number of pixels with ``maskPlane`` bit set.
480 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
481 nPixMasked = np.sum(np.bitwise_and(maskedIm.mask.array, maskBit))/maskBit
static MaskPixelT getPlaneBitMask(const std::vector< std::string > &names)
def tempWideBackgroundContext(self, exposure)
def getPsf(self, exposure, sigma=None)
def run(self, table, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None)
def convolveImage(self, maskedImage, psf, doSmooth=True)
def applyTempLocalBackground(self, exposure, middle, results)
def reEstimateBackground(self, maskedImage, backgrounds)
def applyThreshold(self, middle, bbox, factor=1.0, factorNeg=None)
def clearUnwantedResults(self, mask, results)
def clearMask(self, mask)
def display(self, exposure, results, convolvedImage=None)
def finalizeFootprints(self, mask, results, sigma, factor=1.0, factorNeg=None)
def calculateThreshold(self, exposure, seed, sigma=None, minFractionSourcesFactor=1.0, isBgTweak=False)
def __init__(self, *args, **kwargs)
def tweakBackground(self, exposure, bgLevel, bgList=None)
def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None)
def _computeBrightDetectionMask(self, maskedImage, convolveResults)
def countMaskedPixels(maskedIm, maskPlane)