2__all__ = [
"DynamicDetectionConfig",
"DynamicDetectionTask"]
9from .detection
import SourceDetectionConfig, SourceDetectionTask
10from .skyObjects
import SkyObjectsTask
23 """Configuration for DynamicDetectionTask
25 prelimThresholdFactor = Field(dtype=float, default=0.5,
26 doc="Factor by which to multiply the main detection threshold "
27 "(thresholdValue) to use for first pass (to find sky objects).")
28 prelimNegMultiplier = Field(dtype=float, default=2.5,
29 doc=
"Multiplier for the negative (relative to positive) polarity "
30 "detections threshold to use for first pass (to find sky objects).")
31 skyObjects = ConfigurableField(target=SkyObjectsTask, doc=
"Generate sky objects.")
32 doBackgroundTweak = Field(dtype=bool, default=
True,
33 doc=
"Tweak background level so median PSF flux of sky objects is zero?")
34 minFractionSources = Field(dtype=float, default=0.02,
35 doc=
"Minimum fraction of the requested number of sky sources for dynamic "
36 "detection to be considered a success. If the number of good sky sources "
37 "identified falls below this threshold, a NoWorkFound error is raised so "
38 "that this dataId is no longer considered in downstream processing.")
39 doBrightPrelimDetection = Field(dtype=bool, default=
True,
40 doc=
"Do initial bright detection pass where footprints are grown "
41 "by brightGrowFactor?")
42 brightMultiplier = Field(dtype=float, default=2000.0,
43 doc=
"Multiplier to apply to the prelimThresholdFactor for the "
44 "\"bright\" detections stage (want this to be large to only "
45 "detect the brightest sources).")
46 brightNegFactor = Field(dtype=float, default=2.2,
47 doc=
"Factor by which to multiply the threshold for the negative polatiry "
48 "detections for the \"bright\" detections stage (this needs to be fairly "
49 "low given the nature of the negative polarity detections in the very "
50 "large positive polarity threshold).")
51 brightGrowFactor = Field(dtype=int, default=40,
52 doc=
"Factor by which to grow the footprints of sources detected in the "
53 "\"bright\" detections stage (want this to be large to mask wings of "
55 brightMaskFractionMax = Field(dtype=float, default=0.95,
56 doc=
"Maximum allowed fraction of masked pixes from the \"bright\" "
57 "detection stage (to mask regions unsuitable for sky sourcess). "
58 "If this fraction is exeeded, the detection threshold for this stage "
59 "will be increased by bisectFactor until the fraction of masked "
60 "pixels drops below this threshold.")
61 bisectFactor = Field(dtype=float, default=1.2,
62 doc=
"Factor by which to increase thresholds in brightMaskFractionMax loop.")
65 SourceDetectionConfig.setDefaults(self)
67 for maskStr
in [
"INTRP",
"SAT"]:
73 """Detection of sources on an image with a dynamic threshold
75 We first detect sources using a lower threshold than normal (see config
76 parameter ``prelimThresholdFactor``) in order to identify good sky regions
77 (configurable ``skyObjects``). Then we perform forced PSF photometry on
78 those sky regions. Using those PSF flux measurements
and estimated errors,
79 we set the threshold so that the stdev of the measurements matches the
80 median estimated error.
82 Besides the usual initialisation of configurables, we also set up
83 the forced measurement which
is deliberately
not represented
in
84 this Task
's configuration parameters because we're using it
as
85 part of the algorithm
and we don
't want to allow it to be modified.
87 ConfigClass = DynamicDetectionConfig
88 _DefaultName = "dynamicDetection"
92 SourceDetectionTask.__init__(self, *args, **kwargs)
93 self.makeSubtask(
"skyObjects")
96 config = ForcedMeasurementTask.ConfigClass()
97 config.plugins.names = [
'base_TransformedCentroid',
'base_PsfFlux',
'base_LocalBackground']
99 for slot
in (
"shape",
"psfShape",
"apFlux",
"modelFlux",
"gaussianFlux",
"calibFlux"):
100 setattr(config.slots, slot,
None)
101 config.copyColumns = {}
103 self.
skyMeasurement = ForcedMeasurementTask(config=config, name=
"skyMeasurement", parentTask=self,
107 """Calculate new threshold
109 This is the main functional addition to the vanilla
110 `SourceDetectionTask`.
112 We identify sky objects
and perform forced PSF photometry on
113 them. Using those PSF flux measurements
and estimated errors,
114 we set the threshold so that the stdev of the measurements
115 matches the median estimated error.
120 Exposure on which we
're detecting sources.
122 RNG seed to use for finding sky objects.
123 sigma : `float`, optional
124 Gaussian sigma of smoothing kernel;
if not provided,
125 will be deduced
from the exposure
's PSF.
129 result : `lsst.pipe.base.Struct`
130 Result struct with components:
133 Multiplicative factor to be applied to the
134 configured detection threshold (`float`).
136 Additive factor to be applied to the background
142 Raised
if the number of good sky sources found
is less than the
143 minimum fraction (``self.config.minFractionSources``) of the number
144 requested (``self.skyObjects.config.nSources``).
146 wcsIsNone = exposure.getWcs() is None
148 self.log.info(
"WCS for exposure is None. Setting a dummy WCS for dynamic detection.")
151 cdMatrix=makeCdMatrix(scale=1e-5*geom.degrees)))
152 fp = self.skyObjects.
run(exposure.maskedImage.mask, seed)
154 skyFootprints.setFootprints(fp)
156 catalog = SourceCatalog(table)
157 catalog.reserve(len(skyFootprints.getFootprints()))
158 skyFootprints.makeSources(catalog)
159 key = catalog.getCentroidSlot().getMeasKey()
160 for source
in catalog:
161 peaks = source.getFootprint().getPeaks()
162 assert len(peaks) == 1
163 source.set(key, peaks[0].getF())
164 source.updateCoord(exposure.getWcs())
170 fluxes = catalog[
"base_PsfFlux_instFlux"]
171 area = catalog[
"base_PsfFlux_area"]
172 bg = catalog[
"base_LocalBackground_instFlux"]
174 good = (~catalog[
"base_PsfFlux_flag"] & ~catalog[
"base_LocalBackground_flag"]
175 & np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg))
177 minNumSources = int(self.config.minFractionSources*self.skyObjects.config.nSources)
178 if good.sum() < minNumSources:
179 raise NoWorkFound(f
"Insufficient good sky source flux measurements ({good.sum()} < "
180 f
"{minNumSources}) for dynamic threshold calculation.")
182 self.log.info(
"Number of good sky sources used for dynamic detection: %d (of %d requested).",
183 good.sum(), self.skyObjects.config.nSources)
184 bgMedian = np.median((fluxes/area)[good])
186 lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0])
187 stdevMeas = 0.741*(uq - lq)
188 medianError = np.median(catalog[
"base_PsfFlux_instFluxErr"][good])
190 exposure.setWcs(
None)
191 return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian)
193 def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None,
195 """Detect footprints with a dynamic threshold
197 This varies from the vanilla ``detectFootprints`` method because we
198 do detection three times: first
with a high threshold to detect
199 "bright" (both positive
and negative, the latter to identify very
200 over-subtracted regions) sources
for which we grow the DETECTED
and
201 DETECTED_NEGATIVE masks significantly to account
for wings. Second,
202 with a low threshold to mask all non-empty regions of the image. These
203 two masks are combined
and used to identify regions of sky
204 uncontaminated by objects. A final round of detection
is then done
205 with the new calculated threshold.
210 Exposure to process; DETECTED{,_NEGATIVE} mask plane will be
212 doSmooth : `bool`, optional
213 If
True, smooth the image before detection using a Gaussian
215 sigma : `float`, optional
216 Gaussian Sigma of PSF (pixels); used
for smoothing
and to grow
217 detections;
if `
None` then measure the sigma of the PSF of the
219 clearMask : `bool`, optional
220 Clear both DETECTED
and DETECTED_NEGATIVE planes before running
222 expId : `int`, optional
223 Exposure identifier, used
as a seed
for the random number
224 generator. If absent, the seed will be the sum of the image.
225 background : `lsst.afw.math.BackgroundList`, optional
226 Background that was already subtracted
from the exposure; will be
227 modified
in-place
if ``reEstimateBackground=
True``.
231 resutls : `lsst.pipe.base.Struct`
232 The results `~lsst.pipe.base.Struct` contains:
235 Positive polarity footprints.
238 Negative polarity footprints.
241 Number of footprints
in positive
or 0
if detection polarity was
244 Number of footprints
in negative
or 0
if detection polarity was
247 Re-estimated background. `
None`
or the input ``background``
248 if ``reEstimateBackground==
False``.
249 (`lsst.afw.math.BackgroundList`)
251 Multiplication factor applied to the configured detection
254 Results
from preliminary detection
pass.
255 (`lsst.pipe.base.Struct`)
257 maskedImage = exposure.maskedImage
262 oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask([
"DETECTED",
263 "DETECTED_NEGATIVE"])
269 psf = self.
getPsf(exposure, sigma=sigma)
270 convolveResults = self.
convolveImage(maskedImage, psf, doSmooth=doSmooth)
272 if self.config.doBrightPrelimDetection:
275 middle = convolveResults.middle
276 sigma = convolveResults.sigma
278 middle, maskedImage.getBBox(), factor=self.config.prelimThresholdFactor,
279 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor
282 maskedImage.mask, prelim, sigma, factor=self.config.prelimThresholdFactor,
283 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor
285 if self.config.doBrightPrelimDetection:
288 maskedImage.mask.array |= brightDetectedMask
292 seed = (expId
if expId
is not None else int(maskedImage.image.array.sum())) % (2**31 - 1)
294 factor = threshResults.multiplicative
295 self.log.info(
"Modifying configured detection threshold by factor %f to %f",
296 factor, factor*self.config.thresholdValue)
301 maskedImage.mask.array |= oldDetected
304 results = self.
applyThreshold(middle, maskedImage.getBBox(), factor)
305 results.prelim = prelim
306 results.background = background
if background
is not None else lsst.afw.math.BackgroundList()
307 if self.config.doTempLocalBackground:
313 if self.config.reEstimateBackground:
316 self.
display(exposure, results, middle)
318 if self.config.doBackgroundTweak:
327 originalMask = maskedImage.mask.array.copy()
330 convolveResults = self.
convolveImage(maskedImage, psf, doSmooth=doSmooth)
331 tweakDetResults = self.
applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor)
335 maskedImage.mask.array[:] = originalMask
341 """Modify the background by a constant value
346 Exposure for which to tweak background.
348 Background level to remove
349 bgList : `lsst.afw.math.BackgroundList`, optional
350 List of backgrounds to append to.
355 Constant background model.
357 self.log.info("Tweaking background by %f to match sky photometry", bgLevel)
358 exposure.image -= bgLevel
359 bgStats = lsst.afw.image.MaskedImageF(1, 1)
360 bgStats.set(bgLevel, 0, bgLevel)
362 bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER,
363 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0,
False)
364 if bgList
is not None:
365 bgList.append(bgData)
369 """Perform an initial bright source detection pass.
371 Perform an initial bright object detection pass using a high detection
372 threshold. The footprints
in this
pass are grown significantly more
373 than
is typical to account
for wings around bright sources. The
374 negative polarity detections
in this
pass help
in masking severely
375 over-subtracted regions.
377 A maximum fraction of masked pixel
from this
pass is ensured via
378 the config ``brightMaskFractionMax``. If the masked pixel fraction
is
379 above this value, the detection thresholds here are increased by
380 ``bisectFactor``
in a
while loop until the detected masked fraction
381 falls below this value.
386 Masked image on which to run the detection.
387 convolveResults : `lsst.pipe.base.Struct`
388 The results of the self.
convolveImage function
with attributes:
391 Convolved image, without the edges
394 Gaussian sigma used
for the convolution (`float`).
398 brightDetectedMask : `numpy.ndarray`
399 Boolean array representing the union of the bright detection
pass
400 DETECTED
and DETECTED_NEGATIVE masks.
404 self.config.prelimThresholdFactor*self.config.brightMultiplier/self.config.bisectFactor
406 brightNegFactor = self.config.brightNegFactor/self.config.bisectFactor
410 brightMaskFractionMax = self.config.brightMaskFractionMax
416 while nPixDetNeg/nPix > brightMaskFractionMax
or nPixDet/nPix > brightMaskFractionMax:
418 brightPosFactor *= self.config.bisectFactor
419 brightNegFactor *= self.config.bisectFactor
420 prelimBright = self.
applyThreshold(convolveResults.middle, maskedImage.getBBox(),
421 factor=brightPosFactor, factorNeg=brightNegFactor)
423 maskedImage.mask, prelimBright, convolveResults.sigma*self.config.brightGrowFactor,
424 factor=brightPosFactor, factorNeg=brightNegFactor
427 nPix = maskedImage.mask.array.size
429 self.log.info(
"Number (%) of bright DETECTED pix: {} ({:.1f}%)".
430 format(nPixDet, 100*nPixDet/nPix))
432 self.log.info(
"Number (%) of bright DETECTED_NEGATIVE pix: {} ({:.1f}%)".
433 format(nPixDetNeg, 100*nPixDetNeg/nPix))
434 if nPixDetNeg/nPix > brightMaskFractionMax
or nPixDet/nPix > brightMaskFractionMax:
435 self.log.warn(
"Too high a fraction (%.1f > %.1f) of pixels were masked with current "
436 "\"bright\" detection round thresholds. Increasing by a factor of %f "
437 "and trying again.", max(nPixDetNeg, nPixDet)/nPix,
438 brightMaskFractionMax, self.config.bisectFactor)
442 brightDetectedMask = (maskedImage.mask.array
443 & maskedImage.mask.getPlaneBitMask([
"DETECTED",
"DETECTED_NEGATIVE"]))
445 return brightDetectedMask
449 """Count the number of pixels in a given mask plane.
454 Masked image to examine.
456 Name of the mask plane to examine.
461 Number of pixels with ``maskPlane`` bit set.
463 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
464 nPixMasked = np.sum(np.bitwise_and(maskedIm.mask.array, maskBit))/maskBit
applyTempLocalBackground(self, exposure, middle, results)
run(self, table, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None, background=None)
tempWideBackgroundContext(self, exposure)
clearUnwantedResults(self, mask, results)
getPsf(self, exposure, sigma=None)
applyThreshold(self, middle, bbox, factor=1.0, factorNeg=None)
convolveImage(self, maskedImage, psf, doSmooth=True)
display(self, exposure, results, convolvedImage=None)
finalizeFootprints(self, mask, results, sigma, factor=1.0, factorNeg=None)
reEstimateBackground(self, maskedImage, backgrounds)
tweakBackground(self, exposure, bgLevel, bgList=None)
_computeBrightDetectionMask(self, maskedImage, convolveResults)
detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None, background=None)
__init__(self, *args, **kwargs)
calculateThreshold(self, exposure, seed, sigma=None)
countMaskedPixels(maskedIm, maskPlane)