2__all__ = [
"DynamicDetectionConfig",
"DynamicDetectionTask"]
9from .detection
import SourceDetectionConfig, SourceDetectionTask
10from .skyObjects
import SkyObjectsTask
23 """Configuration for DynamicDetectionTask
25 prelimThresholdFactor = Field(dtype=float, default=0.5,
26 doc="Factor by which to multiply the main detection threshold "
27 "(thresholdValue) to use for first pass (to find sky objects).")
28 prelimNegMultiplier = Field(dtype=float, default=2.5,
29 doc=
"Multiplier for the negative (relative to positive) polarity "
30 "detections threshold to use for first pass (to find sky objects).")
31 skyObjects = ConfigurableField(target=SkyObjectsTask, doc=
"Generate sky objects.")
32 doBackgroundTweak = Field(dtype=bool, default=
True,
33 doc=
"Tweak background level so median PSF flux of sky objects is zero?")
34 minFractionSources = Field(dtype=float, default=0.02,
35 doc=
"Minimum fraction of the requested number of sky sources for dynamic "
36 "detection to be considered a success. If the number of good sky sources "
37 "identified falls below this threshold, a NoWorkFound error is raised so "
38 "that this dataId is no longer considered in downstream processing.")
39 doBrightPrelimDetection = Field(dtype=bool, default=
True,
40 doc=
"Do initial bright detection pass where footprints are grown "
41 "by brightGrowFactor?")
42 brightMultiplier = Field(dtype=float, default=2000.0,
43 doc=
"Multiplier to apply to the prelimThresholdFactor for the "
44 "\"bright\" detections stage (want this to be large to only "
45 "detect the brightest sources).")
46 brightNegFactor = Field(dtype=float, default=2.2,
47 doc=
"Factor by which to multiply the threshold for the negative polatiry "
48 "detections for the \"bright\" detections stage (this needs to be fairly "
49 "low given the nature of the negative polarity detections in the very "
50 "large positive polarity threshold).")
51 brightGrowFactor = Field(dtype=int, default=40,
52 doc=
"Factor by which to grow the footprints of sources detected in the "
53 "\"bright\" detections stage (want this to be large to mask wings of "
55 brightMaskFractionMax = Field(dtype=float, default=0.95,
56 doc=
"Maximum allowed fraction of masked pixes from the \"bright\" "
57 "detection stage (to mask regions unsuitable for sky sourcess). "
58 "If this fraction is exeeded, the detection threshold for this stage "
59 "will be increased by bisectFactor until the fraction of masked "
60 "pixels drops below this threshold.")
61 bisectFactor = Field(dtype=float, default=1.2,
62 doc=
"Factor by which to increase thresholds in brightMaskFractionMax loop.")
65 SourceDetectionConfig.setDefaults(self)
67 for maskStr
in [
"SAT"]:
73 """Detection of sources on an image with a dynamic threshold
75 We first detect sources using a lower threshold than normal (see config
76 parameter ``prelimThresholdFactor``) in order to identify good sky regions
77 (configurable ``skyObjects``). Then we perform forced PSF photometry on
78 those sky regions. Using those PSF flux measurements
and estimated errors,
79 we set the threshold so that the stdev of the measurements matches the
80 median estimated error.
82 Besides the usual initialisation of configurables, we also set up
83 the forced measurement which
is deliberately
not represented
in
84 this Task
's configuration parameters because we're using it
as
85 part of the algorithm
and we don
't want to allow it to be modified.
87 ConfigClass = DynamicDetectionConfig
88 _DefaultName = "dynamicDetection"
92 SourceDetectionTask.__init__(self, *args, **kwargs)
93 self.makeSubtask(
"skyObjects")
96 config = ForcedMeasurementTask.ConfigClass()
97 config.plugins.names = [
'base_TransformedCentroid',
'base_PsfFlux',
'base_LocalBackground']
99 for slot
in (
"shape",
"psfShape",
"apFlux",
"modelFlux",
"gaussianFlux",
"calibFlux"):
100 setattr(config.slots, slot,
None)
101 config.copyColumns = {}
103 self.
skyMeasurement = ForcedMeasurementTask(config=config, name=
"skyMeasurement", parentTask=self,
106 def calculateThreshold(self, exposure, seed, sigma=None, minFractionSourcesFactor=1.0, isBgTweak=False):
107 """Calculate new threshold
109 This is the main functional addition to the vanilla
110 `SourceDetectionTask`.
112 We identify sky objects
and perform forced PSF photometry on
113 them. Using those PSF flux measurements
and estimated errors,
114 we set the threshold so that the stdev of the measurements
115 matches the median estimated error.
120 Exposure on which we
're detecting sources.
122 RNG seed to use for finding sky objects.
123 sigma : `float`, optional
124 Gaussian sigma of smoothing kernel;
if not provided,
125 will be deduced
from the exposure
's PSF.
126 minFractionSourcesFactor : `float`
127 Change the fraction of required sky sources from that set
in
128 ``self.config.minFractionSources`` by this factor. NOTE: this
129 is intended
for use
in the background tweak
pass (the detection
130 threshold
is much lower there, so many more pixels end up marked
131 as DETECTED
or DETECTED_NEGATIVE, leaving less room
for sky
134 Set to ``
True``
for the background tweak
pass (
for more helpful
139 result : `lsst.pipe.base.Struct`
140 Result struct
with components:
143 Multiplicative factor to be applied to the
144 configured detection threshold (`float`).
146 Additive factor to be applied to the background
152 Raised
if the number of good sky sources found
is less than the
154 (``self.config.minFractionSources``*``minFractionSourcesFactor``)
155 of the number requested (``self.skyObjects.config.nSources``).
157 wcsIsNone = exposure.getWcs() is None
159 self.log.info(
"WCS for exposure is None. Setting a dummy WCS for dynamic detection.")
162 cdMatrix=makeCdMatrix(scale=1e-5*geom.degrees)))
163 fp = self.skyObjects.
run(exposure.maskedImage.mask, seed)
165 skyFootprints.setFootprints(fp)
167 catalog = SourceCatalog(table)
168 catalog.reserve(len(skyFootprints.getFootprints()))
169 skyFootprints.makeSources(catalog)
170 key = catalog.getCentroidSlot().getMeasKey()
171 for source
in catalog:
172 peaks = source.getFootprint().getPeaks()
173 assert len(peaks) == 1
174 source.set(key, peaks[0].getF())
175 source.updateCoord(exposure.getWcs())
181 fluxes = catalog[
"base_PsfFlux_instFlux"]
182 area = catalog[
"base_PsfFlux_area"]
183 bg = catalog[
"base_LocalBackground_instFlux"]
185 good = (~catalog[
"base_PsfFlux_flag"] & ~catalog[
"base_LocalBackground_flag"]
186 & np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg))
188 minNumSources = int(self.config.minFractionSources*self.skyObjects.config.nSources)
191 if minFractionSourcesFactor != 1.0:
192 minNumSources = max(3, int(minNumSources*minFractionSourcesFactor))
193 if good.sum() < minNumSources:
195 msg = (f
"Insufficient good sky source flux measurements ({good.sum()} < "
196 f
"{minNumSources}) for dynamic threshold calculation.")
198 msg = (f
"Insufficient good sky source flux measurements ({good.sum()} < "
199 f
"{minNumSources}) for background tweak calculation.")
201 nPix = exposure.mask.array.size
203 nGoodPix = np.sum(exposure.mask.array & badPixelMask == 0)
204 if nGoodPix/nPix > 0.2:
206 nDetectedPix = np.sum(exposure.mask.array & detectedPixelMask != 0)
207 msg += (f
" However, {nGoodPix}/{nPix} pixels are not marked NO_DATA or BAD, "
208 "so there should be sufficient area to locate suitable sky sources. "
209 f
"Note that {nDetectedPix} of {nGoodPix} \"good\" pixels were marked "
210 "as DETECTED or DETECTED_NEGATIVE.")
211 raise RuntimeError(msg)
212 raise NoWorkFound(msg)
215 self.log.info(
"Number of good sky sources used for dynamic detection: %d (of %d requested).",
216 good.sum(), self.skyObjects.config.nSources)
218 self.log.info(
"Number of good sky sources used for dynamic detection background tweak:"
219 " %d (of %d requested).", good.sum(), self.skyObjects.config.nSources)
220 bgMedian = np.median((fluxes/area)[good])
222 lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0])
223 stdevMeas = 0.741*(uq - lq)
224 medianError = np.median(catalog[
"base_PsfFlux_instFluxErr"][good])
226 exposure.setWcs(
None)
227 return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian)
229 def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None):
230 """Detect footprints with a dynamic threshold
232 This varies from the vanilla ``detectFootprints`` method because we
233 do detection three times: first
with a high threshold to detect
234 "bright" (both positive
and negative, the latter to identify very
235 over-subtracted regions) sources
for which we grow the DETECTED
and
236 DETECTED_NEGATIVE masks significantly to account
for wings. Second,
237 with a low threshold to mask all non-empty regions of the image. These
238 two masks are combined
and used to identify regions of sky
239 uncontaminated by objects. A final round of detection
is then done
240 with the new calculated threshold.
245 Exposure to process; DETECTED{,_NEGATIVE} mask plane will be
247 doSmooth : `bool`, optional
248 If
True, smooth the image before detection using a Gaussian
250 sigma : `float`, optional
251 Gaussian Sigma of PSF (pixels); used
for smoothing
and to grow
252 detections;
if `
None` then measure the sigma of the PSF of the
254 clearMask : `bool`, optional
255 Clear both DETECTED
and DETECTED_NEGATIVE planes before running
257 expId : `int`, optional
258 Exposure identifier, used
as a seed
for the random number
259 generator. If absent, the seed will be the sum of the image.
263 resutls : `lsst.pipe.base.Struct`
264 The results `~lsst.pipe.base.Struct` contains:
267 Positive polarity footprints.
270 Negative polarity footprints.
273 Number of footprints
in positive
or 0
if detection polarity was
276 Number of footprints
in negative
or 0
if detection polarity was
279 Re-estimated background. `
None`
if
280 ``reEstimateBackground==
False``.
281 (`lsst.afw.math.BackgroundList`)
283 Multiplication factor applied to the configured detection
286 Results
from preliminary detection
pass.
287 (`lsst.pipe.base.Struct`)
289 maskedImage = exposure.maskedImage
294 oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask([
"DETECTED",
295 "DETECTED_NEGATIVE"])
296 nPix = maskedImage.mask.array.size
298 nGoodPix = np.sum(maskedImage.mask.array & badPixelMask == 0)
299 self.log.info(
"Number of good data pixels (i.e. not NO_DATA or BAD): {} ({:.1f}% of total)".
300 format(nGoodPix, 100*nGoodPix/nPix))
306 psf = self.
getPsf(exposure, sigma=sigma)
307 convolveResults = self.
convolveImage(maskedImage, psf, doSmooth=doSmooth)
309 if self.config.doBrightPrelimDetection:
312 middle = convolveResults.middle
313 sigma = convolveResults.sigma
315 middle, maskedImage.getBBox(), factor=self.config.prelimThresholdFactor,
316 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor
319 maskedImage.mask, prelim, sigma, factor=self.config.prelimThresholdFactor,
320 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor
322 if self.config.doBrightPrelimDetection:
325 maskedImage.mask.array |= brightDetectedMask
329 seed = (expId
if expId
is not None else int(maskedImage.image.array.sum())) % (2**31 - 1)
331 factor = threshResults.multiplicative
332 self.log.info(
"Modifying configured detection threshold by factor %f to %f",
333 factor, factor*self.config.thresholdValue)
338 maskedImage.mask.array |= oldDetected
341 results = self.
applyThreshold(middle, maskedImage.getBBox(), factor)
342 results.prelim = prelim
343 results.background = lsst.afw.math.BackgroundList()
344 if self.config.doTempLocalBackground:
350 if self.config.reEstimateBackground:
353 self.
display(exposure, results, middle)
355 if self.config.doBackgroundTweak:
364 originalMask = maskedImage.mask.array.copy()
367 convolveResults = self.
convolveImage(maskedImage, psf, doSmooth=doSmooth)
368 tweakDetResults = self.
applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor)
370 bgLevel = self.
calculateThreshold(exposure, seed, sigma=sigma, minFractionSourcesFactor=0.5,
371 isBgTweak=
True).additive
373 maskedImage.mask.array[:] = originalMask
379 """Modify the background by a constant value
384 Exposure for which to tweak background.
386 Background level to remove
387 bgList : `lsst.afw.math.BackgroundList`, optional
388 List of backgrounds to append to.
393 Constant background model.
395 self.log.info("Tweaking background by %f to match sky photometry", bgLevel)
396 exposure.image -= bgLevel
397 bgStats = lsst.afw.image.MaskedImageF(1, 1)
398 bgStats.set(bgLevel, 0, bgLevel)
400 bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER,
401 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0,
False)
402 if bgList
is not None:
403 bgList.append(bgData)
406 def _computeBrightDetectionMask(self, maskedImage, convolveResults):
407 """Perform an initial bright source detection pass.
409 Perform an initial bright object detection pass using a high detection
410 threshold. The footprints
in this
pass are grown significantly more
411 than
is typical to account
for wings around bright sources. The
412 negative polarity detections
in this
pass help
in masking severely
413 over-subtracted regions.
415 A maximum fraction of masked pixel
from this
pass is ensured via
416 the config ``brightMaskFractionMax``. If the masked pixel fraction
is
417 above this value, the detection thresholds here are increased by
418 ``bisectFactor``
in a
while loop until the detected masked fraction
419 falls below this value.
424 Masked image on which to run the detection.
425 convolveResults : `lsst.pipe.base.Struct`
426 The results of the self.
convolveImage function
with attributes:
429 Convolved image, without the edges
432 Gaussian sigma used
for the convolution (`float`).
436 brightDetectedMask : `numpy.ndarray`
437 Boolean array representing the union of the bright detection
pass
438 DETECTED
and DETECTED_NEGATIVE masks.
442 self.config.prelimThresholdFactor*self.config.brightMultiplier/self.config.bisectFactor
444 brightNegFactor = self.config.brightNegFactor/self.config.bisectFactor
448 brightMaskFractionMax = self.config.brightMaskFractionMax
454 while nPixDetNeg/nPix > brightMaskFractionMax
or nPixDet/nPix > brightMaskFractionMax:
456 brightPosFactor *= self.config.bisectFactor
457 brightNegFactor *= self.config.bisectFactor
458 prelimBright = self.
applyThreshold(convolveResults.middle, maskedImage.getBBox(),
459 factor=brightPosFactor, factorNeg=brightNegFactor)
461 maskedImage.mask, prelimBright, convolveResults.sigma*self.config.brightGrowFactor,
462 factor=brightPosFactor, factorNeg=brightNegFactor
465 nPix = maskedImage.mask.array.size
467 self.log.info(
"Number (%) of bright DETECTED pix: {} ({:.1f}%)".
468 format(nPixDet, 100*nPixDet/nPix))
470 self.log.info(
"Number (%) of bright DETECTED_NEGATIVE pix: {} ({:.1f}%)".
471 format(nPixDetNeg, 100*nPixDetNeg/nPix))
472 if nPixDetNeg/nPix > brightMaskFractionMax
or nPixDet/nPix > brightMaskFractionMax:
473 self.log.warn(
"Too high a fraction (%.1f > %.1f) of pixels were masked with current "
474 "\"bright\" detection round thresholds. Increasing by a factor of %f "
475 "and trying again.", max(nPixDetNeg, nPixDet)/nPix,
476 brightMaskFractionMax, self.config.bisectFactor)
480 brightDetectedMask = (maskedImage.mask.array
481 & maskedImage.mask.getPlaneBitMask([
"DETECTED",
"DETECTED_NEGATIVE"]))
483 return brightDetectedMask
487 """Count the number of pixels in a given mask plane.
492 Masked image to examine.
494 Name of the mask plane to examine.
499 Number of pixels with ``maskPlane`` bit set.
501 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
502 nPixMasked = np.sum(np.bitwise_and(maskedIm.mask.array, maskBit))/maskBit
static MaskPixelT getPlaneBitMask(const std::vector< std::string > &names)
def tempWideBackgroundContext(self, exposure)
def getPsf(self, exposure, sigma=None)
def run(self, table, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None)
def convolveImage(self, maskedImage, psf, doSmooth=True)
def applyTempLocalBackground(self, exposure, middle, results)
def reEstimateBackground(self, maskedImage, backgrounds)
def applyThreshold(self, middle, bbox, factor=1.0, factorNeg=None)
def clearUnwantedResults(self, mask, results)
def clearMask(self, mask)
def display(self, exposure, results, convolvedImage=None)
def finalizeFootprints(self, mask, results, sigma, factor=1.0, factorNeg=None)
def calculateThreshold(self, exposure, seed, sigma=None, minFractionSourcesFactor=1.0, isBgTweak=False)
def __init__(self, *args, **kwargs)
def tweakBackground(self, exposure, bgLevel, bgList=None)
def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None)
def _computeBrightDetectionMask(self, maskedImage, convolveResults)
def countMaskedPixels(maskedIm, maskPlane)