Coverage for python/lsst/meas/algorithms/dynamicDetection.py: 16%

170 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-25 00:22 -0700

1 

2__all__ = ["DynamicDetectionConfig", "DynamicDetectionTask"] 

3 

4import numpy as np 

5 

6from lsst.pex.config import Field, ConfigurableField 

7from lsst.pipe.base import Struct, NoWorkFound 

8 

9from .detection import SourceDetectionConfig, SourceDetectionTask 

10from .skyObjects import SkyObjectsTask 

11 

12from lsst.afw.detection import FootprintSet 

13from lsst.afw.table import SourceCatalog, SourceTable 

14from lsst.meas.base import ForcedMeasurementTask 

15 

16import lsst.afw.image 

17import lsst.afw.math 

18 

19 

20class DynamicDetectionConfig(SourceDetectionConfig): 

21 """Configuration for DynamicDetectionTask 

22 """ 

23 prelimThresholdFactor = Field(dtype=float, default=0.5, 

24 doc="Factor by which to multiply the main detection threshold " 

25 "(thresholdValue) to use for first pass (to find sky objects).") 

26 prelimNegMultiplier = Field(dtype=float, default=2.5, 

27 doc="Multiplier for the negative (relative to positive) polarity " 

28 "detections threshold to use for first pass (to find sky objects).") 

29 skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects.") 

30 doBackgroundTweak = Field(dtype=bool, default=True, 

31 doc="Tweak background level so median PSF flux of sky objects is zero?") 

32 minFractionSources = Field(dtype=float, default=0.02, 

33 doc="Minimum fraction of the requested number of sky sources for dynamic " 

34 "detection to be considered a success. If the number of good sky sources " 

35 "identified falls below this threshold, a NoWorkFound error is raised so " 

36 "that this dataId is no longer considered in downstream processing.") 

37 doBrightPrelimDetection = Field(dtype=bool, default=True, 

38 doc="Do initial bright detection pass where footprints are grown " 

39 "by brightGrowFactor?") 

40 brightMultiplier = Field(dtype=float, default=2000.0, 

41 doc="Multiplier to apply to the prelimThresholdFactor for the " 

42 "\"bright\" detections stage (want this to be large to only " 

43 "detect the brightest sources).") 

44 brightNegFactor = Field(dtype=float, default=2.2, 

45 doc="Factor by which to multiply the threshold for the negative polatiry " 

46 "detections for the \"bright\" detections stage (this needs to be fairly " 

47 "low given the nature of the negative polarity detections in the very " 

48 "large positive polarity threshold).") 

49 brightGrowFactor = Field(dtype=int, default=40, 

50 doc="Factor by which to grow the footprints of sources detected in the " 

51 "\"bright\" detections stage (want this to be large to mask wings of " 

52 "bright sources).") 

53 brightMaskFractionMax = Field(dtype=float, default=0.95, 

54 doc="Maximum allowed fraction of masked pixes from the \"bright\" " 

55 "detection stage (to mask regions unsuitable for sky sourcess). " 

56 "If this fraction is exeeded, the detection threshold for this stage " 

57 "will be increased by bisectFactor until the fraction of masked " 

58 "pixels drops below this threshold.") 

59 bisectFactor = Field(dtype=float, default=1.2, 

60 doc="Factor by which to increase thresholds in brightMaskFractionMax loop.") 

61 

62 def setDefaults(self): 

63 SourceDetectionConfig.setDefaults(self) 

64 self.skyObjects.nSources = 1000 # For good statistics 

65 for maskStr in ["SAT"]: 

66 if maskStr not in self.skyObjects.avoidMask: 

67 self.skyObjects.avoidMask.append(maskStr) 

68 

69 

70class DynamicDetectionTask(SourceDetectionTask): 

71 """Detection of sources on an image with a dynamic threshold 

72 

73 We first detect sources using a lower threshold than normal (see config 

74 parameter ``prelimThresholdFactor``) in order to identify good sky regions 

75 (configurable ``skyObjects``). Then we perform forced PSF photometry on 

76 those sky regions. Using those PSF flux measurements and estimated errors, 

77 we set the threshold so that the stdev of the measurements matches the 

78 median estimated error. 

79 

80 Besides the usual initialisation of configurables, we also set up 

81 the forced measurement which is deliberately not represented in 

82 this Task's configuration parameters because we're using it as 

83 part of the algorithm and we don't want to allow it to be modified. 

84 """ 

85 ConfigClass = DynamicDetectionConfig 

86 _DefaultName = "dynamicDetection" 

87 

88 def __init__(self, *args, **kwargs): 

89 

90 SourceDetectionTask.__init__(self, *args, **kwargs) 

91 self.makeSubtask("skyObjects") 

92 

93 # Set up forced measurement. 

94 config = ForcedMeasurementTask.ConfigClass() 

95 config.plugins.names = ['base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground'] 

96 # We'll need the "centroid" and "psfFlux" slots 

97 for slot in ("shape", "psfShape", "apFlux", "modelFlux", "gaussianFlux", "calibFlux"): 

98 setattr(config.slots, slot, None) 

99 config.copyColumns = {} 

100 self.skySchema = SourceTable.makeMinimalSchema() 

101 self.skyMeasurement = ForcedMeasurementTask(config=config, name="skyMeasurement", parentTask=self, 

102 refSchema=self.skySchema) 

103 

104 def calculateThreshold(self, exposure, seed, sigma=None, minFractionSourcesFactor=1.0, isBgTweak=False): 

105 """Calculate new threshold 

106 

107 This is the main functional addition to the vanilla 

108 `SourceDetectionTask`. 

109 

110 We identify sky objects and perform forced PSF photometry on 

111 them. Using those PSF flux measurements and estimated errors, 

112 we set the threshold so that the stdev of the measurements 

113 matches the median estimated error. 

114 

115 Parameters 

116 ---------- 

117 exposure : `lsst.afw.image.Exposure` 

118 Exposure on which we're detecting sources. 

119 seed : `int` 

120 RNG seed to use for finding sky objects. 

121 sigma : `float`, optional 

122 Gaussian sigma of smoothing kernel; if not provided, 

123 will be deduced from the exposure's PSF. 

124 minFractionSourcesFactor : `float` 

125 Change the fraction of required sky sources from that set in 

126 ``self.config.minFractionSources`` by this factor. NOTE: this 

127 is intended for use in the background tweak pass (the detection 

128 threshold is much lower there, so many more pixels end up marked 

129 as DETECTED or DETECTED_NEGATIVE, leaving less room for sky 

130 object placement). 

131 isBgTweak : `bool` 

132 Set to ``True`` for the background tweak pass (for more helpful 

133 log messages). 

134 

135 Returns 

136 ------- 

137 result : `lsst.pipe.base.Struct` 

138 Result struct with components: 

139 

140 - ``multiplicative``: multiplicative factor to be applied to the 

141 configured detection threshold (`float`). 

142 - ``additive``: additive factor to be applied to the background 

143 level (`float`). 

144 

145 Raises 

146 ------ 

147 NoWorkFound 

148 Raised if the number of good sky sources found is less than the 

149 minimum fraction 

150 (``self.config.minFractionSources``*``minFractionSourcesFactor``) 

151 of the number requested (``self.skyObjects.config.nSources``). 

152 """ 

153 # Make a catalog of sky objects 

154 fp = self.skyObjects.run(exposure.maskedImage.mask, seed) 

155 skyFootprints = FootprintSet(exposure.getBBox()) 

156 skyFootprints.setFootprints(fp) 

157 table = SourceTable.make(self.skyMeasurement.schema) 

158 catalog = SourceCatalog(table) 

159 catalog.reserve(len(skyFootprints.getFootprints())) 

160 skyFootprints.makeSources(catalog) 

161 key = catalog.getCentroidSlot().getMeasKey() 

162 for source in catalog: 

163 peaks = source.getFootprint().getPeaks() 

164 assert len(peaks) == 1 

165 source.set(key, peaks[0].getF()) 

166 source.updateCoord(exposure.getWcs()) 

167 

168 # Forced photometry on sky objects 

169 self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs()) 

170 

171 # Calculate new threshold 

172 fluxes = catalog["base_PsfFlux_instFlux"] 

173 area = catalog["base_PsfFlux_area"] 

174 bg = catalog["base_LocalBackground_instFlux"] 

175 

176 good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] 

177 & np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg)) 

178 

179 minNumSources = int(self.config.minFractionSources*self.skyObjects.config.nSources) 

180 # Reduce the number of sky sources required if requested, but ensure 

181 # a minumum of 3. 

182 if minFractionSourcesFactor != 1.0: 

183 minNumSources = max(3, int(minNumSources*minFractionSourcesFactor)) 

184 if good.sum() < minNumSources: 

185 if not isBgTweak: 

186 msg = (f"Insufficient good sky source flux measurements ({good.sum()} < " 

187 f"{minNumSources}) for dynamic threshold calculation.") 

188 else: 

189 msg = (f"Insufficient good sky source flux measurements ({good.sum()} < " 

190 f"{minNumSources}) for background tweak calculation.") 

191 

192 nPix = exposure.mask.array.size 

193 badPixelMask = lsst.afw.image.Mask.getPlaneBitMask(["NO_DATA", "BAD"]) 

194 nGoodPix = np.sum(exposure.mask.array & badPixelMask == 0) 

195 if nGoodPix/nPix > 0.2: 

196 detectedPixelMask = lsst.afw.image.Mask.getPlaneBitMask(["DETECTED", "DETECTED_NEGATIVE"]) 

197 nDetectedPix = np.sum(exposure.mask.array & detectedPixelMask != 0) 

198 msg += (f" However, {nGoodPix}/{nPix} pixels are not marked NO_DATA or BAD, " 

199 "so there should be sufficient area to locate suitable sky sources. " 

200 f"Note that {nDetectedPix} of {nGoodPix} \"good\" pixels were marked " 

201 "as DETECTED or DETECTED_NEGATIVE.") 

202 raise RuntimeError(msg) 

203 raise NoWorkFound(msg) 

204 

205 if not isBgTweak: 

206 self.log.info("Number of good sky sources used for dynamic detection: %d (of %d requested).", 

207 good.sum(), self.skyObjects.config.nSources) 

208 else: 

209 self.log.info("Number of good sky sources used for dynamic detection background tweak:" 

210 " %d (of %d requested).", good.sum(), self.skyObjects.config.nSources) 

211 bgMedian = np.median((fluxes/area)[good]) 

212 

213 lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0]) 

214 stdevMeas = 0.741*(uq - lq) 

215 medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good]) 

216 return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian) 

217 

218 def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None): 

219 """Detect footprints with a dynamic threshold 

220 

221 This varies from the vanilla ``detectFootprints`` method because we 

222 do detection three times: first with a high threshold to detect 

223 "bright" (both positive and negative, the latter to identify very 

224 over-subtracted regions) sources for which we grow the DETECTED and 

225 DETECTED_NEGATIVE masks significantly to account for wings. Second, 

226 with a low threshold to mask all non-empty regions of the image. These 

227 two masks are combined and used to identify regions of sky 

228 uncontaminated by objects. A final round of detection is then done 

229 with the new calculated threshold. 

230 

231 Parameters 

232 ---------- 

233 exposure : `lsst.afw.image.Exposure` 

234 Exposure to process; DETECTED{,_NEGATIVE} mask plane will be 

235 set in-place. 

236 doSmooth : `bool`, optional 

237 If True, smooth the image before detection using a Gaussian 

238 of width ``sigma``. 

239 sigma : `float`, optional 

240 Gaussian Sigma of PSF (pixels); used for smoothing and to grow 

241 detections; if `None` then measure the sigma of the PSF of the 

242 ``exposure``. 

243 clearMask : `bool`, optional 

244 Clear both DETECTED and DETECTED_NEGATIVE planes before running 

245 detection. 

246 expId : `int`, optional 

247 Exposure identifier, used as a seed for the random number 

248 generator. If absent, the seed will be the sum of the image. 

249 

250 Return Struct contents 

251 ---------------------- 

252 positive : `lsst.afw.detection.FootprintSet` 

253 Positive polarity footprints (may be `None`) 

254 negative : `lsst.afw.detection.FootprintSet` 

255 Negative polarity footprints (may be `None`) 

256 numPos : `int` 

257 Number of footprints in positive or 0 if detection polarity was 

258 negative. 

259 numNeg : `int` 

260 Number of footprints in negative or 0 if detection polarity was 

261 positive. 

262 background : `lsst.afw.math.BackgroundList` 

263 Re-estimated background. `None` if 

264 ``reEstimateBackground==False``. 

265 factor : `float` 

266 Multiplication factor applied to the configured detection 

267 threshold. 

268 prelim : `lsst.pipe.base.Struct` 

269 Results from preliminary detection pass. 

270 """ 

271 maskedImage = exposure.maskedImage 

272 

273 if clearMask: 

274 self.clearMask(maskedImage.mask) 

275 else: 

276 oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(["DETECTED", 

277 "DETECTED_NEGATIVE"]) 

278 nPix = maskedImage.mask.array.size 

279 badPixelMask = lsst.afw.image.Mask.getPlaneBitMask(["NO_DATA", "BAD"]) 

280 nGoodPix = np.sum(maskedImage.mask.array & badPixelMask == 0) 

281 self.log.info("Number of good data pixels (i.e. not NO_DATA or BAD): {} ({:.1f}% of total)". 

282 format(nGoodPix, 100*nGoodPix/nPix)) 

283 

284 with self.tempWideBackgroundContext(exposure): 

285 # Could potentially smooth with a wider kernel than the PSF in order to better pick up the 

286 # wings of stars and galaxies, but for now sticking with the PSF as that's more simple. 

287 psf = self.getPsf(exposure, sigma=sigma) 

288 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

289 

290 if self.config.doBrightPrelimDetection: 

291 brightDetectedMask = self._computeBrightDetectionMask(maskedImage, convolveResults) 

292 

293 middle = convolveResults.middle 

294 sigma = convolveResults.sigma 

295 prelim = self.applyThreshold( 

296 middle, maskedImage.getBBox(), factor=self.config.prelimThresholdFactor, 

297 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor 

298 ) 

299 self.finalizeFootprints( 

300 maskedImage.mask, prelim, sigma, factor=self.config.prelimThresholdFactor, 

301 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor 

302 ) 

303 if self.config.doBrightPrelimDetection: 

304 # Combine prelim and bright detection masks for multiplier 

305 # determination. 

306 maskedImage.mask.array |= brightDetectedMask 

307 

308 # Calculate the proper threshold 

309 # seed needs to fit in a C++ 'int' so pybind doesn't choke on it 

310 seed = (expId if expId is not None else int(maskedImage.image.array.sum())) % (2**31 - 1) 

311 threshResults = self.calculateThreshold(exposure, seed, sigma=sigma) 

312 factor = threshResults.multiplicative 

313 self.log.info("Modifying configured detection threshold by factor %f to %f", 

314 factor, factor*self.config.thresholdValue) 

315 

316 # Blow away preliminary (low threshold) detection mask 

317 self.clearMask(maskedImage.mask) 

318 if not clearMask: 

319 maskedImage.mask.array |= oldDetected 

320 

321 # Rinse and repeat thresholding with new calculated threshold 

322 results = self.applyThreshold(middle, maskedImage.getBBox(), factor) 

323 results.prelim = prelim 

324 results.background = lsst.afw.math.BackgroundList() 

325 if self.config.doTempLocalBackground: 

326 self.applyTempLocalBackground(exposure, middle, results) 

327 self.finalizeFootprints(maskedImage.mask, results, sigma, factor=factor) 

328 

329 self.clearUnwantedResults(maskedImage.mask, results) 

330 

331 if self.config.reEstimateBackground: 

332 self.reEstimateBackground(maskedImage, results.background) 

333 

334 self.display(exposure, results, middle) 

335 

336 if self.config.doBackgroundTweak: 

337 # Re-do the background tweak after any temporary backgrounds have been restored 

338 # 

339 # But we want to keep any large-scale background (e.g., scattered light from bright stars) 

340 # from being selected for sky objects in the calculation, so do another detection pass without 

341 # either the local or wide temporary background subtraction; the DETECTED pixels will mark 

342 # the area to ignore. 

343 originalMask = maskedImage.mask.array.copy() 

344 try: 

345 self.clearMask(exposure.mask) 

346 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

347 tweakDetResults = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor) 

348 self.finalizeFootprints(maskedImage.mask, tweakDetResults, sigma, factor=factor) 

349 bgLevel = self.calculateThreshold(exposure, seed, sigma=sigma, minFractionSourcesFactor=0.5, 

350 isBgTweak=True).additive 

351 finally: 

352 maskedImage.mask.array[:] = originalMask 

353 self.tweakBackground(exposure, bgLevel, results.background) 

354 

355 return results 

356 

357 def tweakBackground(self, exposure, bgLevel, bgList=None): 

358 """Modify the background by a constant value 

359 

360 Parameters 

361 ---------- 

362 exposure : `lsst.afw.image.Exposure` 

363 Exposure for which to tweak background. 

364 bgLevel : `float` 

365 Background level to remove 

366 bgList : `lsst.afw.math.BackgroundList`, optional 

367 List of backgrounds to append to. 

368 

369 Returns 

370 ------- 

371 bg : `lsst.afw.math.BackgroundMI` 

372 Constant background model. 

373 """ 

374 self.log.info("Tweaking background by %f to match sky photometry", bgLevel) 

375 exposure.image -= bgLevel 

376 bgStats = lsst.afw.image.MaskedImageF(1, 1) 

377 bgStats.set(bgLevel, 0, bgLevel) 

378 bg = lsst.afw.math.BackgroundMI(exposure.getBBox(), bgStats) 

379 bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER, 

380 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0, False) 

381 if bgList is not None: 

382 bgList.append(bgData) 

383 return bg 

384 

385 def _computeBrightDetectionMask(self, maskedImage, convolveResults): 

386 """Perform an initial bright source detection pass. 

387 

388 Perform an initial bright object detection pass using a high detection 

389 threshold. The footprints in this pass are grown significantly more 

390 than is typical to account for wings around bright sources. The 

391 negative polarity detections in this pass help in masking severely 

392 over-subtracted regions. 

393 

394 A maximum fraction of masked pixel from this pass is ensured via 

395 the config ``brightMaskFractionMax``. If the masked pixel fraction is 

396 above this value, the detection thresholds here are increased by 

397 ``bisectFactor`` in a while loop until the detected masked fraction 

398 falls below this value. 

399 

400 Parameters 

401 ---------- 

402 maskedImage : `lsst.afw.image.MaskedImage` 

403 Masked image on which to run the detection. 

404 convolveResults : `lsst.pipe.base.Struct` 

405 The results of the self.convolveImage function with attributes: 

406 

407 ``middle`` 

408 Convolved image, without the edges 

409 (`lsst.afw.image.MaskedImage`). 

410 ``sigma`` 

411 Gaussian sigma used for the convolution (`float`). 

412 

413 Returns 

414 ------- 

415 brightDetectedMask : `numpy.ndarray` 

416 Boolean array representing the union of the bright detection pass 

417 DETECTED and DETECTED_NEGATIVE masks. 

418 """ 

419 # Initialize some parameters. 

420 brightPosFactor = ( 

421 self.config.prelimThresholdFactor*self.config.brightMultiplier/self.config.bisectFactor 

422 ) 

423 brightNegFactor = self.config.brightNegFactor/self.config.bisectFactor 

424 nPix = 1 

425 nPixDet = 1 

426 nPixDetNeg = 1 

427 brightMaskFractionMax = self.config.brightMaskFractionMax 

428 

429 # Loop until masked fraction is smaller than 

430 # brightMaskFractionMax, increasing the thresholds by 

431 # config.bisectFactor on each iteration (rarely necessary 

432 # for current defaults). 

433 while nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax: 

434 self.clearMask(maskedImage.mask) 

435 brightPosFactor *= self.config.bisectFactor 

436 brightNegFactor *= self.config.bisectFactor 

437 prelimBright = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), 

438 factor=brightPosFactor, factorNeg=brightNegFactor) 

439 self.finalizeFootprints( 

440 maskedImage.mask, prelimBright, convolveResults.sigma*self.config.brightGrowFactor, 

441 factor=brightPosFactor, factorNeg=brightNegFactor 

442 ) 

443 # Check that not too many pixels got masked. 

444 nPix = maskedImage.mask.array.size 

445 nPixDet = countMaskedPixels(maskedImage, "DETECTED") 

446 self.log.info("Number (%) of bright DETECTED pix: {} ({:.1f}%)". 

447 format(nPixDet, 100*nPixDet/nPix)) 

448 nPixDetNeg = countMaskedPixels(maskedImage, "DETECTED_NEGATIVE") 

449 self.log.info("Number (%) of bright DETECTED_NEGATIVE pix: {} ({:.1f}%)". 

450 format(nPixDetNeg, 100*nPixDetNeg/nPix)) 

451 if nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax: 

452 self.log.warn("Too high a fraction (%.1f > %.1f) of pixels were masked with current " 

453 "\"bright\" detection round thresholds. Increasing by a factor of %f " 

454 "and trying again.", max(nPixDetNeg, nPixDet)/nPix, 

455 brightMaskFractionMax, self.config.bisectFactor) 

456 

457 # Save the mask planes from the "bright" detection round, then 

458 # clear them before moving on to the "prelim" detection phase. 

459 brightDetectedMask = (maskedImage.mask.array 

460 & maskedImage.mask.getPlaneBitMask(["DETECTED", "DETECTED_NEGATIVE"])) 

461 self.clearMask(maskedImage.mask) 

462 return brightDetectedMask 

463 

464 

465def countMaskedPixels(maskedIm, maskPlane): 

466 """Count the number of pixels in a given mask plane. 

467 

468 Parameters 

469 ---------- 

470 maskedIm : `lsst.afw.image.MaskedImage` 

471 Masked image to examine. 

472 maskPlane : `str` 

473 Name of the mask plane to examine. 

474 

475 Returns 

476 ------- 

477 nPixMasked : `int` 

478 Number of pixels with ``maskPlane`` bit set. 

479 """ 

480 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

481 nPixMasked = np.sum(np.bitwise_and(maskedIm.mask.array, maskBit))/maskBit 

482 return nPixMasked