Coverage for python/lsst/meas/algorithms/dynamicDetection.py: 16%

178 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2024-01-17 19:18 +0000

1 

2__all__ = ["DynamicDetectionConfig", "DynamicDetectionTask"] 

3 

4import numpy as np 

5 

6from lsst.pex.config import Field, ConfigurableField 

7from lsst.pipe.base import Struct, NoWorkFound 

8 

9from .detection import SourceDetectionConfig, SourceDetectionTask 

10from .skyObjects import SkyObjectsTask 

11 

12from lsst.afw.detection import FootprintSet 

13from lsst.afw.geom import makeCdMatrix, makeSkyWcs 

14from lsst.afw.table import SourceCatalog, SourceTable 

15from lsst.meas.base import ForcedMeasurementTask 

16 

17import lsst.afw.image 

18import lsst.afw.math 

19import lsst.geom as geom 

20 

21 

22class DynamicDetectionConfig(SourceDetectionConfig): 

23 """Configuration for DynamicDetectionTask 

24 """ 

25 prelimThresholdFactor = Field(dtype=float, default=0.5, 

26 doc="Factor by which to multiply the main detection threshold " 

27 "(thresholdValue) to use for first pass (to find sky objects).") 

28 prelimNegMultiplier = Field(dtype=float, default=2.5, 

29 doc="Multiplier for the negative (relative to positive) polarity " 

30 "detections threshold to use for first pass (to find sky objects).") 

31 skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects.") 

32 doBackgroundTweak = Field(dtype=bool, default=True, 

33 doc="Tweak background level so median PSF flux of sky objects is zero?") 

34 minFractionSources = Field(dtype=float, default=0.02, 

35 doc="Minimum fraction of the requested number of sky sources for dynamic " 

36 "detection to be considered a success. If the number of good sky sources " 

37 "identified falls below this threshold, a NoWorkFound error is raised so " 

38 "that this dataId is no longer considered in downstream processing.") 

39 doBrightPrelimDetection = Field(dtype=bool, default=True, 

40 doc="Do initial bright detection pass where footprints are grown " 

41 "by brightGrowFactor?") 

42 brightMultiplier = Field(dtype=float, default=2000.0, 

43 doc="Multiplier to apply to the prelimThresholdFactor for the " 

44 "\"bright\" detections stage (want this to be large to only " 

45 "detect the brightest sources).") 

46 brightNegFactor = Field(dtype=float, default=2.2, 

47 doc="Factor by which to multiply the threshold for the negative polatiry " 

48 "detections for the \"bright\" detections stage (this needs to be fairly " 

49 "low given the nature of the negative polarity detections in the very " 

50 "large positive polarity threshold).") 

51 brightGrowFactor = Field(dtype=int, default=40, 

52 doc="Factor by which to grow the footprints of sources detected in the " 

53 "\"bright\" detections stage (want this to be large to mask wings of " 

54 "bright sources).") 

55 brightMaskFractionMax = Field(dtype=float, default=0.95, 

56 doc="Maximum allowed fraction of masked pixes from the \"bright\" " 

57 "detection stage (to mask regions unsuitable for sky sourcess). " 

58 "If this fraction is exeeded, the detection threshold for this stage " 

59 "will be increased by bisectFactor until the fraction of masked " 

60 "pixels drops below this threshold.") 

61 bisectFactor = Field(dtype=float, default=1.2, 

62 doc="Factor by which to increase thresholds in brightMaskFractionMax loop.") 

63 

64 def setDefaults(self): 

65 SourceDetectionConfig.setDefaults(self) 

66 self.skyObjects.nSources = 1000 # For good statistics 

67 for maskStr in ["SAT"]: 

68 if maskStr not in self.skyObjects.avoidMask: 

69 self.skyObjects.avoidMask.append(maskStr) 

70 

71 

72class DynamicDetectionTask(SourceDetectionTask): 

73 """Detection of sources on an image with a dynamic threshold 

74 

75 We first detect sources using a lower threshold than normal (see config 

76 parameter ``prelimThresholdFactor``) in order to identify good sky regions 

77 (configurable ``skyObjects``). Then we perform forced PSF photometry on 

78 those sky regions. Using those PSF flux measurements and estimated errors, 

79 we set the threshold so that the stdev of the measurements matches the 

80 median estimated error. 

81 

82 Besides the usual initialisation of configurables, we also set up 

83 the forced measurement which is deliberately not represented in 

84 this Task's configuration parameters because we're using it as 

85 part of the algorithm and we don't want to allow it to be modified. 

86 """ 

87 ConfigClass = DynamicDetectionConfig 

88 _DefaultName = "dynamicDetection" 

89 

90 def __init__(self, *args, **kwargs): 

91 

92 SourceDetectionTask.__init__(self, *args, **kwargs) 

93 self.makeSubtask("skyObjects") 

94 

95 # Set up forced measurement. 

96 config = ForcedMeasurementTask.ConfigClass() 

97 config.plugins.names = ['base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground'] 

98 # We'll need the "centroid" and "psfFlux" slots 

99 for slot in ("shape", "psfShape", "apFlux", "modelFlux", "gaussianFlux", "calibFlux"): 

100 setattr(config.slots, slot, None) 

101 config.copyColumns = {} 

102 self.skySchema = SourceTable.makeMinimalSchema() 

103 self.skyMeasurement = ForcedMeasurementTask(config=config, name="skyMeasurement", parentTask=self, 

104 refSchema=self.skySchema) 

105 

106 def calculateThreshold(self, exposure, seed, sigma=None, minFractionSourcesFactor=1.0, isBgTweak=False): 

107 """Calculate new threshold 

108 

109 This is the main functional addition to the vanilla 

110 `SourceDetectionTask`. 

111 

112 We identify sky objects and perform forced PSF photometry on 

113 them. Using those PSF flux measurements and estimated errors, 

114 we set the threshold so that the stdev of the measurements 

115 matches the median estimated error. 

116 

117 Parameters 

118 ---------- 

119 exposure : `lsst.afw.image.Exposure` 

120 Exposure on which we're detecting sources. 

121 seed : `int` 

122 RNG seed to use for finding sky objects. 

123 sigma : `float`, optional 

124 Gaussian sigma of smoothing kernel; if not provided, 

125 will be deduced from the exposure's PSF. 

126 minFractionSourcesFactor : `float` 

127 Change the fraction of required sky sources from that set in 

128 ``self.config.minFractionSources`` by this factor. NOTE: this 

129 is intended for use in the background tweak pass (the detection 

130 threshold is much lower there, so many more pixels end up marked 

131 as DETECTED or DETECTED_NEGATIVE, leaving less room for sky 

132 object placement). 

133 isBgTweak : `bool` 

134 Set to ``True`` for the background tweak pass (for more helpful 

135 log messages). 

136 

137 Returns 

138 ------- 

139 result : `lsst.pipe.base.Struct` 

140 Result struct with components: 

141 

142 ``multiplicative`` 

143 Multiplicative factor to be applied to the 

144 configured detection threshold (`float`). 

145 ``additive`` 

146 Additive factor to be applied to the background 

147 level (`float`). 

148 

149 Raises 

150 ------ 

151 NoWorkFound 

152 Raised if the number of good sky sources found is less than the 

153 minimum fraction 

154 (``self.config.minFractionSources``*``minFractionSourcesFactor``) 

155 of the number requested (``self.skyObjects.config.nSources``). 

156 """ 

157 wcsIsNone = exposure.getWcs() is None 

158 if wcsIsNone: # create a dummy WCS as needed by ForcedMeasurementTask 

159 self.log.info("WCS for exposure is None. Setting a dummy WCS for dynamic detection.") 

160 exposure.setWcs(makeSkyWcs(crpix=geom.Point2D(0, 0), 

161 crval=geom.SpherePoint(0, 0, geom.degrees), 

162 cdMatrix=makeCdMatrix(scale=1e-5*geom.degrees))) 

163 fp = self.skyObjects.run(exposure.maskedImage.mask, seed) 

164 skyFootprints = FootprintSet(exposure.getBBox()) 

165 skyFootprints.setFootprints(fp) 

166 table = SourceTable.make(self.skyMeasurement.schema) 

167 catalog = SourceCatalog(table) 

168 catalog.reserve(len(skyFootprints.getFootprints())) 

169 skyFootprints.makeSources(catalog) 

170 key = catalog.getCentroidSlot().getMeasKey() 

171 for source in catalog: 

172 peaks = source.getFootprint().getPeaks() 

173 assert len(peaks) == 1 

174 source.set(key, peaks[0].getF()) 

175 source.updateCoord(exposure.getWcs()) 

176 

177 # Forced photometry on sky objects 

178 self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs()) 

179 

180 # Calculate new threshold 

181 fluxes = catalog["base_PsfFlux_instFlux"] 

182 area = catalog["base_PsfFlux_area"] 

183 bg = catalog["base_LocalBackground_instFlux"] 

184 

185 good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] 

186 & np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg)) 

187 

188 minNumSources = int(self.config.minFractionSources*self.skyObjects.config.nSources) 

189 # Reduce the number of sky sources required if requested, but ensure 

190 # a minumum of 3. 

191 if minFractionSourcesFactor != 1.0: 

192 minNumSources = max(3, int(minNumSources*minFractionSourcesFactor)) 

193 if good.sum() < minNumSources: 

194 if not isBgTweak: 

195 msg = (f"Insufficient good sky source flux measurements ({good.sum()} < " 

196 f"{minNumSources}) for dynamic threshold calculation.") 

197 else: 

198 msg = (f"Insufficient good sky source flux measurements ({good.sum()} < " 

199 f"{minNumSources}) for background tweak calculation.") 

200 

201 nPix = exposure.mask.array.size 

202 badPixelMask = lsst.afw.image.Mask.getPlaneBitMask(["NO_DATA", "BAD"]) 

203 nGoodPix = np.sum(exposure.mask.array & badPixelMask == 0) 

204 if nGoodPix/nPix > 0.2: 

205 detectedPixelMask = lsst.afw.image.Mask.getPlaneBitMask(["DETECTED", "DETECTED_NEGATIVE"]) 

206 nDetectedPix = np.sum(exposure.mask.array & detectedPixelMask != 0) 

207 msg += (f" However, {nGoodPix}/{nPix} pixels are not marked NO_DATA or BAD, " 

208 "so there should be sufficient area to locate suitable sky sources. " 

209 f"Note that {nDetectedPix} of {nGoodPix} \"good\" pixels were marked " 

210 "as DETECTED or DETECTED_NEGATIVE.") 

211 raise RuntimeError(msg) 

212 raise NoWorkFound(msg) 

213 

214 if not isBgTweak: 

215 self.log.info("Number of good sky sources used for dynamic detection: %d (of %d requested).", 

216 good.sum(), self.skyObjects.config.nSources) 

217 else: 

218 self.log.info("Number of good sky sources used for dynamic detection background tweak:" 

219 " %d (of %d requested).", good.sum(), self.skyObjects.config.nSources) 

220 bgMedian = np.median((fluxes/area)[good]) 

221 

222 lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0]) 

223 stdevMeas = 0.741*(uq - lq) 

224 medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good]) 

225 if wcsIsNone: 

226 exposure.setWcs(None) 

227 return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian) 

228 

229 def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None): 

230 """Detect footprints with a dynamic threshold 

231 

232 This varies from the vanilla ``detectFootprints`` method because we 

233 do detection three times: first with a high threshold to detect 

234 "bright" (both positive and negative, the latter to identify very 

235 over-subtracted regions) sources for which we grow the DETECTED and 

236 DETECTED_NEGATIVE masks significantly to account for wings. Second, 

237 with a low threshold to mask all non-empty regions of the image. These 

238 two masks are combined and used to identify regions of sky 

239 uncontaminated by objects. A final round of detection is then done 

240 with the new calculated threshold. 

241 

242 Parameters 

243 ---------- 

244 exposure : `lsst.afw.image.Exposure` 

245 Exposure to process; DETECTED{,_NEGATIVE} mask plane will be 

246 set in-place. 

247 doSmooth : `bool`, optional 

248 If True, smooth the image before detection using a Gaussian 

249 of width ``sigma``. 

250 sigma : `float`, optional 

251 Gaussian Sigma of PSF (pixels); used for smoothing and to grow 

252 detections; if `None` then measure the sigma of the PSF of the 

253 ``exposure``. 

254 clearMask : `bool`, optional 

255 Clear both DETECTED and DETECTED_NEGATIVE planes before running 

256 detection. 

257 expId : `int`, optional 

258 Exposure identifier, used as a seed for the random number 

259 generator. If absent, the seed will be the sum of the image. 

260 

261 Returns 

262 ------- 

263 resutls : `lsst.pipe.base.Struct` 

264 The results `~lsst.pipe.base.Struct` contains: 

265 

266 ``positive`` 

267 Positive polarity footprints. 

268 (`lsst.afw.detection.FootprintSet` or `None`) 

269 ``negative`` 

270 Negative polarity footprints. 

271 (`lsst.afw.detection.FootprintSet` or `None`) 

272 ``numPos`` 

273 Number of footprints in positive or 0 if detection polarity was 

274 negative. (`int`) 

275 ``numNeg`` 

276 Number of footprints in negative or 0 if detection polarity was 

277 positive. (`int`) 

278 ``background`` 

279 Re-estimated background. `None` if 

280 ``reEstimateBackground==False``. 

281 (`lsst.afw.math.BackgroundList`) 

282 ``factor`` 

283 Multiplication factor applied to the configured detection 

284 threshold. (`float`) 

285 ``prelim`` 

286 Results from preliminary detection pass. 

287 (`lsst.pipe.base.Struct`) 

288 """ 

289 maskedImage = exposure.maskedImage 

290 

291 if clearMask: 

292 self.clearMask(maskedImage.mask) 

293 else: 

294 oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(["DETECTED", 

295 "DETECTED_NEGATIVE"]) 

296 nPix = maskedImage.mask.array.size 

297 badPixelMask = lsst.afw.image.Mask.getPlaneBitMask(["NO_DATA", "BAD"]) 

298 nGoodPix = np.sum(maskedImage.mask.array & badPixelMask == 0) 

299 self.log.info("Number of good data pixels (i.e. not NO_DATA or BAD): {} ({:.1f}% of total)". 

300 format(nGoodPix, 100*nGoodPix/nPix)) 

301 

302 with self.tempWideBackgroundContext(exposure): 

303 # Could potentially smooth with a wider kernel than the PSF in 

304 # order to better pick up the wings of stars and galaxies, but for 

305 # now sticking with the PSF as that's more simple. 

306 psf = self.getPsf(exposure, sigma=sigma) 

307 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

308 

309 if self.config.doBrightPrelimDetection: 

310 brightDetectedMask = self._computeBrightDetectionMask(maskedImage, convolveResults) 

311 

312 middle = convolveResults.middle 

313 sigma = convolveResults.sigma 

314 prelim = self.applyThreshold( 

315 middle, maskedImage.getBBox(), factor=self.config.prelimThresholdFactor, 

316 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor 

317 ) 

318 self.finalizeFootprints( 

319 maskedImage.mask, prelim, sigma, factor=self.config.prelimThresholdFactor, 

320 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor 

321 ) 

322 if self.config.doBrightPrelimDetection: 

323 # Combine prelim and bright detection masks for multiplier 

324 # determination. 

325 maskedImage.mask.array |= brightDetectedMask 

326 

327 # Calculate the proper threshold 

328 # seed needs to fit in a C++ 'int' so pybind doesn't choke on it 

329 seed = (expId if expId is not None else int(maskedImage.image.array.sum())) % (2**31 - 1) 

330 threshResults = self.calculateThreshold(exposure, seed, sigma=sigma) 

331 factor = threshResults.multiplicative 

332 self.log.info("Modifying configured detection threshold by factor %f to %f", 

333 factor, factor*self.config.thresholdValue) 

334 

335 # Blow away preliminary (low threshold) detection mask 

336 self.clearMask(maskedImage.mask) 

337 if not clearMask: 

338 maskedImage.mask.array |= oldDetected 

339 

340 # Rinse and repeat thresholding with new calculated threshold 

341 results = self.applyThreshold(middle, maskedImage.getBBox(), factor) 

342 results.prelim = prelim 

343 results.background = lsst.afw.math.BackgroundList() 

344 if self.config.doTempLocalBackground: 

345 self.applyTempLocalBackground(exposure, middle, results) 

346 self.finalizeFootprints(maskedImage.mask, results, sigma, factor=factor) 

347 

348 self.clearUnwantedResults(maskedImage.mask, results) 

349 

350 if self.config.reEstimateBackground: 

351 self.reEstimateBackground(maskedImage, results.background) 

352 

353 self.display(exposure, results, middle) 

354 

355 if self.config.doBackgroundTweak: 

356 # Re-do the background tweak after any temporary backgrounds have 

357 # been restored. 

358 # 

359 # But we want to keep any large-scale background (e.g., scattered 

360 # light from bright stars) from being selected for sky objects in 

361 # the calculation, so do another detection pass without either the 

362 # local or wide temporary background subtraction; the DETECTED 

363 # pixels will mark the area to ignore. 

364 originalMask = maskedImage.mask.array.copy() 

365 try: 

366 self.clearMask(exposure.mask) 

367 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

368 tweakDetResults = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor) 

369 self.finalizeFootprints(maskedImage.mask, tweakDetResults, sigma, factor=factor) 

370 bgLevel = self.calculateThreshold(exposure, seed, sigma=sigma, minFractionSourcesFactor=0.5, 

371 isBgTweak=True).additive 

372 finally: 

373 maskedImage.mask.array[:] = originalMask 

374 self.tweakBackground(exposure, bgLevel, results.background) 

375 

376 return results 

377 

378 def tweakBackground(self, exposure, bgLevel, bgList=None): 

379 """Modify the background by a constant value 

380 

381 Parameters 

382 ---------- 

383 exposure : `lsst.afw.image.Exposure` 

384 Exposure for which to tweak background. 

385 bgLevel : `float` 

386 Background level to remove 

387 bgList : `lsst.afw.math.BackgroundList`, optional 

388 List of backgrounds to append to. 

389 

390 Returns 

391 ------- 

392 bg : `lsst.afw.math.BackgroundMI` 

393 Constant background model. 

394 """ 

395 self.log.info("Tweaking background by %f to match sky photometry", bgLevel) 

396 exposure.image -= bgLevel 

397 bgStats = lsst.afw.image.MaskedImageF(1, 1) 

398 bgStats.set(bgLevel, 0, bgLevel) 

399 bg = lsst.afw.math.BackgroundMI(exposure.getBBox(), bgStats) 

400 bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER, 

401 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0, False) 

402 if bgList is not None: 

403 bgList.append(bgData) 

404 return bg 

405 

406 def _computeBrightDetectionMask(self, maskedImage, convolveResults): 

407 """Perform an initial bright source detection pass. 

408 

409 Perform an initial bright object detection pass using a high detection 

410 threshold. The footprints in this pass are grown significantly more 

411 than is typical to account for wings around bright sources. The 

412 negative polarity detections in this pass help in masking severely 

413 over-subtracted regions. 

414 

415 A maximum fraction of masked pixel from this pass is ensured via 

416 the config ``brightMaskFractionMax``. If the masked pixel fraction is 

417 above this value, the detection thresholds here are increased by 

418 ``bisectFactor`` in a while loop until the detected masked fraction 

419 falls below this value. 

420 

421 Parameters 

422 ---------- 

423 maskedImage : `lsst.afw.image.MaskedImage` 

424 Masked image on which to run the detection. 

425 convolveResults : `lsst.pipe.base.Struct` 

426 The results of the self.convolveImage function with attributes: 

427 

428 ``middle`` 

429 Convolved image, without the edges 

430 (`lsst.afw.image.MaskedImage`). 

431 ``sigma`` 

432 Gaussian sigma used for the convolution (`float`). 

433 

434 Returns 

435 ------- 

436 brightDetectedMask : `numpy.ndarray` 

437 Boolean array representing the union of the bright detection pass 

438 DETECTED and DETECTED_NEGATIVE masks. 

439 """ 

440 # Initialize some parameters. 

441 brightPosFactor = ( 

442 self.config.prelimThresholdFactor*self.config.brightMultiplier/self.config.bisectFactor 

443 ) 

444 brightNegFactor = self.config.brightNegFactor/self.config.bisectFactor 

445 nPix = 1 

446 nPixDet = 1 

447 nPixDetNeg = 1 

448 brightMaskFractionMax = self.config.brightMaskFractionMax 

449 

450 # Loop until masked fraction is smaller than 

451 # brightMaskFractionMax, increasing the thresholds by 

452 # config.bisectFactor on each iteration (rarely necessary 

453 # for current defaults). 

454 while nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax: 

455 self.clearMask(maskedImage.mask) 

456 brightPosFactor *= self.config.bisectFactor 

457 brightNegFactor *= self.config.bisectFactor 

458 prelimBright = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), 

459 factor=brightPosFactor, factorNeg=brightNegFactor) 

460 self.finalizeFootprints( 

461 maskedImage.mask, prelimBright, convolveResults.sigma*self.config.brightGrowFactor, 

462 factor=brightPosFactor, factorNeg=brightNegFactor 

463 ) 

464 # Check that not too many pixels got masked. 

465 nPix = maskedImage.mask.array.size 

466 nPixDet = countMaskedPixels(maskedImage, "DETECTED") 

467 self.log.info("Number (%) of bright DETECTED pix: {} ({:.1f}%)". 

468 format(nPixDet, 100*nPixDet/nPix)) 

469 nPixDetNeg = countMaskedPixels(maskedImage, "DETECTED_NEGATIVE") 

470 self.log.info("Number (%) of bright DETECTED_NEGATIVE pix: {} ({:.1f}%)". 

471 format(nPixDetNeg, 100*nPixDetNeg/nPix)) 

472 if nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax: 

473 self.log.warn("Too high a fraction (%.1f > %.1f) of pixels were masked with current " 

474 "\"bright\" detection round thresholds. Increasing by a factor of %f " 

475 "and trying again.", max(nPixDetNeg, nPixDet)/nPix, 

476 brightMaskFractionMax, self.config.bisectFactor) 

477 

478 # Save the mask planes from the "bright" detection round, then 

479 # clear them before moving on to the "prelim" detection phase. 

480 brightDetectedMask = (maskedImage.mask.array 

481 & maskedImage.mask.getPlaneBitMask(["DETECTED", "DETECTED_NEGATIVE"])) 

482 self.clearMask(maskedImage.mask) 

483 return brightDetectedMask 

484 

485 

486def countMaskedPixels(maskedIm, maskPlane): 

487 """Count the number of pixels in a given mask plane. 

488 

489 Parameters 

490 ---------- 

491 maskedIm : `lsst.afw.image.MaskedImage` 

492 Masked image to examine. 

493 maskPlane : `str` 

494 Name of the mask plane to examine. 

495 

496 Returns 

497 ------- 

498 nPixMasked : `int` 

499 Number of pixels with ``maskPlane`` bit set. 

500 """ 

501 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

502 nPixMasked = np.sum(np.bitwise_and(maskedIm.mask.array, maskBit))/maskBit 

503 return nPixMasked