Coverage for python/lsst/meas/algorithms/dynamicDetection.py: 18%

159 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-06 02:49 +0000

1 

2__all__ = ["DynamicDetectionConfig", "DynamicDetectionTask"] 

3 

4import numpy as np 

5 

6from lsst.pex.config import Field, ConfigurableField 

7from lsst.pipe.base import Struct, NoWorkFound 

8 

9from .detection import SourceDetectionConfig, SourceDetectionTask 

10from .skyObjects import SkyObjectsTask 

11 

12from lsst.afw.detection import FootprintSet 

13from lsst.afw.geom import makeCdMatrix, makeSkyWcs 

14from lsst.afw.table import SourceCatalog, SourceTable 

15from lsst.meas.base import ForcedMeasurementTask 

16 

17import lsst.afw.image 

18import lsst.afw.math 

19import lsst.geom as geom 

20 

21 

22class DynamicDetectionConfig(SourceDetectionConfig): 

23 """Configuration for DynamicDetectionTask 

24 """ 

25 prelimThresholdFactor = Field(dtype=float, default=0.5, 

26 doc="Factor by which to multiply the main detection threshold " 

27 "(thresholdValue) to use for first pass (to find sky objects).") 

28 prelimNegMultiplier = Field(dtype=float, default=2.5, 

29 doc="Multiplier for the negative (relative to positive) polarity " 

30 "detections threshold to use for first pass (to find sky objects).") 

31 skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects.") 

32 doBackgroundTweak = Field(dtype=bool, default=True, 

33 doc="Tweak background level so median PSF flux of sky objects is zero?") 

34 minFractionSources = Field(dtype=float, default=0.02, 

35 doc="Minimum fraction of the requested number of sky sources for dynamic " 

36 "detection to be considered a success. If the number of good sky sources " 

37 "identified falls below this threshold, a NoWorkFound error is raised so " 

38 "that this dataId is no longer considered in downstream processing.") 

39 doBrightPrelimDetection = Field(dtype=bool, default=True, 

40 doc="Do initial bright detection pass where footprints are grown " 

41 "by brightGrowFactor?") 

42 brightMultiplier = Field(dtype=float, default=2000.0, 

43 doc="Multiplier to apply to the prelimThresholdFactor for the " 

44 "\"bright\" detections stage (want this to be large to only " 

45 "detect the brightest sources).") 

46 brightNegFactor = Field(dtype=float, default=2.2, 

47 doc="Factor by which to multiply the threshold for the negative polatiry " 

48 "detections for the \"bright\" detections stage (this needs to be fairly " 

49 "low given the nature of the negative polarity detections in the very " 

50 "large positive polarity threshold).") 

51 brightGrowFactor = Field(dtype=int, default=40, 

52 doc="Factor by which to grow the footprints of sources detected in the " 

53 "\"bright\" detections stage (want this to be large to mask wings of " 

54 "bright sources).") 

55 brightMaskFractionMax = Field(dtype=float, default=0.95, 

56 doc="Maximum allowed fraction of masked pixes from the \"bright\" " 

57 "detection stage (to mask regions unsuitable for sky sourcess). " 

58 "If this fraction is exeeded, the detection threshold for this stage " 

59 "will be increased by bisectFactor until the fraction of masked " 

60 "pixels drops below this threshold.") 

61 bisectFactor = Field(dtype=float, default=1.2, 

62 doc="Factor by which to increase thresholds in brightMaskFractionMax loop.") 

63 

64 def setDefaults(self): 

65 SourceDetectionConfig.setDefaults(self) 

66 self.skyObjects.nSources = 1000 # For good statistics 

67 for maskStr in ["INTRP", "SAT"]: 

68 if maskStr not in self.skyObjects.avoidMask: 

69 self.skyObjects.avoidMask.append(maskStr) 

70 

71 

72class DynamicDetectionTask(SourceDetectionTask): 

73 """Detection of sources on an image with a dynamic threshold 

74 

75 We first detect sources using a lower threshold than normal (see config 

76 parameter ``prelimThresholdFactor``) in order to identify good sky regions 

77 (configurable ``skyObjects``). Then we perform forced PSF photometry on 

78 those sky regions. Using those PSF flux measurements and estimated errors, 

79 we set the threshold so that the stdev of the measurements matches the 

80 median estimated error. 

81 

82 Besides the usual initialisation of configurables, we also set up 

83 the forced measurement which is deliberately not represented in 

84 this Task's configuration parameters because we're using it as 

85 part of the algorithm and we don't want to allow it to be modified. 

86 """ 

87 ConfigClass = DynamicDetectionConfig 

88 _DefaultName = "dynamicDetection" 

89 

90 def __init__(self, *args, **kwargs): 

91 

92 SourceDetectionTask.__init__(self, *args, **kwargs) 

93 self.makeSubtask("skyObjects") 

94 

95 # Set up forced measurement. 

96 config = ForcedMeasurementTask.ConfigClass() 

97 config.plugins.names = ['base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground'] 

98 # We'll need the "centroid" and "psfFlux" slots 

99 for slot in ("shape", "psfShape", "apFlux", "modelFlux", "gaussianFlux", "calibFlux"): 

100 setattr(config.slots, slot, None) 

101 config.copyColumns = {} 

102 self.skySchema = SourceTable.makeMinimalSchema() 

103 self.skyMeasurement = ForcedMeasurementTask(config=config, name="skyMeasurement", parentTask=self, 

104 refSchema=self.skySchema) 

105 

106 def calculateThreshold(self, exposure, seed, sigma=None): 

107 """Calculate new threshold 

108 

109 This is the main functional addition to the vanilla 

110 `SourceDetectionTask`. 

111 

112 We identify sky objects and perform forced PSF photometry on 

113 them. Using those PSF flux measurements and estimated errors, 

114 we set the threshold so that the stdev of the measurements 

115 matches the median estimated error. 

116 

117 Parameters 

118 ---------- 

119 exposureOrig : `lsst.afw.image.Exposure` 

120 Exposure on which we're detecting sources. 

121 seed : `int` 

122 RNG seed to use for finding sky objects. 

123 sigma : `float`, optional 

124 Gaussian sigma of smoothing kernel; if not provided, 

125 will be deduced from the exposure's PSF. 

126 

127 Returns 

128 ------- 

129 result : `lsst.pipe.base.Struct` 

130 Result struct with components: 

131 

132 ``multiplicative`` 

133 Multiplicative factor to be applied to the 

134 configured detection threshold (`float`). 

135 ``additive`` 

136 Additive factor to be applied to the background 

137 level (`float`). 

138 

139 Raises 

140 ------ 

141 NoWorkFound 

142 Raised if the number of good sky sources found is less than the 

143 minimum fraction (``self.config.minFractionSources``) of the number 

144 requested (``self.skyObjects.config.nSources``). 

145 """ 

146 wcsIsNone = exposure.getWcs() is None 

147 if wcsIsNone: # create a dummy WCS as needed by ForcedMeasurementTask 

148 self.log.info("WCS for exposure is None. Setting a dummy WCS for dynamic detection.") 

149 exposure.setWcs(makeSkyWcs(crpix=geom.Point2D(0, 0), 

150 crval=geom.SpherePoint(0, 0, geom.degrees), 

151 cdMatrix=makeCdMatrix(scale=1e-5*geom.degrees))) 

152 fp = self.skyObjects.run(exposure.maskedImage.mask, seed) 

153 skyFootprints = FootprintSet(exposure.getBBox()) 

154 skyFootprints.setFootprints(fp) 

155 table = SourceTable.make(self.skyMeasurement.schema) 

156 catalog = SourceCatalog(table) 

157 catalog.reserve(len(skyFootprints.getFootprints())) 

158 skyFootprints.makeSources(catalog) 

159 key = catalog.getCentroidSlot().getMeasKey() 

160 for source in catalog: 

161 peaks = source.getFootprint().getPeaks() 

162 assert len(peaks) == 1 

163 source.set(key, peaks[0].getF()) 

164 source.updateCoord(exposure.getWcs()) 

165 

166 # Forced photometry on sky objects 

167 self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs()) 

168 

169 # Calculate new threshold 

170 fluxes = catalog["base_PsfFlux_instFlux"] 

171 area = catalog["base_PsfFlux_area"] 

172 bg = catalog["base_LocalBackground_instFlux"] 

173 

174 good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] 

175 & np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg)) 

176 

177 minNumSources = int(self.config.minFractionSources*self.skyObjects.config.nSources) 

178 if good.sum() < minNumSources: 

179 raise NoWorkFound(f"Insufficient good sky source flux measurements ({good.sum()} < " 

180 f"{minNumSources}) for dynamic threshold calculation.") 

181 

182 self.log.info("Number of good sky sources used for dynamic detection: %d (of %d requested).", 

183 good.sum(), self.skyObjects.config.nSources) 

184 bgMedian = np.median((fluxes/area)[good]) 

185 

186 lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0]) 

187 stdevMeas = 0.741*(uq - lq) 

188 medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good]) 

189 if wcsIsNone: 

190 exposure.setWcs(None) 

191 return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian) 

192 

193 def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None, 

194 background=None): 

195 """Detect footprints with a dynamic threshold 

196 

197 This varies from the vanilla ``detectFootprints`` method because we 

198 do detection three times: first with a high threshold to detect 

199 "bright" (both positive and negative, the latter to identify very 

200 over-subtracted regions) sources for which we grow the DETECTED and 

201 DETECTED_NEGATIVE masks significantly to account for wings. Second, 

202 with a low threshold to mask all non-empty regions of the image. These 

203 two masks are combined and used to identify regions of sky 

204 uncontaminated by objects. A final round of detection is then done 

205 with the new calculated threshold. 

206 

207 Parameters 

208 ---------- 

209 exposure : `lsst.afw.image.Exposure` 

210 Exposure to process; DETECTED{,_NEGATIVE} mask plane will be 

211 set in-place. 

212 doSmooth : `bool`, optional 

213 If True, smooth the image before detection using a Gaussian 

214 of width ``sigma``. 

215 sigma : `float`, optional 

216 Gaussian Sigma of PSF (pixels); used for smoothing and to grow 

217 detections; if `None` then measure the sigma of the PSF of the 

218 ``exposure``. 

219 clearMask : `bool`, optional 

220 Clear both DETECTED and DETECTED_NEGATIVE planes before running 

221 detection. 

222 expId : `int`, optional 

223 Exposure identifier, used as a seed for the random number 

224 generator. If absent, the seed will be the sum of the image. 

225 background : `lsst.afw.math.BackgroundList`, optional 

226 Background that was already subtracted from the exposure; will be 

227 modified in-place if ``reEstimateBackground=True``. 

228 

229 Returns 

230 ------- 

231 resutls : `lsst.pipe.base.Struct` 

232 The results `~lsst.pipe.base.Struct` contains: 

233 

234 ``positive`` 

235 Positive polarity footprints. 

236 (`lsst.afw.detection.FootprintSet` or `None`) 

237 ``negative`` 

238 Negative polarity footprints. 

239 (`lsst.afw.detection.FootprintSet` or `None`) 

240 ``numPos`` 

241 Number of footprints in positive or 0 if detection polarity was 

242 negative. (`int`) 

243 ``numNeg`` 

244 Number of footprints in negative or 0 if detection polarity was 

245 positive. (`int`) 

246 ``background`` 

247 Re-estimated background. `None` or the input ``background`` 

248 if ``reEstimateBackground==False``. 

249 (`lsst.afw.math.BackgroundList`) 

250 ``factor`` 

251 Multiplication factor applied to the configured detection 

252 threshold. (`float`) 

253 ``prelim`` 

254 Results from preliminary detection pass. 

255 (`lsst.pipe.base.Struct`) 

256 """ 

257 maskedImage = exposure.maskedImage 

258 

259 if clearMask: 

260 self.clearMask(maskedImage.mask) 

261 else: 

262 oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(["DETECTED", 

263 "DETECTED_NEGATIVE"]) 

264 

265 with self.tempWideBackgroundContext(exposure): 

266 # Could potentially smooth with a wider kernel than the PSF in 

267 # order to better pick up the wings of stars and galaxies, but for 

268 # now sticking with the PSF as that's more simple. 

269 psf = self.getPsf(exposure, sigma=sigma) 

270 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

271 

272 if self.config.doBrightPrelimDetection: 

273 brightDetectedMask = self._computeBrightDetectionMask(maskedImage, convolveResults) 

274 

275 middle = convolveResults.middle 

276 sigma = convolveResults.sigma 

277 prelim = self.applyThreshold( 

278 middle, maskedImage.getBBox(), factor=self.config.prelimThresholdFactor, 

279 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor 

280 ) 

281 self.finalizeFootprints( 

282 maskedImage.mask, prelim, sigma, factor=self.config.prelimThresholdFactor, 

283 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor 

284 ) 

285 if self.config.doBrightPrelimDetection: 

286 # Combine prelim and bright detection masks for multiplier 

287 # determination. 

288 maskedImage.mask.array |= brightDetectedMask 

289 

290 # Calculate the proper threshold 

291 # seed needs to fit in a C++ 'int' so pybind doesn't choke on it 

292 seed = (expId if expId is not None else int(maskedImage.image.array.sum())) % (2**31 - 1) 

293 threshResults = self.calculateThreshold(exposure, seed, sigma=sigma) 

294 factor = threshResults.multiplicative 

295 self.log.info("Modifying configured detection threshold by factor %f to %f", 

296 factor, factor*self.config.thresholdValue) 

297 

298 # Blow away preliminary (low threshold) detection mask 

299 self.clearMask(maskedImage.mask) 

300 if not clearMask: 

301 maskedImage.mask.array |= oldDetected 

302 

303 # Rinse and repeat thresholding with new calculated threshold 

304 results = self.applyThreshold(middle, maskedImage.getBBox(), factor) 

305 results.prelim = prelim 

306 results.background = background if background is not None else lsst.afw.math.BackgroundList() 

307 if self.config.doTempLocalBackground: 

308 self.applyTempLocalBackground(exposure, middle, results) 

309 self.finalizeFootprints(maskedImage.mask, results, sigma, factor=factor) 

310 

311 self.clearUnwantedResults(maskedImage.mask, results) 

312 

313 if self.config.reEstimateBackground: 

314 self.reEstimateBackground(maskedImage, results.background) 

315 

316 self.display(exposure, results, middle) 

317 

318 if self.config.doBackgroundTweak: 

319 # Re-do the background tweak after any temporary backgrounds have 

320 # been restored. 

321 # 

322 # But we want to keep any large-scale background (e.g., scattered 

323 # light from bright stars) from being selected for sky objects in 

324 # the calculation, so do another detection pass without either the 

325 # local or wide temporary background subtraction; the DETECTED 

326 # pixels will mark the area to ignore. 

327 originalMask = maskedImage.mask.array.copy() 

328 try: 

329 self.clearMask(exposure.mask) 

330 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

331 tweakDetResults = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor) 

332 self.finalizeFootprints(maskedImage.mask, tweakDetResults, sigma, factor=factor) 

333 bgLevel = self.calculateThreshold(exposure, seed, sigma=sigma).additive 

334 finally: 

335 maskedImage.mask.array[:] = originalMask 

336 self.tweakBackground(exposure, bgLevel, results.background) 

337 

338 return results 

339 

340 def tweakBackground(self, exposure, bgLevel, bgList=None): 

341 """Modify the background by a constant value 

342 

343 Parameters 

344 ---------- 

345 exposure : `lsst.afw.image.Exposure` 

346 Exposure for which to tweak background. 

347 bgLevel : `float` 

348 Background level to remove 

349 bgList : `lsst.afw.math.BackgroundList`, optional 

350 List of backgrounds to append to. 

351 

352 Returns 

353 ------- 

354 bg : `lsst.afw.math.BackgroundMI` 

355 Constant background model. 

356 """ 

357 self.log.info("Tweaking background by %f to match sky photometry", bgLevel) 

358 exposure.image -= bgLevel 

359 bgStats = lsst.afw.image.MaskedImageF(1, 1) 

360 bgStats.set(bgLevel, 0, bgLevel) 

361 bg = lsst.afw.math.BackgroundMI(exposure.getBBox(), bgStats) 

362 bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER, 

363 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0, False) 

364 if bgList is not None: 

365 bgList.append(bgData) 

366 return bg 

367 

368 def _computeBrightDetectionMask(self, maskedImage, convolveResults): 

369 """Perform an initial bright source detection pass. 

370 

371 Perform an initial bright object detection pass using a high detection 

372 threshold. The footprints in this pass are grown significantly more 

373 than is typical to account for wings around bright sources. The 

374 negative polarity detections in this pass help in masking severely 

375 over-subtracted regions. 

376 

377 A maximum fraction of masked pixel from this pass is ensured via 

378 the config ``brightMaskFractionMax``. If the masked pixel fraction is 

379 above this value, the detection thresholds here are increased by 

380 ``bisectFactor`` in a while loop until the detected masked fraction 

381 falls below this value. 

382 

383 Parameters 

384 ---------- 

385 maskedImage : `lsst.afw.image.MaskedImage` 

386 Masked image on which to run the detection. 

387 convolveResults : `lsst.pipe.base.Struct` 

388 The results of the self.convolveImage function with attributes: 

389 

390 ``middle`` 

391 Convolved image, without the edges 

392 (`lsst.afw.image.MaskedImage`). 

393 ``sigma`` 

394 Gaussian sigma used for the convolution (`float`). 

395 

396 Returns 

397 ------- 

398 brightDetectedMask : `numpy.ndarray` 

399 Boolean array representing the union of the bright detection pass 

400 DETECTED and DETECTED_NEGATIVE masks. 

401 """ 

402 # Initialize some parameters. 

403 brightPosFactor = ( 

404 self.config.prelimThresholdFactor*self.config.brightMultiplier/self.config.bisectFactor 

405 ) 

406 brightNegFactor = self.config.brightNegFactor/self.config.bisectFactor 

407 nPix = 1 

408 nPixDet = 1 

409 nPixDetNeg = 1 

410 brightMaskFractionMax = self.config.brightMaskFractionMax 

411 

412 # Loop until masked fraction is smaller than 

413 # brightMaskFractionMax, increasing the thresholds by 

414 # config.bisectFactor on each iteration (rarely necessary 

415 # for current defaults). 

416 while nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax: 

417 self.clearMask(maskedImage.mask) 

418 brightPosFactor *= self.config.bisectFactor 

419 brightNegFactor *= self.config.bisectFactor 

420 prelimBright = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), 

421 factor=brightPosFactor, factorNeg=brightNegFactor) 

422 self.finalizeFootprints( 

423 maskedImage.mask, prelimBright, convolveResults.sigma*self.config.brightGrowFactor, 

424 factor=brightPosFactor, factorNeg=brightNegFactor 

425 ) 

426 # Check that not too many pixels got masked. 

427 nPix = maskedImage.mask.array.size 

428 nPixDet = countMaskedPixels(maskedImage, "DETECTED") 

429 self.log.info("Number (%) of bright DETECTED pix: {} ({:.1f}%)". 

430 format(nPixDet, 100*nPixDet/nPix)) 

431 nPixDetNeg = countMaskedPixels(maskedImage, "DETECTED_NEGATIVE") 

432 self.log.info("Number (%) of bright DETECTED_NEGATIVE pix: {} ({:.1f}%)". 

433 format(nPixDetNeg, 100*nPixDetNeg/nPix)) 

434 if nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax: 

435 self.log.warn("Too high a fraction (%.1f > %.1f) of pixels were masked with current " 

436 "\"bright\" detection round thresholds. Increasing by a factor of %f " 

437 "and trying again.", max(nPixDetNeg, nPixDet)/nPix, 

438 brightMaskFractionMax, self.config.bisectFactor) 

439 

440 # Save the mask planes from the "bright" detection round, then 

441 # clear them before moving on to the "prelim" detection phase. 

442 brightDetectedMask = (maskedImage.mask.array 

443 & maskedImage.mask.getPlaneBitMask(["DETECTED", "DETECTED_NEGATIVE"])) 

444 self.clearMask(maskedImage.mask) 

445 return brightDetectedMask 

446 

447 

448def countMaskedPixels(maskedIm, maskPlane): 

449 """Count the number of pixels in a given mask plane. 

450 

451 Parameters 

452 ---------- 

453 maskedIm : `lsst.afw.image.MaskedImage` 

454 Masked image to examine. 

455 maskPlane : `str` 

456 Name of the mask plane to examine. 

457 

458 Returns 

459 ------- 

460 nPixMasked : `int` 

461 Number of pixels with ``maskPlane`` bit set. 

462 """ 

463 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

464 nPixMasked = np.sum(np.bitwise_and(maskedIm.mask.array, maskBit))/maskBit 

465 return nPixMasked