Coverage for python/lsst/meas/algorithms/dynamicDetection.py: 18%

159 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-19 09:22 +0000

1 

2__all__ = ["DynamicDetectionConfig", "DynamicDetectionTask"] 

3 

4import numpy as np 

5 

6from lsst.pex.config import Field, ConfigurableField 

7from lsst.pipe.base import Struct, NoWorkFound 

8 

9from .detection import SourceDetectionConfig, SourceDetectionTask 

10from .skyObjects import SkyObjectsTask 

11 

12from lsst.afw.detection import FootprintSet 

13from lsst.afw.geom import makeCdMatrix, makeSkyWcs 

14from lsst.afw.table import SourceCatalog, SourceTable 

15from lsst.meas.base import ForcedMeasurementTask 

16 

17import lsst.afw.image 

18import lsst.afw.math 

19import lsst.geom as geom 

20 

21 

22class DynamicDetectionConfig(SourceDetectionConfig): 

23 """Configuration for DynamicDetectionTask 

24 """ 

25 prelimThresholdFactor = Field(dtype=float, default=0.5, 

26 doc="Factor by which to multiply the main detection threshold " 

27 "(thresholdValue) to use for first pass (to find sky objects).") 

28 prelimNegMultiplier = Field(dtype=float, default=2.5, 

29 doc="Multiplier for the negative (relative to positive) polarity " 

30 "detections threshold to use for first pass (to find sky objects).") 

31 skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects.") 

32 doBackgroundTweak = Field(dtype=bool, default=True, 

33 doc="Tweak background level so median PSF flux of sky objects is zero?") 

34 minFractionSources = Field(dtype=float, default=0.02, 

35 doc="Minimum fraction of the requested number of sky sources for dynamic " 

36 "detection to be considered a success. If the number of good sky sources " 

37 "identified falls below this threshold, a NoWorkFound error is raised so " 

38 "that this dataId is no longer considered in downstream processing.") 

39 doBrightPrelimDetection = Field(dtype=bool, default=True, 

40 doc="Do initial bright detection pass where footprints are grown " 

41 "by brightGrowFactor?") 

42 brightMultiplier = Field(dtype=float, default=2000.0, 

43 doc="Multiplier to apply to the prelimThresholdFactor for the " 

44 "\"bright\" detections stage (want this to be large to only " 

45 "detect the brightest sources).") 

46 brightNegFactor = Field(dtype=float, default=2.2, 

47 doc="Factor by which to multiply the threshold for the negative polatiry " 

48 "detections for the \"bright\" detections stage (this needs to be fairly " 

49 "low given the nature of the negative polarity detections in the very " 

50 "large positive polarity threshold).") 

51 brightGrowFactor = Field(dtype=int, default=40, 

52 doc="Factor by which to grow the footprints of sources detected in the " 

53 "\"bright\" detections stage (want this to be large to mask wings of " 

54 "bright sources).") 

55 brightMaskFractionMax = Field(dtype=float, default=0.95, 

56 doc="Maximum allowed fraction of masked pixes from the \"bright\" " 

57 "detection stage (to mask regions unsuitable for sky sourcess). " 

58 "If this fraction is exeeded, the detection threshold for this stage " 

59 "will be increased by bisectFactor until the fraction of masked " 

60 "pixels drops below this threshold.") 

61 bisectFactor = Field(dtype=float, default=1.2, 

62 doc="Factor by which to increase thresholds in brightMaskFractionMax loop.") 

63 

64 def setDefaults(self): 

65 SourceDetectionConfig.setDefaults(self) 

66 self.skyObjects.nSources = 1000 # For good statistics 

67 for maskStr in ["INTRP", "SAT"]: 

68 if maskStr not in self.skyObjects.avoidMask: 

69 self.skyObjects.avoidMask.append(maskStr) 

70 

71 

72class DynamicDetectionTask(SourceDetectionTask): 

73 """Detection of sources on an image with a dynamic threshold 

74 

75 We first detect sources using a lower threshold than normal (see config 

76 parameter ``prelimThresholdFactor``) in order to identify good sky regions 

77 (configurable ``skyObjects``). Then we perform forced PSF photometry on 

78 those sky regions. Using those PSF flux measurements and estimated errors, 

79 we set the threshold so that the stdev of the measurements matches the 

80 median estimated error. 

81 

82 Besides the usual initialisation of configurables, we also set up 

83 the forced measurement which is deliberately not represented in 

84 this Task's configuration parameters because we're using it as 

85 part of the algorithm and we don't want to allow it to be modified. 

86 """ 

87 ConfigClass = DynamicDetectionConfig 

88 _DefaultName = "dynamicDetection" 

89 

90 def __init__(self, *args, **kwargs): 

91 

92 SourceDetectionTask.__init__(self, *args, **kwargs) 

93 self.makeSubtask("skyObjects") 

94 

95 # Set up forced measurement. 

96 config = ForcedMeasurementTask.ConfigClass() 

97 config.plugins.names = ['base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground'] 

98 # We'll need the "centroid" and "psfFlux" slots 

99 for slot in ("shape", "psfShape", "apFlux", "modelFlux", "gaussianFlux", "calibFlux"): 

100 setattr(config.slots, slot, None) 

101 config.copyColumns = {} 

102 self.skySchema = SourceTable.makeMinimalSchema() 

103 self.skyMeasurement = ForcedMeasurementTask(config=config, name="skyMeasurement", parentTask=self, 

104 refSchema=self.skySchema) 

105 

106 def calculateThreshold(self, exposure, seed, sigma=None): 

107 """Calculate new threshold 

108 

109 This is the main functional addition to the vanilla 

110 `SourceDetectionTask`. 

111 

112 We identify sky objects and perform forced PSF photometry on 

113 them. Using those PSF flux measurements and estimated errors, 

114 we set the threshold so that the stdev of the measurements 

115 matches the median estimated error. 

116 

117 Parameters 

118 ---------- 

119 exposureOrig : `lsst.afw.image.Exposure` 

120 Exposure on which we're detecting sources. 

121 seed : `int` 

122 RNG seed to use for finding sky objects. 

123 sigma : `float`, optional 

124 Gaussian sigma of smoothing kernel; if not provided, 

125 will be deduced from the exposure's PSF. 

126 

127 Returns 

128 ------- 

129 result : `lsst.pipe.base.Struct` 

130 Result struct with components: 

131 

132 ``multiplicative`` 

133 Multiplicative factor to be applied to the 

134 configured detection threshold (`float`). 

135 ``additive`` 

136 Additive factor to be applied to the background 

137 level (`float`). 

138 

139 Raises 

140 ------ 

141 NoWorkFound 

142 Raised if the number of good sky sources found is less than the 

143 minimum fraction (``self.config.minFractionSources``) of the number 

144 requested (``self.skyObjects.config.nSources``). 

145 """ 

146 wcsIsNone = exposure.getWcs() is None 

147 if wcsIsNone: # create a dummy WCS as needed by ForcedMeasurementTask 

148 self.log.info("WCS for exposure is None. Setting a dummy WCS for dynamic detection.") 

149 exposure.setWcs(makeSkyWcs(crpix=geom.Point2D(0, 0), 

150 crval=geom.SpherePoint(0, 0, geom.degrees), 

151 cdMatrix=makeCdMatrix(scale=1e-5*geom.degrees))) 

152 fp = self.skyObjects.run(exposure.maskedImage.mask, seed) 

153 skyFootprints = FootprintSet(exposure.getBBox()) 

154 skyFootprints.setFootprints(fp) 

155 table = SourceTable.make(self.skyMeasurement.schema) 

156 catalog = SourceCatalog(table) 

157 catalog.reserve(len(skyFootprints.getFootprints())) 

158 skyFootprints.makeSources(catalog) 

159 key = catalog.getCentroidSlot().getMeasKey() 

160 for source in catalog: 

161 peaks = source.getFootprint().getPeaks() 

162 assert len(peaks) == 1 

163 source.set(key, peaks[0].getF()) 

164 source.updateCoord(exposure.getWcs()) 

165 

166 # Forced photometry on sky objects 

167 self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs()) 

168 

169 # Calculate new threshold 

170 fluxes = catalog["base_PsfFlux_instFlux"] 

171 area = catalog["base_PsfFlux_area"] 

172 bg = catalog["base_LocalBackground_instFlux"] 

173 

174 good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] 

175 & np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg)) 

176 

177 minNumSources = int(self.config.minFractionSources*self.skyObjects.config.nSources) 

178 if good.sum() < minNumSources: 

179 raise NoWorkFound(f"Insufficient good sky source flux measurements ({good.sum()} < " 

180 f"{minNumSources}) for dynamic threshold calculation.") 

181 

182 self.log.info("Number of good sky sources used for dynamic detection: %d (of %d requested).", 

183 good.sum(), self.skyObjects.config.nSources) 

184 bgMedian = np.median((fluxes/area)[good]) 

185 

186 lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0]) 

187 stdevMeas = 0.741*(uq - lq) 

188 medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good]) 

189 if wcsIsNone: 

190 exposure.setWcs(None) 

191 return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian) 

192 

193 def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None): 

194 """Detect footprints with a dynamic threshold 

195 

196 This varies from the vanilla ``detectFootprints`` method because we 

197 do detection three times: first with a high threshold to detect 

198 "bright" (both positive and negative, the latter to identify very 

199 over-subtracted regions) sources for which we grow the DETECTED and 

200 DETECTED_NEGATIVE masks significantly to account for wings. Second, 

201 with a low threshold to mask all non-empty regions of the image. These 

202 two masks are combined and used to identify regions of sky 

203 uncontaminated by objects. A final round of detection is then done 

204 with the new calculated threshold. 

205 

206 Parameters 

207 ---------- 

208 exposure : `lsst.afw.image.Exposure` 

209 Exposure to process; DETECTED{,_NEGATIVE} mask plane will be 

210 set in-place. 

211 doSmooth : `bool`, optional 

212 If True, smooth the image before detection using a Gaussian 

213 of width ``sigma``. 

214 sigma : `float`, optional 

215 Gaussian Sigma of PSF (pixels); used for smoothing and to grow 

216 detections; if `None` then measure the sigma of the PSF of the 

217 ``exposure``. 

218 clearMask : `bool`, optional 

219 Clear both DETECTED and DETECTED_NEGATIVE planes before running 

220 detection. 

221 expId : `int`, optional 

222 Exposure identifier, used as a seed for the random number 

223 generator. If absent, the seed will be the sum of the image. 

224 

225 Returns 

226 ------- 

227 resutls : `lsst.pipe.base.Struct` 

228 The results `~lsst.pipe.base.Struct` contains: 

229 

230 ``positive`` 

231 Positive polarity footprints. 

232 (`lsst.afw.detection.FootprintSet` or `None`) 

233 ``negative`` 

234 Negative polarity footprints. 

235 (`lsst.afw.detection.FootprintSet` or `None`) 

236 ``numPos`` 

237 Number of footprints in positive or 0 if detection polarity was 

238 negative. (`int`) 

239 ``numNeg`` 

240 Number of footprints in negative or 0 if detection polarity was 

241 positive. (`int`) 

242 ``background`` 

243 Re-estimated background. `None` if 

244 ``reEstimateBackground==False``. 

245 (`lsst.afw.math.BackgroundList`) 

246 ``factor`` 

247 Multiplication factor applied to the configured detection 

248 threshold. (`float`) 

249 ``prelim`` 

250 Results from preliminary detection pass. 

251 (`lsst.pipe.base.Struct`) 

252 """ 

253 maskedImage = exposure.maskedImage 

254 

255 if clearMask: 

256 self.clearMask(maskedImage.mask) 

257 else: 

258 oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(["DETECTED", 

259 "DETECTED_NEGATIVE"]) 

260 

261 with self.tempWideBackgroundContext(exposure): 

262 # Could potentially smooth with a wider kernel than the PSF in 

263 # order to better pick up the wings of stars and galaxies, but for 

264 # now sticking with the PSF as that's more simple. 

265 psf = self.getPsf(exposure, sigma=sigma) 

266 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

267 

268 if self.config.doBrightPrelimDetection: 

269 brightDetectedMask = self._computeBrightDetectionMask(maskedImage, convolveResults) 

270 

271 middle = convolveResults.middle 

272 sigma = convolveResults.sigma 

273 prelim = self.applyThreshold( 

274 middle, maskedImage.getBBox(), factor=self.config.prelimThresholdFactor, 

275 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor 

276 ) 

277 self.finalizeFootprints( 

278 maskedImage.mask, prelim, sigma, factor=self.config.prelimThresholdFactor, 

279 factorNeg=self.config.prelimNegMultiplier*self.config.prelimThresholdFactor 

280 ) 

281 if self.config.doBrightPrelimDetection: 

282 # Combine prelim and bright detection masks for multiplier 

283 # determination. 

284 maskedImage.mask.array |= brightDetectedMask 

285 

286 # Calculate the proper threshold 

287 # seed needs to fit in a C++ 'int' so pybind doesn't choke on it 

288 seed = (expId if expId is not None else int(maskedImage.image.array.sum())) % (2**31 - 1) 

289 threshResults = self.calculateThreshold(exposure, seed, sigma=sigma) 

290 factor = threshResults.multiplicative 

291 self.log.info("Modifying configured detection threshold by factor %f to %f", 

292 factor, factor*self.config.thresholdValue) 

293 

294 # Blow away preliminary (low threshold) detection mask 

295 self.clearMask(maskedImage.mask) 

296 if not clearMask: 

297 maskedImage.mask.array |= oldDetected 

298 

299 # Rinse and repeat thresholding with new calculated threshold 

300 results = self.applyThreshold(middle, maskedImage.getBBox(), factor) 

301 results.prelim = prelim 

302 results.background = lsst.afw.math.BackgroundList() 

303 if self.config.doTempLocalBackground: 

304 self.applyTempLocalBackground(exposure, middle, results) 

305 self.finalizeFootprints(maskedImage.mask, results, sigma, factor=factor) 

306 

307 self.clearUnwantedResults(maskedImage.mask, results) 

308 

309 if self.config.reEstimateBackground: 

310 self.reEstimateBackground(maskedImage, results.background) 

311 

312 self.display(exposure, results, middle) 

313 

314 if self.config.doBackgroundTweak: 

315 # Re-do the background tweak after any temporary backgrounds have 

316 # been restored. 

317 # 

318 # But we want to keep any large-scale background (e.g., scattered 

319 # light from bright stars) from being selected for sky objects in 

320 # the calculation, so do another detection pass without either the 

321 # local or wide temporary background subtraction; the DETECTED 

322 # pixels will mark the area to ignore. 

323 originalMask = maskedImage.mask.array.copy() 

324 try: 

325 self.clearMask(exposure.mask) 

326 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

327 tweakDetResults = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor) 

328 self.finalizeFootprints(maskedImage.mask, tweakDetResults, sigma, factor=factor) 

329 bgLevel = self.calculateThreshold(exposure, seed, sigma=sigma).additive 

330 finally: 

331 maskedImage.mask.array[:] = originalMask 

332 self.tweakBackground(exposure, bgLevel, results.background) 

333 

334 return results 

335 

336 def tweakBackground(self, exposure, bgLevel, bgList=None): 

337 """Modify the background by a constant value 

338 

339 Parameters 

340 ---------- 

341 exposure : `lsst.afw.image.Exposure` 

342 Exposure for which to tweak background. 

343 bgLevel : `float` 

344 Background level to remove 

345 bgList : `lsst.afw.math.BackgroundList`, optional 

346 List of backgrounds to append to. 

347 

348 Returns 

349 ------- 

350 bg : `lsst.afw.math.BackgroundMI` 

351 Constant background model. 

352 """ 

353 self.log.info("Tweaking background by %f to match sky photometry", bgLevel) 

354 exposure.image -= bgLevel 

355 bgStats = lsst.afw.image.MaskedImageF(1, 1) 

356 bgStats.set(bgLevel, 0, bgLevel) 

357 bg = lsst.afw.math.BackgroundMI(exposure.getBBox(), bgStats) 

358 bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER, 

359 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0, False) 

360 if bgList is not None: 

361 bgList.append(bgData) 

362 return bg 

363 

364 def _computeBrightDetectionMask(self, maskedImage, convolveResults): 

365 """Perform an initial bright source detection pass. 

366 

367 Perform an initial bright object detection pass using a high detection 

368 threshold. The footprints in this pass are grown significantly more 

369 than is typical to account for wings around bright sources. The 

370 negative polarity detections in this pass help in masking severely 

371 over-subtracted regions. 

372 

373 A maximum fraction of masked pixel from this pass is ensured via 

374 the config ``brightMaskFractionMax``. If the masked pixel fraction is 

375 above this value, the detection thresholds here are increased by 

376 ``bisectFactor`` in a while loop until the detected masked fraction 

377 falls below this value. 

378 

379 Parameters 

380 ---------- 

381 maskedImage : `lsst.afw.image.MaskedImage` 

382 Masked image on which to run the detection. 

383 convolveResults : `lsst.pipe.base.Struct` 

384 The results of the self.convolveImage function with attributes: 

385 

386 ``middle`` 

387 Convolved image, without the edges 

388 (`lsst.afw.image.MaskedImage`). 

389 ``sigma`` 

390 Gaussian sigma used for the convolution (`float`). 

391 

392 Returns 

393 ------- 

394 brightDetectedMask : `numpy.ndarray` 

395 Boolean array representing the union of the bright detection pass 

396 DETECTED and DETECTED_NEGATIVE masks. 

397 """ 

398 # Initialize some parameters. 

399 brightPosFactor = ( 

400 self.config.prelimThresholdFactor*self.config.brightMultiplier/self.config.bisectFactor 

401 ) 

402 brightNegFactor = self.config.brightNegFactor/self.config.bisectFactor 

403 nPix = 1 

404 nPixDet = 1 

405 nPixDetNeg = 1 

406 brightMaskFractionMax = self.config.brightMaskFractionMax 

407 

408 # Loop until masked fraction is smaller than 

409 # brightMaskFractionMax, increasing the thresholds by 

410 # config.bisectFactor on each iteration (rarely necessary 

411 # for current defaults). 

412 while nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax: 

413 self.clearMask(maskedImage.mask) 

414 brightPosFactor *= self.config.bisectFactor 

415 brightNegFactor *= self.config.bisectFactor 

416 prelimBright = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), 

417 factor=brightPosFactor, factorNeg=brightNegFactor) 

418 self.finalizeFootprints( 

419 maskedImage.mask, prelimBright, convolveResults.sigma*self.config.brightGrowFactor, 

420 factor=brightPosFactor, factorNeg=brightNegFactor 

421 ) 

422 # Check that not too many pixels got masked. 

423 nPix = maskedImage.mask.array.size 

424 nPixDet = countMaskedPixels(maskedImage, "DETECTED") 

425 self.log.info("Number (%) of bright DETECTED pix: {} ({:.1f}%)". 

426 format(nPixDet, 100*nPixDet/nPix)) 

427 nPixDetNeg = countMaskedPixels(maskedImage, "DETECTED_NEGATIVE") 

428 self.log.info("Number (%) of bright DETECTED_NEGATIVE pix: {} ({:.1f}%)". 

429 format(nPixDetNeg, 100*nPixDetNeg/nPix)) 

430 if nPixDetNeg/nPix > brightMaskFractionMax or nPixDet/nPix > brightMaskFractionMax: 

431 self.log.warn("Too high a fraction (%.1f > %.1f) of pixels were masked with current " 

432 "\"bright\" detection round thresholds. Increasing by a factor of %f " 

433 "and trying again.", max(nPixDetNeg, nPixDet)/nPix, 

434 brightMaskFractionMax, self.config.bisectFactor) 

435 

436 # Save the mask planes from the "bright" detection round, then 

437 # clear them before moving on to the "prelim" detection phase. 

438 brightDetectedMask = (maskedImage.mask.array 

439 & maskedImage.mask.getPlaneBitMask(["DETECTED", "DETECTED_NEGATIVE"])) 

440 self.clearMask(maskedImage.mask) 

441 return brightDetectedMask 

442 

443 

444def countMaskedPixels(maskedIm, maskPlane): 

445 """Count the number of pixels in a given mask plane. 

446 

447 Parameters 

448 ---------- 

449 maskedIm : `lsst.afw.image.MaskedImage` 

450 Masked image to examine. 

451 maskPlane : `str` 

452 Name of the mask plane to examine. 

453 

454 Returns 

455 ------- 

456 nPixMasked : `int` 

457 Number of pixels with ``maskPlane`` bit set. 

458 """ 

459 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

460 nPixMasked = np.sum(np.bitwise_and(maskedIm.mask.array, maskBit))/maskBit 

461 return nPixMasked