Coverage for python/lsst/meas/algorithms/dynamicDetection.py: 21%

109 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-30 10:04 +0000

1 

2__all__ = ["DynamicDetectionConfig", "DynamicDetectionTask"] 

3 

4import numpy as np 

5 

6from lsst.pex.config import Field, ConfigurableField 

7from lsst.pipe.base import Struct 

8 

9from .detection import SourceDetectionConfig, SourceDetectionTask 

10from .skyObjects import SkyObjectsTask 

11 

12from lsst.afw.detection import FootprintSet 

13from lsst.afw.table import SourceCatalog, SourceTable 

14from lsst.meas.base import ForcedMeasurementTask 

15 

16import lsst.afw.image 

17import lsst.afw.math 

18 

19 

20class DynamicDetectionConfig(SourceDetectionConfig): 

21 """Configuration for DynamicDetectionTask 

22 """ 

23 prelimThresholdFactor = Field(dtype=float, default=0.5, 

24 doc="Fraction of the threshold to use for first pass (to find sky objects)") 

25 skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects") 

26 doBackgroundTweak = Field(dtype=bool, default=True, 

27 doc="Tweak background level so median PSF flux of sky objects is zero?") 

28 minNumSources = Field(dtype=int, default=10, 

29 doc="Minimum number of sky sources in statistical sample; " 

30 "if below this number, we refuse to modify the threshold.") 

31 

32 def setDefaults(self): 

33 SourceDetectionConfig.setDefaults(self) 

34 self.skyObjects.nSources = 1000 # For good statistics 

35 

36 

37class DynamicDetectionTask(SourceDetectionTask): 

38 """Detection of sources on an image with a dynamic threshold 

39 

40 We first detect sources using a lower threshold than normal (see config 

41 parameter ``prelimThresholdFactor``) in order to identify good sky regions 

42 (configurable ``skyObjects``). Then we perform forced PSF photometry on 

43 those sky regions. Using those PSF flux measurements and estimated errors, 

44 we set the threshold so that the stdev of the measurements matches the 

45 median estimated error. 

46 

47 Besides the usual initialisation of configurables, we also set up 

48 the forced measurement which is deliberately not represented in 

49 this Task's configuration parameters because we're using it as 

50 part of the algorithm and we don't want to allow it to be modified. 

51 """ 

52 ConfigClass = DynamicDetectionConfig 

53 _DefaultName = "dynamicDetection" 

54 

55 def __init__(self, *args, **kwargs): 

56 

57 SourceDetectionTask.__init__(self, *args, **kwargs) 

58 self.makeSubtask("skyObjects") 

59 

60 # Set up forced measurement. 

61 config = ForcedMeasurementTask.ConfigClass() 

62 config.plugins.names = ['base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground'] 

63 # We'll need the "centroid" and "psfFlux" slots 

64 for slot in ("shape", "psfShape", "apFlux", "modelFlux", "gaussianFlux", "calibFlux"): 

65 setattr(config.slots, slot, None) 

66 config.copyColumns = {} 

67 self.skySchema = SourceTable.makeMinimalSchema() 

68 self.skyMeasurement = ForcedMeasurementTask(config=config, name="skyMeasurement", parentTask=self, 

69 refSchema=self.skySchema) 

70 

71 def calculateThreshold(self, exposure, seed, sigma=None): 

72 """Calculate new threshold 

73 

74 This is the main functional addition to the vanilla 

75 `SourceDetectionTask`. 

76 

77 We identify sky objects and perform forced PSF photometry on 

78 them. Using those PSF flux measurements and estimated errors, 

79 we set the threshold so that the stdev of the measurements 

80 matches the median estimated error. 

81 

82 Parameters 

83 ---------- 

84 exposure : `lsst.afw.image.Exposure` 

85 Exposure on which we're detecting sources. 

86 seed : `int` 

87 RNG seed to use for finding sky objects. 

88 sigma : `float`, optional 

89 Gaussian sigma of smoothing kernel; if not provided, 

90 will be deduced from the exposure's PSF. 

91 

92 Returns 

93 ------- 

94 result : `lsst.pipe.base.Struct` 

95 Result struct with components: 

96 

97 ``multiplicative`` 

98 Multiplicative factor to be applied to the 

99 configured detection threshold (`float`). 

100 ``additive`` 

101 Additive factor to be applied to the background 

102 level (`float`). 

103 """ 

104 # Make a catalog of sky objects 

105 fp = self.skyObjects.run(exposure.maskedImage.mask, seed) 

106 skyFootprints = FootprintSet(exposure.getBBox()) 

107 skyFootprints.setFootprints(fp) 

108 table = SourceTable.make(self.skyMeasurement.schema) 

109 catalog = SourceCatalog(table) 

110 catalog.reserve(len(skyFootprints.getFootprints())) 

111 skyFootprints.makeSources(catalog) 

112 key = catalog.getCentroidSlot().getMeasKey() 

113 for source in catalog: 

114 peaks = source.getFootprint().getPeaks() 

115 assert len(peaks) == 1 

116 source.set(key, peaks[0].getF()) 

117 source.updateCoord(exposure.getWcs()) 

118 

119 # Forced photometry on sky objects 

120 self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs()) 

121 

122 # Calculate new threshold 

123 fluxes = catalog["base_PsfFlux_instFlux"] 

124 area = catalog["base_PsfFlux_area"] 

125 bg = catalog["base_LocalBackground_instFlux"] 

126 

127 good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] 

128 & np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg)) 

129 

130 if good.sum() < self.config.minNumSources: 

131 self.log.warning("Insufficient good flux measurements (%d < %d) for dynamic threshold" 

132 " calculation", good.sum(), self.config.minNumSources) 

133 return Struct(multiplicative=1.0, additive=0.0) 

134 

135 bgMedian = np.median((fluxes/area)[good]) 

136 

137 lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0]) 

138 stdevMeas = 0.741*(uq - lq) 

139 medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good]) 

140 return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian) 

141 

142 def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True, expId=None): 

143 """Detect footprints with a dynamic threshold 

144 

145 This varies from the vanilla ``detectFootprints`` method because we 

146 do detection twice: one with a low threshold so that we can find 

147 sky uncontaminated by objects, then one more with the new calculated 

148 threshold. 

149 

150 Parameters 

151 ---------- 

152 exposure : `lsst.afw.image.Exposure` 

153 Exposure to process; DETECTED{,_NEGATIVE} mask plane will be 

154 set in-place. 

155 doSmooth : `bool`, optional 

156 If True, smooth the image before detection using a Gaussian 

157 of width ``sigma``. 

158 sigma : `float`, optional 

159 Gaussian Sigma of PSF (pixels); used for smoothing and to grow 

160 detections; if `None` then measure the sigma of the PSF of the 

161 ``exposure``. 

162 clearMask : `bool`, optional 

163 Clear both DETECTED and DETECTED_NEGATIVE planes before running 

164 detection. 

165 expId : `int`, optional 

166 Exposure identifier, used as a seed for the random number 

167 generator. If absent, the seed will be the sum of the image. 

168 

169 Returns 

170 ------- 

171 resutls : `lsst.pipe.base.Struct` 

172 The results `~lsst.pipe.base.Struct` contains: 

173 

174 ``positive`` 

175 Positive polarity footprints. 

176 (`lsst.afw.detection.FootprintSet` or `None`) 

177 ``negative`` 

178 Negative polarity footprints. 

179 (`lsst.afw.detection.FootprintSet` or `None`) 

180 ``numPos`` 

181 Number of footprints in positive or 0 if detection polarity was 

182 negative. (`int`) 

183 ``numNeg`` 

184 Number of footprints in negative or 0 if detection polarity was 

185 positive. (`int`) 

186 ``background`` 

187 Re-estimated background. `None` if 

188 ``reEstimateBackground==False``. 

189 (`lsst.afw.math.BackgroundList`) 

190 ``factor`` 

191 Multiplication factor applied to the configured detection 

192 threshold. (`float`) 

193 ``prelim`` 

194 Results from preliminary detection pass. 

195 (`lsst.pipe.base.Struct`) 

196 """ 

197 maskedImage = exposure.maskedImage 

198 

199 if clearMask: 

200 self.clearMask(maskedImage.mask) 

201 else: 

202 oldDetected = maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(["DETECTED", 

203 "DETECTED_NEGATIVE"]) 

204 

205 with self.tempWideBackgroundContext(exposure): 

206 # Could potentially smooth with a wider kernel than the PSF in order to better pick up the 

207 # wings of stars and galaxies, but for now sticking with the PSF as that's more simple. 

208 psf = self.getPsf(exposure, sigma=sigma) 

209 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

210 middle = convolveResults.middle 

211 sigma = convolveResults.sigma 

212 prelim = self.applyThreshold(middle, maskedImage.getBBox(), self.config.prelimThresholdFactor) 

213 self.finalizeFootprints(maskedImage.mask, prelim, sigma, self.config.prelimThresholdFactor) 

214 

215 # Calculate the proper threshold 

216 # seed needs to fit in a C++ 'int' so pybind doesn't choke on it 

217 seed = (expId if expId is not None else int(maskedImage.image.array.sum())) % (2**31 - 1) 

218 threshResults = self.calculateThreshold(exposure, seed, sigma=sigma) 

219 factor = threshResults.multiplicative 

220 self.log.info("Modifying configured detection threshold by factor %f to %f", 

221 factor, factor*self.config.thresholdValue) 

222 

223 # Blow away preliminary (low threshold) detection mask 

224 self.clearMask(maskedImage.mask) 

225 if not clearMask: 

226 maskedImage.mask.array |= oldDetected 

227 

228 # Rinse and repeat thresholding with new calculated threshold 

229 results = self.applyThreshold(middle, maskedImage.getBBox(), factor) 

230 results.prelim = prelim 

231 results.background = lsst.afw.math.BackgroundList() 

232 if self.config.doTempLocalBackground: 

233 self.applyTempLocalBackground(exposure, middle, results) 

234 self.finalizeFootprints(maskedImage.mask, results, sigma, factor) 

235 

236 self.clearUnwantedResults(maskedImage.mask, results) 

237 

238 if self.config.reEstimateBackground: 

239 self.reEstimateBackground(maskedImage, results.background) 

240 

241 self.display(exposure, results, middle) 

242 

243 if self.config.doBackgroundTweak: 

244 # Re-do the background tweak after any temporary backgrounds have been restored 

245 # 

246 # But we want to keep any large-scale background (e.g., scattered light from bright stars) 

247 # from being selected for sky objects in the calculation, so do another detection pass without 

248 # either the local or wide temporary background subtraction; the DETECTED pixels will mark 

249 # the area to ignore. 

250 originalMask = maskedImage.mask.array.copy() 

251 try: 

252 self.clearMask(exposure.mask) 

253 convolveResults = self.convolveImage(maskedImage, psf, doSmooth=doSmooth) 

254 tweakDetResults = self.applyThreshold(convolveResults.middle, maskedImage.getBBox(), factor) 

255 self.finalizeFootprints(maskedImage.mask, tweakDetResults, sigma, factor) 

256 bgLevel = self.calculateThreshold(exposure, seed, sigma=sigma).additive 

257 finally: 

258 maskedImage.mask.array[:] = originalMask 

259 self.tweakBackground(exposure, bgLevel, results.background) 

260 

261 return results 

262 

263 def tweakBackground(self, exposure, bgLevel, bgList=None): 

264 """Modify the background by a constant value 

265 

266 Parameters 

267 ---------- 

268 exposure : `lsst.afw.image.Exposure` 

269 Exposure for which to tweak background. 

270 bgLevel : `float` 

271 Background level to remove 

272 bgList : `lsst.afw.math.BackgroundList`, optional 

273 List of backgrounds to append to. 

274 

275 Returns 

276 ------- 

277 bg : `lsst.afw.math.BackgroundMI` 

278 Constant background model. 

279 """ 

280 self.log.info("Tweaking background by %f to match sky photometry", bgLevel) 

281 exposure.image -= bgLevel 

282 bgStats = lsst.afw.image.MaskedImageF(1, 1) 

283 bgStats.set(bgLevel, 0, bgLevel) 

284 bg = lsst.afw.math.BackgroundMI(exposure.getBBox(), bgStats) 

285 bgData = (bg, lsst.afw.math.Interpolate.LINEAR, lsst.afw.math.REDUCE_INTERP_ORDER, 

286 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0, False) 

287 if bgList is not None: 

288 bgList.append(bgData) 

289 return bg