Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 32%

121 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-04 10:35 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import lsst.afw.table as afwTable 

23import lsst.daf.base as dafBase 

24from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

25from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask 

26import lsst.meas.extensions.trailedSources # noqa: F401 

27from lsst.obs.base import ExposureIdInfo 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.utils 

31from lsst.utils.timer import timeMethod 

32 

33from . import DipoleFitTask 

34 

35__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask"] 

36 

37 

38class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

39 dimensions=("instrument", "visit", "detector"), 

40 defaultTemplates={"coaddName": "deep", 

41 "warpTypeSuffix": "", 

42 "fakesType": ""}): 

43 science = pipeBase.connectionTypes.Input( 

44 doc="Input science exposure.", 

45 dimensions=("instrument", "visit", "detector"), 

46 storageClass="ExposureF", 

47 name="{fakesType}calexp" 

48 ) 

49 matchedTemplate = pipeBase.connectionTypes.Input( 

50 doc="Warped and PSF-matched template used to create the difference image.", 

51 dimensions=("instrument", "visit", "detector"), 

52 storageClass="ExposureF", 

53 name="{fakesType}{coaddName}Diff_matchedExp", 

54 ) 

55 difference = pipeBase.connectionTypes.Input( 

56 doc="Result of subtracting template from science.", 

57 dimensions=("instrument", "visit", "detector"), 

58 storageClass="ExposureF", 

59 name="{fakesType}{coaddName}Diff_differenceTempExp", 

60 ) 

61 outputSchema = pipeBase.connectionTypes.InitOutput( 

62 doc="Schema (as an example catalog) for output DIASource catalog.", 

63 storageClass="SourceCatalog", 

64 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

65 ) 

66 diaSources = pipeBase.connectionTypes.Output( 

67 doc="Detected diaSources on the difference image.", 

68 dimensions=("instrument", "visit", "detector"), 

69 storageClass="SourceCatalog", 

70 name="{fakesType}{coaddName}Diff_diaSrc", 

71 ) 

72 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

73 doc="Difference image with detection mask plane filled in.", 

74 dimensions=("instrument", "visit", "detector"), 

75 storageClass="ExposureF", 

76 name="{fakesType}{coaddName}Diff_differenceExp", 

77 ) 

78 

79 

80class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

81 pipelineConnections=DetectAndMeasureConnections): 

82 """Config for DetectAndMeasureTask 

83 """ 

84 doMerge = pexConfig.Field( 

85 dtype=bool, 

86 default=True, 

87 doc="Merge positive and negative diaSources with grow radius " 

88 "set by growFootprint" 

89 ) 

90 doForcedMeasurement = pexConfig.Field( 

91 dtype=bool, 

92 default=True, 

93 doc="Force photometer diaSource locations on PVI?") 

94 doAddMetrics = pexConfig.Field( 

95 dtype=bool, 

96 default=False, 

97 doc="Add columns to the source table to hold analysis metrics?" 

98 ) 

99 detection = pexConfig.ConfigurableField( 

100 target=SourceDetectionTask, 

101 doc="Final source detection for diaSource measurement", 

102 ) 

103 measurement = pexConfig.ConfigurableField( 

104 target=DipoleFitTask, 

105 doc="Task to measure sources on the difference image.", 

106 ) 

107 doApCorr = lsst.pex.config.Field( 

108 dtype=bool, 

109 default=True, 

110 doc="Run subtask to apply aperture corrections" 

111 ) 

112 applyApCorr = lsst.pex.config.ConfigurableField( 

113 target=ApplyApCorrTask, 

114 doc="Task to apply aperture corrections" 

115 ) 

116 forcedMeasurement = pexConfig.ConfigurableField( 

117 target=ForcedMeasurementTask, 

118 doc="Task to force photometer science image at diaSource locations.", 

119 ) 

120 growFootprint = pexConfig.Field( 

121 dtype=int, 

122 default=2, 

123 doc="Grow positive and negative footprints by this many pixels before merging" 

124 ) 

125 diaSourceMatchRadius = pexConfig.Field( 

126 dtype=float, 

127 default=0.5, 

128 doc="Match radius (in arcseconds) for DiaSource to Source association" 

129 ) 

130 doSkySources = pexConfig.Field( 

131 dtype=bool, 

132 default=False, 

133 doc="Generate sky sources?", 

134 ) 

135 skySources = pexConfig.ConfigurableField( 

136 target=SkyObjectsTask, 

137 doc="Generate sky sources", 

138 ) 

139 

140 def setDefaults(self): 

141 # DiaSource Detection 

142 self.detection.thresholdPolarity = "both" 

143 self.detection.thresholdValue = 5.0 

144 self.detection.reEstimateBackground = False 

145 self.detection.thresholdType = "pixel_stdev" 

146 

147 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

148 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

149 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

150 'base_LocalPhotoCalib', 

151 'base_LocalWcs'] 

152 

153 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

154 self.forcedMeasurement.copyColumns = { 

155 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

156 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

157 self.forcedMeasurement.slots.shape = None 

158 

159 

160class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

161 """Detect and measure sources on a difference image. 

162 """ 

163 ConfigClass = DetectAndMeasureConfig 

164 _DefaultName = "detectAndMeasure" 

165 

166 def __init__(self, **kwargs): 

167 super().__init__(**kwargs) 

168 self.schema = afwTable.SourceTable.makeMinimalSchema() 

169 

170 self.algMetadata = dafBase.PropertyList() 

171 self.makeSubtask("detection", schema=self.schema) 

172 self.makeSubtask("measurement", schema=self.schema, 

173 algMetadata=self.algMetadata) 

174 if self.config.doApCorr: 

175 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

176 if self.config.doForcedMeasurement: 

177 self.schema.addField( 

178 "ip_diffim_forced_PsfFlux_instFlux", "D", 

179 "Forced PSF flux measured on the direct image.", 

180 units="count") 

181 self.schema.addField( 

182 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

183 "Forced PSF flux error measured on the direct image.", 

184 units="count") 

185 self.schema.addField( 

186 "ip_diffim_forced_PsfFlux_area", "F", 

187 "Forced PSF flux effective area of PSF.", 

188 units="pixel") 

189 self.schema.addField( 

190 "ip_diffim_forced_PsfFlux_flag", "Flag", 

191 "Forced PSF flux general failure flag.") 

192 self.schema.addField( 

193 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

194 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

195 self.schema.addField( 

196 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

197 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

198 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

199 

200 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

201 self.schema.addField("srcMatchId", "L", "unique id of source match") 

202 if self.config.doSkySources: 

203 self.makeSubtask("skySources") 

204 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

205 

206 # initialize InitOutputs 

207 self.outputSchema = afwTable.SourceCatalog(self.schema) 

208 self.outputSchema.getTable().setMetadata(self.algMetadata) 

209 

210 @staticmethod 

211 def makeIdFactory(expId, expBits): 

212 """Create IdFactory instance for unique 64 bit diaSource id-s. 

213 

214 Parameters 

215 ---------- 

216 expId : `int` 

217 Exposure id. 

218 

219 expBits: `int` 

220 Number of used bits in ``expId``. 

221 

222 Notes 

223 ----- 

224 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

225 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

226 low value end of the integer. 

227 

228 Returns 

229 ------- 

230 idFactory: `lsst.afw.table.IdFactory` 

231 """ 

232 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

233 

234 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

235 inputRefs: pipeBase.InputQuantizedConnection, 

236 outputRefs: pipeBase.OutputQuantizedConnection): 

237 inputs = butlerQC.get(inputRefs) 

238 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

239 returnMaxBits=True) 

240 idFactory = self.makeIdFactory(expId=expId, expBits=expBits) 

241 

242 outputs = self.run(inputs['science'], 

243 inputs['matchedTemplate'], 

244 inputs['difference'], 

245 idFactory=idFactory) 

246 butlerQC.put(outputs, outputRefs) 

247 

248 @timeMethod 

249 def run(self, science, matchedTemplate, difference, 

250 idFactory=None): 

251 """Detect and measure sources on a difference image. 

252 

253 Parameters 

254 ---------- 

255 science : `lsst.afw.image.ExposureF` 

256 Science exposure that the template was subtracted from. 

257 matchedTemplate : `lsst.afw.image.ExposureF` 

258 Warped and PSF-matched template that was used produce the 

259 difference image. 

260 difference : `lsst.afw.image.ExposureF` 

261 Result of subtracting template from the science image. 

262 idFactory : `lsst.afw.table.IdFactory`, optional 

263 Generator object to assign ids to detected sources in the difference image. 

264 

265 Returns 

266 ------- 

267 results : `lsst.pipe.base.Struct` 

268 

269 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

270 Subtracted exposure with detection mask applied. 

271 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

272 The catalog of detected sources. 

273 """ 

274 # Ensure that we start with an empty detection mask. 

275 mask = difference.mask 

276 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

277 

278 table = afwTable.SourceTable.make(self.schema, idFactory) 

279 table.setMetadata(self.algMetadata) 

280 results = self.detection.run( 

281 table=table, 

282 exposure=difference, 

283 doSmooth=True, 

284 ) 

285 

286 if self.config.doMerge: 

287 fpSet = results.fpSets.positive 

288 fpSet.merge(results.fpSets.negative, self.config.growFootprint, 

289 self.config.growFootprint, False) 

290 diaSources = afwTable.SourceCatalog(table) 

291 fpSet.makeSources(diaSources) 

292 self.log.info("Merging detections into %d sources", len(diaSources)) 

293 else: 

294 diaSources = results.sources 

295 

296 if self.config.doSkySources: 

297 self.addSkySources(diaSources, difference.mask, difference.info.id) 

298 

299 self.measureDiaSources(diaSources, science, difference, matchedTemplate) 

300 

301 if self.config.doForcedMeasurement: 

302 self.measureForcedSources(diaSources, science, difference.getWcs()) 

303 

304 return pipeBase.Struct( 

305 subtractedMeasuredExposure=difference, 

306 diaSources=diaSources, 

307 ) 

308 

309 def addSkySources(self, diaSources, mask, seed): 

310 """Add sources in empty regions of the difference image 

311 for measuring the background. 

312 

313 Parameters 

314 ---------- 

315 diaSources : `lsst.afw.table.SourceCatalog` 

316 The catalog of detected sources. 

317 mask : `lsst.afw.image.Mask` 

318 Mask plane for determining regions where Sky sources can be added. 

319 seed : `int` 

320 Seed value to initialize the random number generator. 

321 """ 

322 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

323 if skySourceFootprints: 

324 for foot in skySourceFootprints: 

325 s = diaSources.addNew() 

326 s.setFootprint(foot) 

327 s.set(self.skySourceKey, True) 

328 

329 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

330 """Use (matched) template and science image to constrain dipole fitting. 

331 

332 Parameters 

333 ---------- 

334 diaSources : `lsst.afw.table.SourceCatalog` 

335 The catalog of detected sources. 

336 science : `lsst.afw.image.ExposureF` 

337 Science exposure that the template was subtracted from. 

338 difference : `lsst.afw.image.ExposureF` 

339 Result of subtracting template from the science image. 

340 matchedTemplate : `lsst.afw.image.ExposureF` 

341 Warped and PSF-matched template that was used produce the 

342 difference image. 

343 """ 

344 # Note that this may not be correct if we convolved the science image. 

345 # In the future we may wish to persist the matchedScience image. 

346 self.measurement.run(diaSources, difference, science, matchedTemplate) 

347 if self.config.doApCorr: 

348 self.applyApCorr.run( 

349 catalog=diaSources, 

350 apCorrMap=difference.getInfo().getApCorrMap() 

351 ) 

352 

353 def measureForcedSources(self, diaSources, science, wcs): 

354 """Perform forced measurement of the diaSources on the science image. 

355 

356 Parameters 

357 ---------- 

358 diaSources : `lsst.afw.table.SourceCatalog` 

359 The catalog of detected sources. 

360 science : `lsst.afw.image.ExposureF` 

361 Science exposure that the template was subtracted from. 

362 wcs : `lsst.afw.geom.SkyWcs` 

363 Coordinate system definition (wcs) for the exposure. 

364 """ 

365 # Run forced psf photometry on the PVI at the diaSource locations. 

366 # Copy the measured flux and error into the diaSource. 

367 forcedSources = self.forcedMeasurement.generateMeasCat( 

368 science, diaSources, wcs) 

369 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

370 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

371 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

372 "ip_diffim_forced_PsfFlux_instFlux", True) 

373 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

374 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

375 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

376 "ip_diffim_forced_PsfFlux_area", True) 

377 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

378 "ip_diffim_forced_PsfFlux_flag", True) 

379 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

380 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

381 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

382 "ip_diffim_forced_PsfFlux_flag_edge", True) 

383 for diaSource, forcedSource in zip(diaSources, forcedSources): 

384 diaSource.assign(forcedSource, mapper)