Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 35%

122 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-20 03:27 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import lsst.afw.table as afwTable 

23import lsst.daf.base as dafBase 

24from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

25from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask 

26import lsst.meas.extensions.trailedSources # noqa: F401 

27from lsst.obs.base import ExposureIdInfo 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.utils 

31from lsst.utils.timer import timeMethod 

32 

33from . import DipoleFitTask 

34 

35__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask"] 

36 

37 

38class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

39 dimensions=("instrument", "visit", "detector"), 

40 defaultTemplates={"coaddName": "deep", 

41 "warpTypeSuffix": "", 

42 "fakesType": ""}): 

43 science = pipeBase.connectionTypes.Input( 

44 doc="Input science exposure.", 

45 dimensions=("instrument", "visit", "detector"), 

46 storageClass="ExposureF", 

47 name="{fakesType}calexp" 

48 ) 

49 template = pipeBase.connectionTypes.Input( 

50 doc="Warped and PSF-matched template used to create the difference image.", 

51 dimensions=("instrument", "visit", "detector"), 

52 storageClass="ExposureF", 

53 name="{fakesType}{coaddName}Diff_matchedExp", 

54 ) 

55 difference = pipeBase.connectionTypes.Input( 

56 doc="Result of subtracting template from science.", 

57 dimensions=("instrument", "visit", "detector"), 

58 storageClass="ExposureF", 

59 name="{fakesType}{coaddName}Diff_differenceTempExp", 

60 ) 

61 selectSources = pipeBase.connectionTypes.Input( 

62 doc="Sources measured on the science exposure; will be matched to the " 

63 "detected sources on the difference image.", 

64 dimensions=("instrument", "visit", "detector"), 

65 storageClass="SourceCatalog", 

66 name="{fakesType}src", 

67 ) 

68 outputSchema = pipeBase.connectionTypes.InitOutput( 

69 doc="Schema (as an example catalog) for output DIASource catalog.", 

70 storageClass="SourceCatalog", 

71 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

72 ) 

73 diaSources = pipeBase.connectionTypes.Output( 

74 doc="Detected diaSources on the difference image.", 

75 dimensions=("instrument", "visit", "detector"), 

76 storageClass="SourceCatalog", 

77 name="{fakesType}{coaddName}Diff_diaSrc", 

78 ) 

79 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

80 doc="Difference image with detection mask plane filled in.", 

81 dimensions=("instrument", "visit", "detector"), 

82 storageClass="ExposureF", 

83 name="{fakesType}{coaddName}Diff_differenceExp", 

84 ) 

85 

86 

87class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

88 pipelineConnections=DetectAndMeasureConnections): 

89 """Config for DetectAndMeasureTask 

90 """ 

91 doMerge = pexConfig.Field( 

92 dtype=bool, 

93 default=True, 

94 doc="Merge positive and negative diaSources with grow radius " 

95 "set by growFootprint" 

96 ) 

97 doForcedMeasurement = pexConfig.Field( 

98 dtype=bool, 

99 default=True, 

100 doc="Force photometer diaSource locations on PVI?") 

101 doAddMetrics = pexConfig.Field( 

102 dtype=bool, 

103 default=False, 

104 doc="Add columns to the source table to hold analysis metrics?" 

105 ) 

106 detection = pexConfig.ConfigurableField( 

107 target=SourceDetectionTask, 

108 doc="Final source detection for diaSource measurement", 

109 ) 

110 measurement = pexConfig.ConfigurableField( 

111 target=DipoleFitTask, 

112 doc="Task to measure sources on the difference image.", 

113 ) 

114 doApCorr = lsst.pex.config.Field( 

115 dtype=bool, 

116 default=True, 

117 doc="Run subtask to apply aperture corrections" 

118 ) 

119 applyApCorr = lsst.pex.config.ConfigurableField( 

120 target=ApplyApCorrTask, 

121 doc="Task to apply aperture corrections" 

122 ) 

123 forcedMeasurement = pexConfig.ConfigurableField( 

124 target=ForcedMeasurementTask, 

125 doc="Task to force photometer science image at diaSource locations.", 

126 ) 

127 growFootprint = pexConfig.Field( 

128 dtype=int, 

129 default=2, 

130 doc="Grow positive and negative footprints by this many pixels before merging" 

131 ) 

132 diaSourceMatchRadius = pexConfig.Field( 

133 dtype=float, 

134 default=0.5, 

135 doc="Match radius (in arcseconds) for DiaSource to Source association" 

136 ) 

137 doSkySources = pexConfig.Field( 

138 dtype=bool, 

139 default=False, 

140 doc="Generate sky sources?", 

141 ) 

142 skySources = pexConfig.ConfigurableField( 

143 target=SkyObjectsTask, 

144 doc="Generate sky sources", 

145 ) 

146 

147 def setDefaults(self): 

148 # DiaSource Detection 

149 self.detection.thresholdPolarity = "both" 

150 self.detection.thresholdValue = 5.0 

151 self.detection.reEstimateBackground = False 

152 self.detection.thresholdType = "pixel_stdev" 

153 

154 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

155 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

156 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

157 'base_LocalPhotoCalib', 

158 'base_LocalWcs'] 

159 

160 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

161 self.forcedMeasurement.copyColumns = { 

162 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

163 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

164 self.forcedMeasurement.slots.shape = None 

165 

166 

167class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

168 """Detect and measure sources on a difference image. 

169 """ 

170 ConfigClass = DetectAndMeasureConfig 

171 _DefaultName = "detectAndMeasure" 

172 

173 def __init__(self, **kwargs): 

174 super().__init__(**kwargs) 

175 self.schema = afwTable.SourceTable.makeMinimalSchema() 

176 

177 self.algMetadata = dafBase.PropertyList() 

178 self.makeSubtask("detection", schema=self.schema) 

179 self.makeSubtask("measurement", schema=self.schema, 

180 algMetadata=self.algMetadata) 

181 if self.config.doApCorr: 

182 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

183 if self.config.doForcedMeasurement: 

184 self.schema.addField( 

185 "ip_diffim_forced_PsfFlux_instFlux", "D", 

186 "Forced PSF flux measured on the direct image.", 

187 units="count") 

188 self.schema.addField( 

189 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

190 "Forced PSF flux error measured on the direct image.", 

191 units="count") 

192 self.schema.addField( 

193 "ip_diffim_forced_PsfFlux_area", "F", 

194 "Forced PSF flux effective area of PSF.", 

195 units="pixel") 

196 self.schema.addField( 

197 "ip_diffim_forced_PsfFlux_flag", "Flag", 

198 "Forced PSF flux general failure flag.") 

199 self.schema.addField( 

200 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

201 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

202 self.schema.addField( 

203 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

204 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

205 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

206 

207 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

208 self.schema.addField("srcMatchId", "L", "unique id of source match") 

209 if self.config.doSkySources: 

210 self.makeSubtask("skySources") 

211 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

212 

213 # initialize InitOutputs 

214 self.outputSchema = afwTable.SourceCatalog(self.schema) 

215 self.outputSchema.getTable().setMetadata(self.algMetadata) 

216 

217 @staticmethod 

218 def makeIdFactory(expId, expBits): 

219 """Create IdFactory instance for unique 64 bit diaSource id-s. 

220 

221 Parameters 

222 ---------- 

223 expId : `int` 

224 Exposure id. 

225 

226 expBits: `int` 

227 Number of used bits in ``expId``. 

228 

229 Notes 

230 ----- 

231 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

232 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

233 low value end of the integer. 

234 

235 Returns 

236 ------- 

237 idFactory: `lsst.afw.table.IdFactory` 

238 """ 

239 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

240 

241 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

242 inputRefs: pipeBase.InputQuantizedConnection, 

243 outputRefs: pipeBase.OutputQuantizedConnection): 

244 inputs = butlerQC.get(inputRefs) 

245 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

246 returnMaxBits=True) 

247 idFactory = self.makeIdFactory(expId=expId, expBits=expBits) 

248 

249 outputs = self.run(inputs['science'], 

250 inputs['template'], 

251 inputs['difference'], 

252 inputs['selectSources'], 

253 idFactory=idFactory) 

254 butlerQC.put(outputs, outputRefs) 

255 

256 @timeMethod 

257 def run(self, science, template, difference, selectSources, 

258 idFactory=None): 

259 """Detect and measure sources on a difference image. 

260 

261 Parameters 

262 ---------- 

263 science : `lsst.afw.image.ExposureF` 

264 Science exposure that the template was subtracted from. 

265 template : `lsst.afw.image.ExposureF` 

266 Warped and PSF-matched template that was used produce the 

267 difference image. 

268 difference : `lsst.afw.image.ExposureF` 

269 Result of subtracting template from the science image. 

270 selectSources : `lsst.afw.table.SourceCatalog`, optional 

271 Identified sources on the science exposure. 

272 idFactory : `lsst.afw.table.IdFactory` 

273 Generator object to assign ids to detected sources in the difference image. 

274 

275 Returns 

276 ------- 

277 results : `lsst.pipe.base.Struct` 

278 

279 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

280 Subtracted exposure with detection mask applied. 

281 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

282 The catalog of detected sources. 

283 """ 

284 # Ensure that we start with an empty detection mask. 

285 mask = difference.mask 

286 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

287 

288 table = afwTable.SourceTable.make(self.schema, idFactory) 

289 table.setMetadata(self.algMetadata) 

290 results = self.detection.run( 

291 table=table, 

292 exposure=difference, 

293 doSmooth=True, 

294 ) 

295 

296 if self.config.doMerge: 

297 fpSet = results.fpSets.positive 

298 fpSet.merge(results.fpSets.negative, self.config.growFootprint, 

299 self.config.growFootprint, False) 

300 diaSources = afwTable.SourceCatalog(table) 

301 fpSet.makeSources(diaSources) 

302 self.log.info("Merging detections into %d sources", len(diaSources)) 

303 else: 

304 diaSources = results.sources 

305 

306 if self.config.doSkySources: 

307 self.addSkySources(diaSources, difference.mask, difference.info.id) 

308 

309 self.measureDiaSources(diaSources, science, difference, template) 

310 

311 if self.config.doForcedMeasurement: 

312 self.measureForcedSources(diaSources, science, difference.getWcs()) 

313 

314 return pipeBase.Struct( 

315 subtractedMeasuredExposure=difference, 

316 diaSources=diaSources, 

317 ) 

318 

319 def addSkySources(self, diaSources, mask, seed): 

320 """Add sources in empty regions of the difference image 

321 for measuring the background. 

322 

323 Parameters 

324 ---------- 

325 diaSources : `lsst.afw.table.SourceCatalog` 

326 The catalog of detected sources. 

327 mask : `lsst.afw.image.Mask` 

328 Mask plane for determining regions where Sky sources can be added. 

329 seed : `int` 

330 Seed value to initialize the random number generator. 

331 """ 

332 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

333 if skySourceFootprints: 

334 for foot in skySourceFootprints: 

335 s = diaSources.addNew() 

336 s.setFootprint(foot) 

337 s.set(self.skySourceKey, True) 

338 

339 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

340 """Use (matched) template and science image to constrain dipole fitting. 

341 

342 Parameters 

343 ---------- 

344 diaSources : `lsst.afw.table.SourceCatalog` 

345 The catalog of detected sources. 

346 science : `lsst.afw.image.ExposureF` 

347 Science exposure that the template was subtracted from. 

348 difference : `lsst.afw.image.ExposureF` 

349 Result of subtracting template from the science image. 

350 matchedTemplate : `lsst.afw.image.ExposureF` 

351 Warped and PSF-matched template that was used produce the 

352 difference image. 

353 """ 

354 # Note that this may not be correct if we convolved the science image. 

355 # In the future we may wish to persist the matchedScience image. 

356 self.measurement.run(diaSources, difference, science, matchedTemplate) 

357 if self.config.doApCorr: 

358 self.applyApCorr.run( 

359 catalog=diaSources, 

360 apCorrMap=difference.getInfo().getApCorrMap() 

361 ) 

362 

363 def measureForcedSources(self, diaSources, science, wcs): 

364 """Perform forced measurement of the diaSources on the science image. 

365 

366 Parameters 

367 ---------- 

368 diaSources : `lsst.afw.table.SourceCatalog` 

369 The catalog of detected sources. 

370 science : `lsst.afw.image.ExposureF` 

371 Science exposure that the template was subtracted from. 

372 wcs : `lsst.afw.geom.SkyWcs` 

373 Coordinate system definition (wcs) for the exposure. 

374 """ 

375 # Run forced psf photometry on the PVI at the diaSource locations. 

376 # Copy the measured flux and error into the diaSource. 

377 forcedSources = self.forcedMeasurement.generateMeasCat( 

378 science, diaSources, wcs) 

379 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

380 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

381 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

382 "ip_diffim_forced_PsfFlux_instFlux", True) 

383 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

384 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

385 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

386 "ip_diffim_forced_PsfFlux_area", True) 

387 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

388 "ip_diffim_forced_PsfFlux_flag", True) 

389 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

390 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

391 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

392 "ip_diffim_forced_PsfFlux_flag_edge", True) 

393 for diaSource, forcedSource in zip(diaSources, forcedSources): 

394 diaSource.assign(forcedSource, mapper)