Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 34%

144 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-14 02:51 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import lsst.afw.table as afwTable 

23import lsst.daf.base as dafBase 

24from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

25from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask 

26import lsst.meas.extensions.trailedSources # noqa: F401 

27import lsst.meas.extensions.shapeHSM 

28from lsst.obs.base import ExposureIdInfo 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.utils 

32from lsst.utils.timer import timeMethod 

33 

34from . import DipoleFitTask 

35 

36__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask", 

37 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"] 

38 

39 

40class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

41 dimensions=("instrument", "visit", "detector"), 

42 defaultTemplates={"coaddName": "deep", 

43 "warpTypeSuffix": "", 

44 "fakesType": ""}): 

45 science = pipeBase.connectionTypes.Input( 

46 doc="Input science exposure.", 

47 dimensions=("instrument", "visit", "detector"), 

48 storageClass="ExposureF", 

49 name="{fakesType}calexp" 

50 ) 

51 matchedTemplate = pipeBase.connectionTypes.Input( 

52 doc="Warped and PSF-matched template used to create the difference image.", 

53 dimensions=("instrument", "visit", "detector"), 

54 storageClass="ExposureF", 

55 name="{fakesType}{coaddName}Diff_matchedExp", 

56 ) 

57 difference = pipeBase.connectionTypes.Input( 

58 doc="Result of subtracting template from science.", 

59 dimensions=("instrument", "visit", "detector"), 

60 storageClass="ExposureF", 

61 name="{fakesType}{coaddName}Diff_differenceTempExp", 

62 ) 

63 outputSchema = pipeBase.connectionTypes.InitOutput( 

64 doc="Schema (as an example catalog) for output DIASource catalog.", 

65 storageClass="SourceCatalog", 

66 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

67 ) 

68 diaSources = pipeBase.connectionTypes.Output( 

69 doc="Detected diaSources on the difference image.", 

70 dimensions=("instrument", "visit", "detector"), 

71 storageClass="SourceCatalog", 

72 name="{fakesType}{coaddName}Diff_diaSrc", 

73 ) 

74 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

75 doc="Difference image with detection mask plane filled in.", 

76 dimensions=("instrument", "visit", "detector"), 

77 storageClass="ExposureF", 

78 name="{fakesType}{coaddName}Diff_differenceExp", 

79 ) 

80 

81 

82class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

83 pipelineConnections=DetectAndMeasureConnections): 

84 """Config for DetectAndMeasureTask 

85 """ 

86 doMerge = pexConfig.Field( 

87 dtype=bool, 

88 default=True, 

89 doc="Merge positive and negative diaSources with grow radius " 

90 "set by growFootprint" 

91 ) 

92 doForcedMeasurement = pexConfig.Field( 

93 dtype=bool, 

94 default=True, 

95 doc="Force photometer diaSource locations on PVI?") 

96 doAddMetrics = pexConfig.Field( 

97 dtype=bool, 

98 default=False, 

99 doc="Add columns to the source table to hold analysis metrics?" 

100 ) 

101 detection = pexConfig.ConfigurableField( 

102 target=SourceDetectionTask, 

103 doc="Final source detection for diaSource measurement", 

104 ) 

105 measurement = pexConfig.ConfigurableField( 

106 target=DipoleFitTask, 

107 doc="Task to measure sources on the difference image.", 

108 ) 

109 doApCorr = lsst.pex.config.Field( 

110 dtype=bool, 

111 default=True, 

112 doc="Run subtask to apply aperture corrections" 

113 ) 

114 applyApCorr = lsst.pex.config.ConfigurableField( 

115 target=ApplyApCorrTask, 

116 doc="Task to apply aperture corrections" 

117 ) 

118 forcedMeasurement = pexConfig.ConfigurableField( 

119 target=ForcedMeasurementTask, 

120 doc="Task to force photometer science image at diaSource locations.", 

121 ) 

122 growFootprint = pexConfig.Field( 

123 dtype=int, 

124 default=2, 

125 doc="Grow positive and negative footprints by this many pixels before merging" 

126 ) 

127 diaSourceMatchRadius = pexConfig.Field( 

128 dtype=float, 

129 default=0.5, 

130 doc="Match radius (in arcseconds) for DiaSource to Source association" 

131 ) 

132 doSkySources = pexConfig.Field( 

133 dtype=bool, 

134 default=False, 

135 doc="Generate sky sources?", 

136 ) 

137 skySources = pexConfig.ConfigurableField( 

138 target=SkyObjectsTask, 

139 doc="Generate sky sources", 

140 ) 

141 

142 def setDefaults(self): 

143 # DiaSource Detection 

144 self.detection.thresholdPolarity = "both" 

145 self.detection.thresholdValue = 5.0 

146 self.detection.reEstimateBackground = False 

147 self.detection.thresholdType = "pixel_stdev" 

148 

149 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

150 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

151 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

152 'base_LocalPhotoCalib', 

153 'base_LocalWcs', 

154 'ext_shapeHSM_HsmSourceMoments', 

155 'ext_shapeHSM_HsmPsfMoments', 

156 ] 

157 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

158 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

159 

160 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

161 self.forcedMeasurement.copyColumns = { 

162 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

163 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

164 self.forcedMeasurement.slots.shape = None 

165 

166 

167class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

168 """Detect and measure sources on a difference image. 

169 """ 

170 ConfigClass = DetectAndMeasureConfig 

171 _DefaultName = "detectAndMeasure" 

172 

173 def __init__(self, **kwargs): 

174 super().__init__(**kwargs) 

175 self.schema = afwTable.SourceTable.makeMinimalSchema() 

176 

177 self.algMetadata = dafBase.PropertyList() 

178 self.makeSubtask("detection", schema=self.schema) 

179 self.makeSubtask("measurement", schema=self.schema, 

180 algMetadata=self.algMetadata) 

181 if self.config.doApCorr: 

182 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

183 if self.config.doForcedMeasurement: 

184 self.schema.addField( 

185 "ip_diffim_forced_PsfFlux_instFlux", "D", 

186 "Forced PSF flux measured on the direct image.", 

187 units="count") 

188 self.schema.addField( 

189 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

190 "Forced PSF flux error measured on the direct image.", 

191 units="count") 

192 self.schema.addField( 

193 "ip_diffim_forced_PsfFlux_area", "F", 

194 "Forced PSF flux effective area of PSF.", 

195 units="pixel") 

196 self.schema.addField( 

197 "ip_diffim_forced_PsfFlux_flag", "Flag", 

198 "Forced PSF flux general failure flag.") 

199 self.schema.addField( 

200 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

201 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

202 self.schema.addField( 

203 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

204 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

205 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

206 

207 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

208 self.schema.addField("srcMatchId", "L", "unique id of source match") 

209 if self.config.doSkySources: 

210 self.makeSubtask("skySources") 

211 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

212 

213 # initialize InitOutputs 

214 self.outputSchema = afwTable.SourceCatalog(self.schema) 

215 self.outputSchema.getTable().setMetadata(self.algMetadata) 

216 

217 @staticmethod 

218 def makeIdFactory(expId, expBits): 

219 """Create IdFactory instance for unique 64 bit diaSource id-s. 

220 

221 Parameters 

222 ---------- 

223 expId : `int` 

224 Exposure id. 

225 

226 expBits: `int` 

227 Number of used bits in ``expId``. 

228 

229 Notes 

230 ----- 

231 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

232 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

233 low value end of the integer. 

234 

235 Returns 

236 ------- 

237 idFactory: `lsst.afw.table.IdFactory` 

238 """ 

239 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

240 

241 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

242 inputRefs: pipeBase.InputQuantizedConnection, 

243 outputRefs: pipeBase.OutputQuantizedConnection): 

244 inputs = butlerQC.get(inputRefs) 

245 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

246 returnMaxBits=True) 

247 idFactory = self.makeIdFactory(expId=expId, expBits=expBits) 

248 

249 outputs = self.run(**inputs, idFactory=idFactory) 

250 butlerQC.put(outputs, outputRefs) 

251 

252 @timeMethod 

253 def run(self, science, matchedTemplate, difference, 

254 idFactory=None): 

255 """Detect and measure sources on a difference image. 

256 

257 The difference image will be convolved with a gaussian approximation of 

258 the PSF to form a maximum likelihood image for detection. 

259 Close positive and negative detections will optionally be merged into 

260 dipole diaSources. 

261 Sky sources, or forced detections in background regions, will optionally 

262 be added, and the configured measurement algorithm will be run on all 

263 detections. 

264 

265 Parameters 

266 ---------- 

267 science : `lsst.afw.image.ExposureF` 

268 Science exposure that the template was subtracted from. 

269 matchedTemplate : `lsst.afw.image.ExposureF` 

270 Warped and PSF-matched template that was used produce the 

271 difference image. 

272 difference : `lsst.afw.image.ExposureF` 

273 Result of subtracting template from the science image. 

274 idFactory : `lsst.afw.table.IdFactory`, optional 

275 Generator object to assign ids to detected sources in the difference image. 

276 

277 Returns 

278 ------- 

279 measurementResults : `lsst.pipe.base.Struct` 

280 

281 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

282 Subtracted exposure with detection mask applied. 

283 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

284 The catalog of detected sources. 

285 """ 

286 # Ensure that we start with an empty detection mask. 

287 mask = difference.mask 

288 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

289 

290 table = afwTable.SourceTable.make(self.schema, idFactory) 

291 table.setMetadata(self.algMetadata) 

292 results = self.detection.run( 

293 table=table, 

294 exposure=difference, 

295 doSmooth=True, 

296 ) 

297 

298 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

299 positiveFootprints=results.positive, negativeFootprints=results.negative) 

300 

301 def processResults(self, science, matchedTemplate, difference, sources, table, 

302 positiveFootprints=None, negativeFootprints=None,): 

303 """Measure and process the results of source detection. 

304 

305 Parameters 

306 ---------- 

307 sources : `lsst.afw.table.SourceCatalog` 

308 Detected sources on the difference exposure. 

309 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional 

310 Positive polarity footprints. 

311 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional 

312 Negative polarity footprints. 

313 table : `lsst.afw.table.SourceTable` 

314 Table object that will be used to create the SourceCatalog. 

315 science : `lsst.afw.image.ExposureF` 

316 Science exposure that the template was subtracted from. 

317 matchedTemplate : `lsst.afw.image.ExposureF` 

318 Warped and PSF-matched template that was used produce the 

319 difference image. 

320 difference : `lsst.afw.image.ExposureF` 

321 Result of subtracting template from the science image. 

322 

323 Returns 

324 ------- 

325 measurementResults : `lsst.pipe.base.Struct` 

326 

327 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

328 Subtracted exposure with detection mask applied. 

329 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

330 The catalog of detected sources. 

331 """ 

332 if self.config.doMerge: 

333 fpSet = positiveFootprints 

334 fpSet.merge(negativeFootprints, self.config.growFootprint, 

335 self.config.growFootprint, False) 

336 diaSources = afwTable.SourceCatalog(table) 

337 fpSet.makeSources(diaSources) 

338 self.log.info("Merging detections into %d sources", len(diaSources)) 

339 else: 

340 diaSources = sources 

341 

342 if self.config.doSkySources: 

343 self.addSkySources(diaSources, difference.mask, difference.info.id) 

344 

345 self.measureDiaSources(diaSources, science, difference, matchedTemplate) 

346 

347 if self.config.doForcedMeasurement: 

348 self.measureForcedSources(diaSources, science, difference.getWcs()) 

349 

350 measurementResults = pipeBase.Struct( 

351 subtractedMeasuredExposure=difference, 

352 diaSources=diaSources, 

353 ) 

354 

355 return measurementResults 

356 

357 def addSkySources(self, diaSources, mask, seed): 

358 """Add sources in empty regions of the difference image 

359 for measuring the background. 

360 

361 Parameters 

362 ---------- 

363 diaSources : `lsst.afw.table.SourceCatalog` 

364 The catalog of detected sources. 

365 mask : `lsst.afw.image.Mask` 

366 Mask plane for determining regions where Sky sources can be added. 

367 seed : `int` 

368 Seed value to initialize the random number generator. 

369 """ 

370 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

371 if skySourceFootprints: 

372 for foot in skySourceFootprints: 

373 s = diaSources.addNew() 

374 s.setFootprint(foot) 

375 s.set(self.skySourceKey, True) 

376 

377 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

378 """Use (matched) template and science image to constrain dipole fitting. 

379 

380 Parameters 

381 ---------- 

382 diaSources : `lsst.afw.table.SourceCatalog` 

383 The catalog of detected sources. 

384 science : `lsst.afw.image.ExposureF` 

385 Science exposure that the template was subtracted from. 

386 difference : `lsst.afw.image.ExposureF` 

387 Result of subtracting template from the science image. 

388 matchedTemplate : `lsst.afw.image.ExposureF` 

389 Warped and PSF-matched template that was used produce the 

390 difference image. 

391 """ 

392 # Note that this may not be correct if we convolved the science image. 

393 # In the future we may wish to persist the matchedScience image. 

394 self.measurement.run(diaSources, difference, science, matchedTemplate) 

395 if self.config.doApCorr: 

396 self.applyApCorr.run( 

397 catalog=diaSources, 

398 apCorrMap=difference.getInfo().getApCorrMap() 

399 ) 

400 

401 def measureForcedSources(self, diaSources, science, wcs): 

402 """Perform forced measurement of the diaSources on the science image. 

403 

404 Parameters 

405 ---------- 

406 diaSources : `lsst.afw.table.SourceCatalog` 

407 The catalog of detected sources. 

408 science : `lsst.afw.image.ExposureF` 

409 Science exposure that the template was subtracted from. 

410 wcs : `lsst.afw.geom.SkyWcs` 

411 Coordinate system definition (wcs) for the exposure. 

412 """ 

413 # Run forced psf photometry on the PVI at the diaSource locations. 

414 # Copy the measured flux and error into the diaSource. 

415 forcedSources = self.forcedMeasurement.generateMeasCat( 

416 science, diaSources, wcs) 

417 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

418 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

419 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

420 "ip_diffim_forced_PsfFlux_instFlux", True) 

421 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

422 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

423 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

424 "ip_diffim_forced_PsfFlux_area", True) 

425 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

426 "ip_diffim_forced_PsfFlux_flag", True) 

427 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

428 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

429 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

430 "ip_diffim_forced_PsfFlux_flag_edge", True) 

431 for diaSource, forcedSource in zip(diaSources, forcedSources): 

432 diaSource.assign(forcedSource, mapper) 

433 

434 

435class DetectAndMeasureScoreConnections(DetectAndMeasureConnections): 

436 scoreExposure = pipeBase.connectionTypes.Input( 

437 doc="Maximum likelihood image for detection.", 

438 dimensions=("instrument", "visit", "detector"), 

439 storageClass="ExposureF", 

440 name="{fakesType}{coaddName}Diff_scoreExp", 

441 ) 

442 

443 

444class DetectAndMeasureScoreConfig(DetectAndMeasureConfig, 

445 pipelineConnections=DetectAndMeasureScoreConnections): 

446 pass 

447 

448 

449class DetectAndMeasureScoreTask(DetectAndMeasureTask): 

450 """Detect DIA sources using a score image, 

451 and measure the detections on the difference image. 

452 

453 Source detection is run on the supplied score, or maximum likelihood, 

454 image. Note that no additional convolution will be done in this case. 

455 Close positive and negative detections will optionally be merged into 

456 dipole diaSources. 

457 Sky sources, or forced detections in background regions, will optionally 

458 be added, and the configured measurement algorithm will be run on all 

459 detections. 

460 """ 

461 ConfigClass = DetectAndMeasureScoreConfig 

462 _DefaultName = "detectAndMeasureScore" 

463 

464 @timeMethod 

465 def run(self, science, matchedTemplate, difference, scoreExposure, 

466 idFactory=None): 

467 """Detect and measure sources on a score image. 

468 

469 Parameters 

470 ---------- 

471 science : `lsst.afw.image.ExposureF` 

472 Science exposure that the template was subtracted from. 

473 matchedTemplate : `lsst.afw.image.ExposureF` 

474 Warped and PSF-matched template that was used produce the 

475 difference image. 

476 difference : `lsst.afw.image.ExposureF` 

477 Result of subtracting template from the science image. 

478 scoreExposure : `lsst.afw.image.ExposureF` 

479 Score or maximum likelihood difference image 

480 idFactory : `lsst.afw.table.IdFactory`, optional 

481 Generator object to assign ids to detected sources in the difference image. 

482 

483 Returns 

484 ------- 

485 measurementResults : `lsst.pipe.base.Struct` 

486 

487 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

488 Subtracted exposure with detection mask applied. 

489 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

490 The catalog of detected sources. 

491 """ 

492 # Ensure that we start with an empty detection mask. 

493 mask = scoreExposure.mask 

494 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

495 

496 table = afwTable.SourceTable.make(self.schema, idFactory) 

497 table.setMetadata(self.algMetadata) 

498 # Exclude the edge of the CCD from detection. 

499 # This operation would be performed in the detection subtask if doSmooth=True 

500 # but we need to apply the cut here since we are using a preconvolved image. 

501 goodBBox = scoreExposure.getPsf().getKernel().shrinkBBox(scoreExposure.getBBox()) 

502 results = self.detection.run( 

503 table=table, 

504 exposure=scoreExposure[goodBBox], 

505 doSmooth=False, 

506 ) 

507 # Copy the detection mask from the Score image to the difference image 

508 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox()) 

509 

510 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

511 positiveFootprints=results.positive, negativeFootprints=results.negative)