Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 37%

153 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-12 10:57 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from deprecated.sphinx import deprecated 

23 

24import lsst.afw.table as afwTable 

25import lsst.daf.base as dafBase 

26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig 

28import lsst.meas.extensions.trailedSources # noqa: F401 

29import lsst.meas.extensions.shapeHSM 

30from lsst.obs.base import ExposureIdInfo 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33import lsst.utils 

34from lsst.utils.timer import timeMethod 

35 

36from . import DipoleFitTask 

37 

38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask", 

39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"] 

40 

41 

42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

43 dimensions=("instrument", "visit", "detector"), 

44 defaultTemplates={"coaddName": "deep", 

45 "warpTypeSuffix": "", 

46 "fakesType": ""}): 

47 science = pipeBase.connectionTypes.Input( 

48 doc="Input science exposure.", 

49 dimensions=("instrument", "visit", "detector"), 

50 storageClass="ExposureF", 

51 name="{fakesType}calexp" 

52 ) 

53 matchedTemplate = pipeBase.connectionTypes.Input( 

54 doc="Warped and PSF-matched template used to create the difference image.", 

55 dimensions=("instrument", "visit", "detector"), 

56 storageClass="ExposureF", 

57 name="{fakesType}{coaddName}Diff_matchedExp", 

58 ) 

59 difference = pipeBase.connectionTypes.Input( 

60 doc="Result of subtracting template from science.", 

61 dimensions=("instrument", "visit", "detector"), 

62 storageClass="ExposureF", 

63 name="{fakesType}{coaddName}Diff_differenceTempExp", 

64 ) 

65 outputSchema = pipeBase.connectionTypes.InitOutput( 

66 doc="Schema (as an example catalog) for output DIASource catalog.", 

67 storageClass="SourceCatalog", 

68 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

69 ) 

70 diaSources = pipeBase.connectionTypes.Output( 

71 doc="Detected diaSources on the difference image.", 

72 dimensions=("instrument", "visit", "detector"), 

73 storageClass="SourceCatalog", 

74 name="{fakesType}{coaddName}Diff_diaSrc", 

75 ) 

76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

77 doc="Difference image with detection mask plane filled in.", 

78 dimensions=("instrument", "visit", "detector"), 

79 storageClass="ExposureF", 

80 name="{fakesType}{coaddName}Diff_differenceExp", 

81 ) 

82 

83 

84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

85 pipelineConnections=DetectAndMeasureConnections): 

86 """Config for DetectAndMeasureTask 

87 """ 

88 doMerge = pexConfig.Field( 

89 dtype=bool, 

90 default=True, 

91 doc="Merge positive and negative diaSources with grow radius " 

92 "set by growFootprint" 

93 ) 

94 doForcedMeasurement = pexConfig.Field( 

95 dtype=bool, 

96 default=True, 

97 doc="Force photometer diaSource locations on PVI?") 

98 doAddMetrics = pexConfig.Field( 

99 dtype=bool, 

100 default=False, 

101 doc="Add columns to the source table to hold analysis metrics?" 

102 ) 

103 detection = pexConfig.ConfigurableField( 

104 target=SourceDetectionTask, 

105 doc="Final source detection for diaSource measurement", 

106 ) 

107 measurement = pexConfig.ConfigurableField( 

108 target=DipoleFitTask, 

109 doc="Task to measure sources on the difference image.", 

110 ) 

111 doApCorr = lsst.pex.config.Field( 

112 dtype=bool, 

113 default=True, 

114 doc="Run subtask to apply aperture corrections" 

115 ) 

116 applyApCorr = lsst.pex.config.ConfigurableField( 

117 target=ApplyApCorrTask, 

118 doc="Task to apply aperture corrections" 

119 ) 

120 forcedMeasurement = pexConfig.ConfigurableField( 

121 target=ForcedMeasurementTask, 

122 doc="Task to force photometer science image at diaSource locations.", 

123 ) 

124 growFootprint = pexConfig.Field( 

125 dtype=int, 

126 default=2, 

127 doc="Grow positive and negative footprints by this many pixels before merging" 

128 ) 

129 diaSourceMatchRadius = pexConfig.Field( 

130 dtype=float, 

131 default=0.5, 

132 doc="Match radius (in arcseconds) for DiaSource to Source association" 

133 ) 

134 doSkySources = pexConfig.Field( 

135 dtype=bool, 

136 default=False, 

137 doc="Generate sky sources?", 

138 ) 

139 skySources = pexConfig.ConfigurableField( 

140 target=SkyObjectsTask, 

141 doc="Generate sky sources", 

142 ) 

143 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

144 

145 def setDefaults(self): 

146 # DiaSource Detection 

147 self.detection.thresholdPolarity = "both" 

148 self.detection.thresholdValue = 5.0 

149 self.detection.reEstimateBackground = False 

150 self.detection.thresholdType = "pixel_stdev" 

151 self.detection.excludeMaskPlanes = ["EDGE"] 

152 

153 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

155 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

156 'base_LocalPhotoCalib', 

157 'base_LocalWcs', 

158 'ext_shapeHSM_HsmSourceMoments', 

159 'ext_shapeHSM_HsmPsfMoments', 

160 ] 

161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

163 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0 

164 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0 

165 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

166 self.forcedMeasurement.copyColumns = { 

167 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

168 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

169 self.forcedMeasurement.slots.shape = None 

170 

171 

172class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

173 """Detect and measure sources on a difference image. 

174 """ 

175 ConfigClass = DetectAndMeasureConfig 

176 _DefaultName = "detectAndMeasure" 

177 

178 def __init__(self, **kwargs): 

179 super().__init__(**kwargs) 

180 self.schema = afwTable.SourceTable.makeMinimalSchema() 

181 # Add coordinate error fields: 

182 afwTable.CoordKey.addErrorFields(self.schema) 

183 

184 self.algMetadata = dafBase.PropertyList() 

185 self.makeSubtask("detection", schema=self.schema) 

186 self.makeSubtask("measurement", schema=self.schema, 

187 algMetadata=self.algMetadata) 

188 if self.config.doApCorr: 

189 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

190 if self.config.doForcedMeasurement: 

191 self.schema.addField( 

192 "ip_diffim_forced_PsfFlux_instFlux", "D", 

193 "Forced PSF flux measured on the direct image.", 

194 units="count") 

195 self.schema.addField( 

196 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

197 "Forced PSF flux error measured on the direct image.", 

198 units="count") 

199 self.schema.addField( 

200 "ip_diffim_forced_PsfFlux_area", "F", 

201 "Forced PSF flux effective area of PSF.", 

202 units="pixel") 

203 self.schema.addField( 

204 "ip_diffim_forced_PsfFlux_flag", "Flag", 

205 "Forced PSF flux general failure flag.") 

206 self.schema.addField( 

207 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

208 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

209 self.schema.addField( 

210 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

211 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

212 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

213 

214 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

215 self.schema.addField("srcMatchId", "L", "unique id of source match") 

216 if self.config.doSkySources: 

217 self.makeSubtask("skySources") 

218 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

219 

220 # initialize InitOutputs 

221 self.outputSchema = afwTable.SourceCatalog(self.schema) 

222 self.outputSchema.getTable().setMetadata(self.algMetadata) 

223 

224 # TODO: remove on DM-38687. 

225 @staticmethod 

226 @deprecated( 

227 reason=( 

228 "ID factory construction now depends on configuration; use the " 

229 "idGenerator config field. Will be removed after v26." 

230 ), 

231 version="v26.0", 

232 category=FutureWarning, 

233 ) 

234 def makeIdFactory(expId, expBits): 

235 """Create IdFactory instance for unique 64 bit diaSource id-s. 

236 

237 Parameters 

238 ---------- 

239 expId : `int` 

240 Exposure id. 

241 

242 expBits: `int` 

243 Number of used bits in ``expId``. 

244 

245 Notes 

246 ----- 

247 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

248 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

249 low value end of the integer. 

250 

251 Returns 

252 ------- 

253 idFactory: `lsst.afw.table.IdFactory` 

254 """ 

255 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

256 

257 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

258 inputRefs: pipeBase.InputQuantizedConnection, 

259 outputRefs: pipeBase.OutputQuantizedConnection): 

260 inputs = butlerQC.get(inputRefs) 

261 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

262 idFactory = idGenerator.make_table_id_factory() 

263 outputs = self.run(**inputs, idFactory=idFactory) 

264 butlerQC.put(outputs, outputRefs) 

265 

266 @timeMethod 

267 def run(self, science, matchedTemplate, difference, 

268 idFactory=None): 

269 """Detect and measure sources on a difference image. 

270 

271 The difference image will be convolved with a gaussian approximation of 

272 the PSF to form a maximum likelihood image for detection. 

273 Close positive and negative detections will optionally be merged into 

274 dipole diaSources. 

275 Sky sources, or forced detections in background regions, will optionally 

276 be added, and the configured measurement algorithm will be run on all 

277 detections. 

278 

279 Parameters 

280 ---------- 

281 science : `lsst.afw.image.ExposureF` 

282 Science exposure that the template was subtracted from. 

283 matchedTemplate : `lsst.afw.image.ExposureF` 

284 Warped and PSF-matched template that was used produce the 

285 difference image. 

286 difference : `lsst.afw.image.ExposureF` 

287 Result of subtracting template from the science image. 

288 idFactory : `lsst.afw.table.IdFactory`, optional 

289 Generator object to assign ids to detected sources in the difference image. 

290 

291 Returns 

292 ------- 

293 measurementResults : `lsst.pipe.base.Struct` 

294 

295 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

296 Subtracted exposure with detection mask applied. 

297 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

298 The catalog of detected sources. 

299 """ 

300 # Ensure that we start with an empty detection mask. 

301 mask = difference.mask 

302 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

303 

304 table = afwTable.SourceTable.make(self.schema, idFactory) 

305 table.setMetadata(self.algMetadata) 

306 results = self.detection.run( 

307 table=table, 

308 exposure=difference, 

309 doSmooth=True, 

310 ) 

311 

312 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

313 positiveFootprints=results.positive, negativeFootprints=results.negative) 

314 

315 def processResults(self, science, matchedTemplate, difference, sources, table, 

316 positiveFootprints=None, negativeFootprints=None,): 

317 """Measure and process the results of source detection. 

318 

319 Parameters 

320 ---------- 

321 sources : `lsst.afw.table.SourceCatalog` 

322 Detected sources on the difference exposure. 

323 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional 

324 Positive polarity footprints. 

325 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional 

326 Negative polarity footprints. 

327 table : `lsst.afw.table.SourceTable` 

328 Table object that will be used to create the SourceCatalog. 

329 science : `lsst.afw.image.ExposureF` 

330 Science exposure that the template was subtracted from. 

331 matchedTemplate : `lsst.afw.image.ExposureF` 

332 Warped and PSF-matched template that was used produce the 

333 difference image. 

334 difference : `lsst.afw.image.ExposureF` 

335 Result of subtracting template from the science image. 

336 

337 Returns 

338 ------- 

339 measurementResults : `lsst.pipe.base.Struct` 

340 

341 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

342 Subtracted exposure with detection mask applied. 

343 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

344 The catalog of detected sources. 

345 """ 

346 if self.config.doMerge: 

347 fpSet = positiveFootprints 

348 fpSet.merge(negativeFootprints, self.config.growFootprint, 

349 self.config.growFootprint, False) 

350 diaSources = afwTable.SourceCatalog(table) 

351 fpSet.makeSources(diaSources) 

352 self.log.info("Merging detections into %d sources", len(diaSources)) 

353 else: 

354 diaSources = sources 

355 

356 if self.config.doSkySources: 

357 self.addSkySources(diaSources, difference.mask, difference.info.id) 

358 

359 self.measureDiaSources(diaSources, science, difference, matchedTemplate) 

360 

361 if self.config.doForcedMeasurement: 

362 self.measureForcedSources(diaSources, science, difference.getWcs()) 

363 

364 measurementResults = pipeBase.Struct( 

365 subtractedMeasuredExposure=difference, 

366 diaSources=diaSources, 

367 ) 

368 

369 return measurementResults 

370 

371 def addSkySources(self, diaSources, mask, seed): 

372 """Add sources in empty regions of the difference image 

373 for measuring the background. 

374 

375 Parameters 

376 ---------- 

377 diaSources : `lsst.afw.table.SourceCatalog` 

378 The catalog of detected sources. 

379 mask : `lsst.afw.image.Mask` 

380 Mask plane for determining regions where Sky sources can be added. 

381 seed : `int` 

382 Seed value to initialize the random number generator. 

383 """ 

384 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

385 if skySourceFootprints: 

386 for foot in skySourceFootprints: 

387 s = diaSources.addNew() 

388 s.setFootprint(foot) 

389 s.set(self.skySourceKey, True) 

390 

391 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

392 """Use (matched) template and science image to constrain dipole fitting. 

393 

394 Parameters 

395 ---------- 

396 diaSources : `lsst.afw.table.SourceCatalog` 

397 The catalog of detected sources. 

398 science : `lsst.afw.image.ExposureF` 

399 Science exposure that the template was subtracted from. 

400 difference : `lsst.afw.image.ExposureF` 

401 Result of subtracting template from the science image. 

402 matchedTemplate : `lsst.afw.image.ExposureF` 

403 Warped and PSF-matched template that was used produce the 

404 difference image. 

405 """ 

406 # Note that this may not be correct if we convolved the science image. 

407 # In the future we may wish to persist the matchedScience image. 

408 self.measurement.run(diaSources, difference, science, matchedTemplate) 

409 if self.config.doApCorr: 

410 apCorrMap = difference.getInfo().getApCorrMap() 

411 if apCorrMap is None: 

412 self.log.warning("Difference image does not have valid aperture correction; skipping.") 

413 else: 

414 self.applyApCorr.run( 

415 catalog=diaSources, 

416 apCorrMap=apCorrMap, 

417 ) 

418 

419 def measureForcedSources(self, diaSources, science, wcs): 

420 """Perform forced measurement of the diaSources on the science image. 

421 

422 Parameters 

423 ---------- 

424 diaSources : `lsst.afw.table.SourceCatalog` 

425 The catalog of detected sources. 

426 science : `lsst.afw.image.ExposureF` 

427 Science exposure that the template was subtracted from. 

428 wcs : `lsst.afw.geom.SkyWcs` 

429 Coordinate system definition (wcs) for the exposure. 

430 """ 

431 # Run forced psf photometry on the PVI at the diaSource locations. 

432 # Copy the measured flux and error into the diaSource. 

433 forcedSources = self.forcedMeasurement.generateMeasCat( 

434 science, diaSources, wcs) 

435 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

436 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

437 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

438 "ip_diffim_forced_PsfFlux_instFlux", True) 

439 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

440 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

441 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

442 "ip_diffim_forced_PsfFlux_area", True) 

443 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

444 "ip_diffim_forced_PsfFlux_flag", True) 

445 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

446 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

447 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

448 "ip_diffim_forced_PsfFlux_flag_edge", True) 

449 for diaSource, forcedSource in zip(diaSources, forcedSources): 

450 diaSource.assign(forcedSource, mapper) 

451 

452 

453class DetectAndMeasureScoreConnections(DetectAndMeasureConnections): 

454 scoreExposure = pipeBase.connectionTypes.Input( 

455 doc="Maximum likelihood image for detection.", 

456 dimensions=("instrument", "visit", "detector"), 

457 storageClass="ExposureF", 

458 name="{fakesType}{coaddName}Diff_scoreExp", 

459 ) 

460 

461 

462class DetectAndMeasureScoreConfig(DetectAndMeasureConfig, 

463 pipelineConnections=DetectAndMeasureScoreConnections): 

464 pass 

465 

466 

467class DetectAndMeasureScoreTask(DetectAndMeasureTask): 

468 """Detect DIA sources using a score image, 

469 and measure the detections on the difference image. 

470 

471 Source detection is run on the supplied score, or maximum likelihood, 

472 image. Note that no additional convolution will be done in this case. 

473 Close positive and negative detections will optionally be merged into 

474 dipole diaSources. 

475 Sky sources, or forced detections in background regions, will optionally 

476 be added, and the configured measurement algorithm will be run on all 

477 detections. 

478 """ 

479 ConfigClass = DetectAndMeasureScoreConfig 

480 _DefaultName = "detectAndMeasureScore" 

481 

482 @timeMethod 

483 def run(self, science, matchedTemplate, difference, scoreExposure, 

484 idFactory=None): 

485 """Detect and measure sources on a score image. 

486 

487 Parameters 

488 ---------- 

489 science : `lsst.afw.image.ExposureF` 

490 Science exposure that the template was subtracted from. 

491 matchedTemplate : `lsst.afw.image.ExposureF` 

492 Warped and PSF-matched template that was used produce the 

493 difference image. 

494 difference : `lsst.afw.image.ExposureF` 

495 Result of subtracting template from the science image. 

496 scoreExposure : `lsst.afw.image.ExposureF` 

497 Score or maximum likelihood difference image 

498 idFactory : `lsst.afw.table.IdFactory`, optional 

499 Generator object to assign ids to detected sources in the difference image. 

500 

501 Returns 

502 ------- 

503 measurementResults : `lsst.pipe.base.Struct` 

504 

505 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

506 Subtracted exposure with detection mask applied. 

507 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

508 The catalog of detected sources. 

509 """ 

510 # Ensure that we start with an empty detection mask. 

511 mask = scoreExposure.mask 

512 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

513 

514 table = afwTable.SourceTable.make(self.schema, idFactory) 

515 table.setMetadata(self.algMetadata) 

516 results = self.detection.run( 

517 table=table, 

518 exposure=scoreExposure, 

519 doSmooth=False, 

520 ) 

521 # Copy the detection mask from the Score image to the difference image 

522 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox()) 

523 

524 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

525 positiveFootprints=results.positive, negativeFootprints=results.negative)