Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 37%

155 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-16 12:14 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from deprecated.sphinx import deprecated 

23 

24import lsst.afw.table as afwTable 

25import lsst.daf.base as dafBase 

26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig 

28import lsst.meas.extensions.trailedSources # noqa: F401 

29import lsst.meas.extensions.shapeHSM 

30from lsst.obs.base import ExposureIdInfo 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33import lsst.utils 

34from lsst.utils.timer import timeMethod 

35 

36from . import DipoleFitTask 

37 

38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask", 

39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"] 

40 

41 

42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

43 dimensions=("instrument", "visit", "detector"), 

44 defaultTemplates={"coaddName": "deep", 

45 "warpTypeSuffix": "", 

46 "fakesType": ""}): 

47 science = pipeBase.connectionTypes.Input( 

48 doc="Input science exposure.", 

49 dimensions=("instrument", "visit", "detector"), 

50 storageClass="ExposureF", 

51 name="{fakesType}calexp" 

52 ) 

53 matchedTemplate = pipeBase.connectionTypes.Input( 

54 doc="Warped and PSF-matched template used to create the difference image.", 

55 dimensions=("instrument", "visit", "detector"), 

56 storageClass="ExposureF", 

57 name="{fakesType}{coaddName}Diff_matchedExp", 

58 ) 

59 difference = pipeBase.connectionTypes.Input( 

60 doc="Result of subtracting template from science.", 

61 dimensions=("instrument", "visit", "detector"), 

62 storageClass="ExposureF", 

63 name="{fakesType}{coaddName}Diff_differenceTempExp", 

64 ) 

65 outputSchema = pipeBase.connectionTypes.InitOutput( 

66 doc="Schema (as an example catalog) for output DIASource catalog.", 

67 storageClass="SourceCatalog", 

68 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

69 ) 

70 diaSources = pipeBase.connectionTypes.Output( 

71 doc="Detected diaSources on the difference image.", 

72 dimensions=("instrument", "visit", "detector"), 

73 storageClass="SourceCatalog", 

74 name="{fakesType}{coaddName}Diff_diaSrc", 

75 ) 

76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

77 doc="Difference image with detection mask plane filled in.", 

78 dimensions=("instrument", "visit", "detector"), 

79 storageClass="ExposureF", 

80 name="{fakesType}{coaddName}Diff_differenceExp", 

81 ) 

82 

83 

84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

85 pipelineConnections=DetectAndMeasureConnections): 

86 """Config for DetectAndMeasureTask 

87 """ 

88 doMerge = pexConfig.Field( 

89 dtype=bool, 

90 default=True, 

91 doc="Merge positive and negative diaSources with grow radius " 

92 "set by growFootprint" 

93 ) 

94 doForcedMeasurement = pexConfig.Field( 

95 dtype=bool, 

96 default=True, 

97 doc="Force photometer diaSource locations on PVI?") 

98 doAddMetrics = pexConfig.Field( 

99 dtype=bool, 

100 default=False, 

101 doc="Add columns to the source table to hold analysis metrics?" 

102 ) 

103 detection = pexConfig.ConfigurableField( 

104 target=SourceDetectionTask, 

105 doc="Final source detection for diaSource measurement", 

106 ) 

107 measurement = pexConfig.ConfigurableField( 

108 target=DipoleFitTask, 

109 doc="Task to measure sources on the difference image.", 

110 ) 

111 doApCorr = lsst.pex.config.Field( 

112 dtype=bool, 

113 default=True, 

114 doc="Run subtask to apply aperture corrections" 

115 ) 

116 applyApCorr = lsst.pex.config.ConfigurableField( 

117 target=ApplyApCorrTask, 

118 doc="Task to apply aperture corrections" 

119 ) 

120 forcedMeasurement = pexConfig.ConfigurableField( 

121 target=ForcedMeasurementTask, 

122 doc="Task to force photometer science image at diaSource locations.", 

123 ) 

124 growFootprint = pexConfig.Field( 

125 dtype=int, 

126 default=2, 

127 doc="Grow positive and negative footprints by this many pixels before merging" 

128 ) 

129 diaSourceMatchRadius = pexConfig.Field( 

130 dtype=float, 

131 default=0.5, 

132 doc="Match radius (in arcseconds) for DiaSource to Source association" 

133 ) 

134 doSkySources = pexConfig.Field( 

135 dtype=bool, 

136 default=False, 

137 doc="Generate sky sources?", 

138 ) 

139 skySources = pexConfig.ConfigurableField( 

140 target=SkyObjectsTask, 

141 doc="Generate sky sources", 

142 ) 

143 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

144 

145 def setDefaults(self): 

146 # DiaSource Detection 

147 self.detection.thresholdPolarity = "both" 

148 self.detection.thresholdValue = 5.0 

149 self.detection.reEstimateBackground = False 

150 self.detection.thresholdType = "pixel_stdev" 

151 self.detection.excludeMaskPlanes = ["EDGE"] 

152 

153 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

155 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

156 'base_LocalPhotoCalib', 

157 'base_LocalWcs', 

158 'ext_shapeHSM_HsmSourceMoments', 

159 'ext_shapeHSM_HsmPsfMoments', 

160 ] 

161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

163 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0 

164 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0 

165 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

166 self.forcedMeasurement.copyColumns = { 

167 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

168 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

169 self.forcedMeasurement.slots.shape = None 

170 

171 # Keep track of which footprints contain streaks 

172 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] 

173 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] 

174 

175 

176class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

177 """Detect and measure sources on a difference image. 

178 """ 

179 ConfigClass = DetectAndMeasureConfig 

180 _DefaultName = "detectAndMeasure" 

181 

182 def __init__(self, **kwargs): 

183 super().__init__(**kwargs) 

184 self.schema = afwTable.SourceTable.makeMinimalSchema() 

185 # Add coordinate error fields: 

186 afwTable.CoordKey.addErrorFields(self.schema) 

187 

188 self.algMetadata = dafBase.PropertyList() 

189 self.makeSubtask("detection", schema=self.schema) 

190 self.makeSubtask("measurement", schema=self.schema, 

191 algMetadata=self.algMetadata) 

192 if self.config.doApCorr: 

193 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

194 if self.config.doForcedMeasurement: 

195 self.schema.addField( 

196 "ip_diffim_forced_PsfFlux_instFlux", "D", 

197 "Forced PSF flux measured on the direct image.", 

198 units="count") 

199 self.schema.addField( 

200 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

201 "Forced PSF flux error measured on the direct image.", 

202 units="count") 

203 self.schema.addField( 

204 "ip_diffim_forced_PsfFlux_area", "F", 

205 "Forced PSF flux effective area of PSF.", 

206 units="pixel") 

207 self.schema.addField( 

208 "ip_diffim_forced_PsfFlux_flag", "Flag", 

209 "Forced PSF flux general failure flag.") 

210 self.schema.addField( 

211 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

212 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

213 self.schema.addField( 

214 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

215 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

216 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

217 

218 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

219 self.schema.addField("srcMatchId", "L", "unique id of source match") 

220 if self.config.doSkySources: 

221 self.makeSubtask("skySources") 

222 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

223 

224 # initialize InitOutputs 

225 self.outputSchema = afwTable.SourceCatalog(self.schema) 

226 self.outputSchema.getTable().setMetadata(self.algMetadata) 

227 

228 # TODO: remove on DM-38687. 

229 @staticmethod 

230 @deprecated( 

231 reason=( 

232 "ID factory construction now depends on configuration; use the " 

233 "idGenerator config field. Will be removed after v26." 

234 ), 

235 version="v26.0", 

236 category=FutureWarning, 

237 ) 

238 def makeIdFactory(expId, expBits): 

239 """Create IdFactory instance for unique 64 bit diaSource id-s. 

240 

241 Parameters 

242 ---------- 

243 expId : `int` 

244 Exposure id. 

245 

246 expBits: `int` 

247 Number of used bits in ``expId``. 

248 

249 Notes 

250 ----- 

251 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

252 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

253 low value end of the integer. 

254 

255 Returns 

256 ------- 

257 idFactory: `lsst.afw.table.IdFactory` 

258 """ 

259 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

260 

261 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

262 inputRefs: pipeBase.InputQuantizedConnection, 

263 outputRefs: pipeBase.OutputQuantizedConnection): 

264 inputs = butlerQC.get(inputRefs) 

265 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

266 idFactory = idGenerator.make_table_id_factory() 

267 outputs = self.run(**inputs, idFactory=idFactory) 

268 butlerQC.put(outputs, outputRefs) 

269 

270 @timeMethod 

271 def run(self, science, matchedTemplate, difference, 

272 idFactory=None): 

273 """Detect and measure sources on a difference image. 

274 

275 The difference image will be convolved with a gaussian approximation of 

276 the PSF to form a maximum likelihood image for detection. 

277 Close positive and negative detections will optionally be merged into 

278 dipole diaSources. 

279 Sky sources, or forced detections in background regions, will optionally 

280 be added, and the configured measurement algorithm will be run on all 

281 detections. 

282 

283 Parameters 

284 ---------- 

285 science : `lsst.afw.image.ExposureF` 

286 Science exposure that the template was subtracted from. 

287 matchedTemplate : `lsst.afw.image.ExposureF` 

288 Warped and PSF-matched template that was used produce the 

289 difference image. 

290 difference : `lsst.afw.image.ExposureF` 

291 Result of subtracting template from the science image. 

292 idFactory : `lsst.afw.table.IdFactory`, optional 

293 Generator object to assign ids to detected sources in the difference image. 

294 

295 Returns 

296 ------- 

297 measurementResults : `lsst.pipe.base.Struct` 

298 

299 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

300 Subtracted exposure with detection mask applied. 

301 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

302 The catalog of detected sources. 

303 """ 

304 # Ensure that we start with an empty detection mask. 

305 mask = difference.mask 

306 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

307 

308 table = afwTable.SourceTable.make(self.schema, idFactory) 

309 table.setMetadata(self.algMetadata) 

310 results = self.detection.run( 

311 table=table, 

312 exposure=difference, 

313 doSmooth=True, 

314 ) 

315 

316 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

317 positiveFootprints=results.positive, negativeFootprints=results.negative) 

318 

319 def processResults(self, science, matchedTemplate, difference, sources, table, 

320 positiveFootprints=None, negativeFootprints=None,): 

321 """Measure and process the results of source detection. 

322 

323 Parameters 

324 ---------- 

325 sources : `lsst.afw.table.SourceCatalog` 

326 Detected sources on the difference exposure. 

327 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional 

328 Positive polarity footprints. 

329 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional 

330 Negative polarity footprints. 

331 table : `lsst.afw.table.SourceTable` 

332 Table object that will be used to create the SourceCatalog. 

333 science : `lsst.afw.image.ExposureF` 

334 Science exposure that the template was subtracted from. 

335 matchedTemplate : `lsst.afw.image.ExposureF` 

336 Warped and PSF-matched template that was used produce the 

337 difference image. 

338 difference : `lsst.afw.image.ExposureF` 

339 Result of subtracting template from the science image. 

340 

341 Returns 

342 ------- 

343 measurementResults : `lsst.pipe.base.Struct` 

344 

345 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

346 Subtracted exposure with detection mask applied. 

347 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

348 The catalog of detected sources. 

349 """ 

350 if self.config.doMerge: 

351 fpSet = positiveFootprints 

352 fpSet.merge(negativeFootprints, self.config.growFootprint, 

353 self.config.growFootprint, False) 

354 diaSources = afwTable.SourceCatalog(table) 

355 fpSet.makeSources(diaSources) 

356 self.log.info("Merging detections into %d sources", len(diaSources)) 

357 else: 

358 diaSources = sources 

359 

360 if self.config.doSkySources: 

361 self.addSkySources(diaSources, difference.mask, difference.info.id) 

362 

363 self.measureDiaSources(diaSources, science, difference, matchedTemplate) 

364 

365 if self.config.doForcedMeasurement: 

366 self.measureForcedSources(diaSources, science, difference.getWcs()) 

367 

368 measurementResults = pipeBase.Struct( 

369 subtractedMeasuredExposure=difference, 

370 diaSources=diaSources, 

371 ) 

372 

373 return measurementResults 

374 

375 def addSkySources(self, diaSources, mask, seed): 

376 """Add sources in empty regions of the difference image 

377 for measuring the background. 

378 

379 Parameters 

380 ---------- 

381 diaSources : `lsst.afw.table.SourceCatalog` 

382 The catalog of detected sources. 

383 mask : `lsst.afw.image.Mask` 

384 Mask plane for determining regions where Sky sources can be added. 

385 seed : `int` 

386 Seed value to initialize the random number generator. 

387 """ 

388 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

389 if skySourceFootprints: 

390 for foot in skySourceFootprints: 

391 s = diaSources.addNew() 

392 s.setFootprint(foot) 

393 s.set(self.skySourceKey, True) 

394 

395 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

396 """Use (matched) template and science image to constrain dipole fitting. 

397 

398 Parameters 

399 ---------- 

400 diaSources : `lsst.afw.table.SourceCatalog` 

401 The catalog of detected sources. 

402 science : `lsst.afw.image.ExposureF` 

403 Science exposure that the template was subtracted from. 

404 difference : `lsst.afw.image.ExposureF` 

405 Result of subtracting template from the science image. 

406 matchedTemplate : `lsst.afw.image.ExposureF` 

407 Warped and PSF-matched template that was used produce the 

408 difference image. 

409 """ 

410 # Note that this may not be correct if we convolved the science image. 

411 # In the future we may wish to persist the matchedScience image. 

412 self.measurement.run(diaSources, difference, science, matchedTemplate) 

413 if self.config.doApCorr: 

414 apCorrMap = difference.getInfo().getApCorrMap() 

415 if apCorrMap is None: 

416 self.log.warning("Difference image does not have valid aperture correction; skipping.") 

417 else: 

418 self.applyApCorr.run( 

419 catalog=diaSources, 

420 apCorrMap=apCorrMap, 

421 ) 

422 

423 def measureForcedSources(self, diaSources, science, wcs): 

424 """Perform forced measurement of the diaSources on the science image. 

425 

426 Parameters 

427 ---------- 

428 diaSources : `lsst.afw.table.SourceCatalog` 

429 The catalog of detected sources. 

430 science : `lsst.afw.image.ExposureF` 

431 Science exposure that the template was subtracted from. 

432 wcs : `lsst.afw.geom.SkyWcs` 

433 Coordinate system definition (wcs) for the exposure. 

434 """ 

435 # Run forced psf photometry on the PVI at the diaSource locations. 

436 # Copy the measured flux and error into the diaSource. 

437 forcedSources = self.forcedMeasurement.generateMeasCat( 

438 science, diaSources, wcs) 

439 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

440 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

441 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

442 "ip_diffim_forced_PsfFlux_instFlux", True) 

443 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

444 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

445 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

446 "ip_diffim_forced_PsfFlux_area", True) 

447 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

448 "ip_diffim_forced_PsfFlux_flag", True) 

449 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

450 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

451 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

452 "ip_diffim_forced_PsfFlux_flag_edge", True) 

453 for diaSource, forcedSource in zip(diaSources, forcedSources): 

454 diaSource.assign(forcedSource, mapper) 

455 

456 

457class DetectAndMeasureScoreConnections(DetectAndMeasureConnections): 

458 scoreExposure = pipeBase.connectionTypes.Input( 

459 doc="Maximum likelihood image for detection.", 

460 dimensions=("instrument", "visit", "detector"), 

461 storageClass="ExposureF", 

462 name="{fakesType}{coaddName}Diff_scoreExp", 

463 ) 

464 

465 

466class DetectAndMeasureScoreConfig(DetectAndMeasureConfig, 

467 pipelineConnections=DetectAndMeasureScoreConnections): 

468 pass 

469 

470 

471class DetectAndMeasureScoreTask(DetectAndMeasureTask): 

472 """Detect DIA sources using a score image, 

473 and measure the detections on the difference image. 

474 

475 Source detection is run on the supplied score, or maximum likelihood, 

476 image. Note that no additional convolution will be done in this case. 

477 Close positive and negative detections will optionally be merged into 

478 dipole diaSources. 

479 Sky sources, or forced detections in background regions, will optionally 

480 be added, and the configured measurement algorithm will be run on all 

481 detections. 

482 """ 

483 ConfigClass = DetectAndMeasureScoreConfig 

484 _DefaultName = "detectAndMeasureScore" 

485 

486 @timeMethod 

487 def run(self, science, matchedTemplate, difference, scoreExposure, 

488 idFactory=None): 

489 """Detect and measure sources on a score image. 

490 

491 Parameters 

492 ---------- 

493 science : `lsst.afw.image.ExposureF` 

494 Science exposure that the template was subtracted from. 

495 matchedTemplate : `lsst.afw.image.ExposureF` 

496 Warped and PSF-matched template that was used produce the 

497 difference image. 

498 difference : `lsst.afw.image.ExposureF` 

499 Result of subtracting template from the science image. 

500 scoreExposure : `lsst.afw.image.ExposureF` 

501 Score or maximum likelihood difference image 

502 idFactory : `lsst.afw.table.IdFactory`, optional 

503 Generator object to assign ids to detected sources in the difference image. 

504 

505 Returns 

506 ------- 

507 measurementResults : `lsst.pipe.base.Struct` 

508 

509 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

510 Subtracted exposure with detection mask applied. 

511 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

512 The catalog of detected sources. 

513 """ 

514 # Ensure that we start with an empty detection mask. 

515 mask = scoreExposure.mask 

516 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

517 

518 table = afwTable.SourceTable.make(self.schema, idFactory) 

519 table.setMetadata(self.algMetadata) 

520 results = self.detection.run( 

521 table=table, 

522 exposure=scoreExposure, 

523 doSmooth=False, 

524 ) 

525 # Copy the detection mask from the Score image to the difference image 

526 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox()) 

527 

528 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

529 positiveFootprints=results.positive, negativeFootprints=results.negative)