Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 38%

152 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-25 17:01 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from deprecated.sphinx import deprecated 

23 

24import lsst.afw.table as afwTable 

25import lsst.daf.base as dafBase 

26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig 

28import lsst.meas.extensions.trailedSources # noqa: F401 

29import lsst.meas.extensions.shapeHSM 

30from lsst.obs.base import ExposureIdInfo 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33import lsst.utils 

34from lsst.utils.timer import timeMethod 

35 

36from . import DipoleFitTask 

37 

38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask", 

39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"] 

40 

41 

42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

43 dimensions=("instrument", "visit", "detector"), 

44 defaultTemplates={"coaddName": "deep", 

45 "warpTypeSuffix": "", 

46 "fakesType": ""}): 

47 science = pipeBase.connectionTypes.Input( 

48 doc="Input science exposure.", 

49 dimensions=("instrument", "visit", "detector"), 

50 storageClass="ExposureF", 

51 name="{fakesType}calexp" 

52 ) 

53 matchedTemplate = pipeBase.connectionTypes.Input( 

54 doc="Warped and PSF-matched template used to create the difference image.", 

55 dimensions=("instrument", "visit", "detector"), 

56 storageClass="ExposureF", 

57 name="{fakesType}{coaddName}Diff_matchedExp", 

58 ) 

59 difference = pipeBase.connectionTypes.Input( 

60 doc="Result of subtracting template from science.", 

61 dimensions=("instrument", "visit", "detector"), 

62 storageClass="ExposureF", 

63 name="{fakesType}{coaddName}Diff_differenceTempExp", 

64 ) 

65 outputSchema = pipeBase.connectionTypes.InitOutput( 

66 doc="Schema (as an example catalog) for output DIASource catalog.", 

67 storageClass="SourceCatalog", 

68 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

69 ) 

70 diaSources = pipeBase.connectionTypes.Output( 

71 doc="Detected diaSources on the difference image.", 

72 dimensions=("instrument", "visit", "detector"), 

73 storageClass="SourceCatalog", 

74 name="{fakesType}{coaddName}Diff_diaSrc", 

75 ) 

76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

77 doc="Difference image with detection mask plane filled in.", 

78 dimensions=("instrument", "visit", "detector"), 

79 storageClass="ExposureF", 

80 name="{fakesType}{coaddName}Diff_differenceExp", 

81 ) 

82 

83 

84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

85 pipelineConnections=DetectAndMeasureConnections): 

86 """Config for DetectAndMeasureTask 

87 """ 

88 doMerge = pexConfig.Field( 

89 dtype=bool, 

90 default=True, 

91 doc="Merge positive and negative diaSources with grow radius " 

92 "set by growFootprint" 

93 ) 

94 doForcedMeasurement = pexConfig.Field( 

95 dtype=bool, 

96 default=True, 

97 doc="Force photometer diaSource locations on PVI?") 

98 doAddMetrics = pexConfig.Field( 

99 dtype=bool, 

100 default=False, 

101 doc="Add columns to the source table to hold analysis metrics?" 

102 ) 

103 detection = pexConfig.ConfigurableField( 

104 target=SourceDetectionTask, 

105 doc="Final source detection for diaSource measurement", 

106 ) 

107 measurement = pexConfig.ConfigurableField( 

108 target=DipoleFitTask, 

109 doc="Task to measure sources on the difference image.", 

110 ) 

111 doApCorr = lsst.pex.config.Field( 

112 dtype=bool, 

113 default=True, 

114 doc="Run subtask to apply aperture corrections" 

115 ) 

116 applyApCorr = lsst.pex.config.ConfigurableField( 

117 target=ApplyApCorrTask, 

118 doc="Task to apply aperture corrections" 

119 ) 

120 forcedMeasurement = pexConfig.ConfigurableField( 

121 target=ForcedMeasurementTask, 

122 doc="Task to force photometer science image at diaSource locations.", 

123 ) 

124 growFootprint = pexConfig.Field( 

125 dtype=int, 

126 default=2, 

127 doc="Grow positive and negative footprints by this many pixels before merging" 

128 ) 

129 diaSourceMatchRadius = pexConfig.Field( 

130 dtype=float, 

131 default=0.5, 

132 doc="Match radius (in arcseconds) for DiaSource to Source association" 

133 ) 

134 doSkySources = pexConfig.Field( 

135 dtype=bool, 

136 default=False, 

137 doc="Generate sky sources?", 

138 ) 

139 skySources = pexConfig.ConfigurableField( 

140 target=SkyObjectsTask, 

141 doc="Generate sky sources", 

142 ) 

143 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

144 

145 def setDefaults(self): 

146 # DiaSource Detection 

147 self.detection.thresholdPolarity = "both" 

148 self.detection.thresholdValue = 5.0 

149 self.detection.reEstimateBackground = False 

150 self.detection.thresholdType = "pixel_stdev" 

151 self.detection.excludeMaskPlanes = ["EDGE"] 

152 

153 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

155 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

156 'base_LocalPhotoCalib', 

157 'base_LocalWcs', 

158 'ext_shapeHSM_HsmSourceMoments', 

159 'ext_shapeHSM_HsmPsfMoments', 

160 ] 

161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

163 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0 

164 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0 

165 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

166 self.forcedMeasurement.copyColumns = { 

167 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

168 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

169 self.forcedMeasurement.slots.shape = None 

170 

171 

172class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

173 """Detect and measure sources on a difference image. 

174 """ 

175 ConfigClass = DetectAndMeasureConfig 

176 _DefaultName = "detectAndMeasure" 

177 

178 def __init__(self, **kwargs): 

179 super().__init__(**kwargs) 

180 self.schema = afwTable.SourceTable.makeMinimalSchema() 

181 

182 self.algMetadata = dafBase.PropertyList() 

183 self.makeSubtask("detection", schema=self.schema) 

184 self.makeSubtask("measurement", schema=self.schema, 

185 algMetadata=self.algMetadata) 

186 if self.config.doApCorr: 

187 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

188 if self.config.doForcedMeasurement: 

189 self.schema.addField( 

190 "ip_diffim_forced_PsfFlux_instFlux", "D", 

191 "Forced PSF flux measured on the direct image.", 

192 units="count") 

193 self.schema.addField( 

194 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

195 "Forced PSF flux error measured on the direct image.", 

196 units="count") 

197 self.schema.addField( 

198 "ip_diffim_forced_PsfFlux_area", "F", 

199 "Forced PSF flux effective area of PSF.", 

200 units="pixel") 

201 self.schema.addField( 

202 "ip_diffim_forced_PsfFlux_flag", "Flag", 

203 "Forced PSF flux general failure flag.") 

204 self.schema.addField( 

205 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

206 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

207 self.schema.addField( 

208 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

209 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

210 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

211 

212 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

213 self.schema.addField("srcMatchId", "L", "unique id of source match") 

214 if self.config.doSkySources: 

215 self.makeSubtask("skySources") 

216 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

217 

218 # initialize InitOutputs 

219 self.outputSchema = afwTable.SourceCatalog(self.schema) 

220 self.outputSchema.getTable().setMetadata(self.algMetadata) 

221 

222 # TODO: remove on DM-38687. 

223 @staticmethod 

224 @deprecated( 

225 reason=( 

226 "ID factory construction now depends on configuration; use the " 

227 "idGenerator config field. Will be removed after v26." 

228 ), 

229 version="v26.0", 

230 category=FutureWarning, 

231 ) 

232 def makeIdFactory(expId, expBits): 

233 """Create IdFactory instance for unique 64 bit diaSource id-s. 

234 

235 Parameters 

236 ---------- 

237 expId : `int` 

238 Exposure id. 

239 

240 expBits: `int` 

241 Number of used bits in ``expId``. 

242 

243 Notes 

244 ----- 

245 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

246 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

247 low value end of the integer. 

248 

249 Returns 

250 ------- 

251 idFactory: `lsst.afw.table.IdFactory` 

252 """ 

253 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

254 

255 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

256 inputRefs: pipeBase.InputQuantizedConnection, 

257 outputRefs: pipeBase.OutputQuantizedConnection): 

258 inputs = butlerQC.get(inputRefs) 

259 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

260 idFactory = idGenerator.make_table_id_factory() 

261 outputs = self.run(**inputs, idFactory=idFactory) 

262 butlerQC.put(outputs, outputRefs) 

263 

264 @timeMethod 

265 def run(self, science, matchedTemplate, difference, 

266 idFactory=None): 

267 """Detect and measure sources on a difference image. 

268 

269 The difference image will be convolved with a gaussian approximation of 

270 the PSF to form a maximum likelihood image for detection. 

271 Close positive and negative detections will optionally be merged into 

272 dipole diaSources. 

273 Sky sources, or forced detections in background regions, will optionally 

274 be added, and the configured measurement algorithm will be run on all 

275 detections. 

276 

277 Parameters 

278 ---------- 

279 science : `lsst.afw.image.ExposureF` 

280 Science exposure that the template was subtracted from. 

281 matchedTemplate : `lsst.afw.image.ExposureF` 

282 Warped and PSF-matched template that was used produce the 

283 difference image. 

284 difference : `lsst.afw.image.ExposureF` 

285 Result of subtracting template from the science image. 

286 idFactory : `lsst.afw.table.IdFactory`, optional 

287 Generator object to assign ids to detected sources in the difference image. 

288 

289 Returns 

290 ------- 

291 measurementResults : `lsst.pipe.base.Struct` 

292 

293 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

294 Subtracted exposure with detection mask applied. 

295 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

296 The catalog of detected sources. 

297 """ 

298 # Ensure that we start with an empty detection mask. 

299 mask = difference.mask 

300 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

301 

302 table = afwTable.SourceTable.make(self.schema, idFactory) 

303 table.setMetadata(self.algMetadata) 

304 results = self.detection.run( 

305 table=table, 

306 exposure=difference, 

307 doSmooth=True, 

308 ) 

309 

310 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

311 positiveFootprints=results.positive, negativeFootprints=results.negative) 

312 

313 def processResults(self, science, matchedTemplate, difference, sources, table, 

314 positiveFootprints=None, negativeFootprints=None,): 

315 """Measure and process the results of source detection. 

316 

317 Parameters 

318 ---------- 

319 sources : `lsst.afw.table.SourceCatalog` 

320 Detected sources on the difference exposure. 

321 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional 

322 Positive polarity footprints. 

323 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional 

324 Negative polarity footprints. 

325 table : `lsst.afw.table.SourceTable` 

326 Table object that will be used to create the SourceCatalog. 

327 science : `lsst.afw.image.ExposureF` 

328 Science exposure that the template was subtracted from. 

329 matchedTemplate : `lsst.afw.image.ExposureF` 

330 Warped and PSF-matched template that was used produce the 

331 difference image. 

332 difference : `lsst.afw.image.ExposureF` 

333 Result of subtracting template from the science image. 

334 

335 Returns 

336 ------- 

337 measurementResults : `lsst.pipe.base.Struct` 

338 

339 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

340 Subtracted exposure with detection mask applied. 

341 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

342 The catalog of detected sources. 

343 """ 

344 if self.config.doMerge: 

345 fpSet = positiveFootprints 

346 fpSet.merge(negativeFootprints, self.config.growFootprint, 

347 self.config.growFootprint, False) 

348 diaSources = afwTable.SourceCatalog(table) 

349 fpSet.makeSources(diaSources) 

350 self.log.info("Merging detections into %d sources", len(diaSources)) 

351 else: 

352 diaSources = sources 

353 

354 if self.config.doSkySources: 

355 self.addSkySources(diaSources, difference.mask, difference.info.id) 

356 

357 self.measureDiaSources(diaSources, science, difference, matchedTemplate) 

358 

359 if self.config.doForcedMeasurement: 

360 self.measureForcedSources(diaSources, science, difference.getWcs()) 

361 

362 measurementResults = pipeBase.Struct( 

363 subtractedMeasuredExposure=difference, 

364 diaSources=diaSources, 

365 ) 

366 

367 return measurementResults 

368 

369 def addSkySources(self, diaSources, mask, seed): 

370 """Add sources in empty regions of the difference image 

371 for measuring the background. 

372 

373 Parameters 

374 ---------- 

375 diaSources : `lsst.afw.table.SourceCatalog` 

376 The catalog of detected sources. 

377 mask : `lsst.afw.image.Mask` 

378 Mask plane for determining regions where Sky sources can be added. 

379 seed : `int` 

380 Seed value to initialize the random number generator. 

381 """ 

382 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

383 if skySourceFootprints: 

384 for foot in skySourceFootprints: 

385 s = diaSources.addNew() 

386 s.setFootprint(foot) 

387 s.set(self.skySourceKey, True) 

388 

389 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

390 """Use (matched) template and science image to constrain dipole fitting. 

391 

392 Parameters 

393 ---------- 

394 diaSources : `lsst.afw.table.SourceCatalog` 

395 The catalog of detected sources. 

396 science : `lsst.afw.image.ExposureF` 

397 Science exposure that the template was subtracted from. 

398 difference : `lsst.afw.image.ExposureF` 

399 Result of subtracting template from the science image. 

400 matchedTemplate : `lsst.afw.image.ExposureF` 

401 Warped and PSF-matched template that was used produce the 

402 difference image. 

403 """ 

404 # Note that this may not be correct if we convolved the science image. 

405 # In the future we may wish to persist the matchedScience image. 

406 self.measurement.run(diaSources, difference, science, matchedTemplate) 

407 if self.config.doApCorr: 

408 apCorrMap = difference.getInfo().getApCorrMap() 

409 if apCorrMap is None: 

410 self.log.warning("Difference image does not have valid aperture correction; skipping.") 

411 else: 

412 self.applyApCorr.run( 

413 catalog=diaSources, 

414 apCorrMap=apCorrMap, 

415 ) 

416 

417 def measureForcedSources(self, diaSources, science, wcs): 

418 """Perform forced measurement of the diaSources on the science image. 

419 

420 Parameters 

421 ---------- 

422 diaSources : `lsst.afw.table.SourceCatalog` 

423 The catalog of detected sources. 

424 science : `lsst.afw.image.ExposureF` 

425 Science exposure that the template was subtracted from. 

426 wcs : `lsst.afw.geom.SkyWcs` 

427 Coordinate system definition (wcs) for the exposure. 

428 """ 

429 # Run forced psf photometry on the PVI at the diaSource locations. 

430 # Copy the measured flux and error into the diaSource. 

431 forcedSources = self.forcedMeasurement.generateMeasCat( 

432 science, diaSources, wcs) 

433 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

434 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

435 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

436 "ip_diffim_forced_PsfFlux_instFlux", True) 

437 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

438 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

439 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

440 "ip_diffim_forced_PsfFlux_area", True) 

441 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

442 "ip_diffim_forced_PsfFlux_flag", True) 

443 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

444 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

445 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

446 "ip_diffim_forced_PsfFlux_flag_edge", True) 

447 for diaSource, forcedSource in zip(diaSources, forcedSources): 

448 diaSource.assign(forcedSource, mapper) 

449 

450 

451class DetectAndMeasureScoreConnections(DetectAndMeasureConnections): 

452 scoreExposure = pipeBase.connectionTypes.Input( 

453 doc="Maximum likelihood image for detection.", 

454 dimensions=("instrument", "visit", "detector"), 

455 storageClass="ExposureF", 

456 name="{fakesType}{coaddName}Diff_scoreExp", 

457 ) 

458 

459 

460class DetectAndMeasureScoreConfig(DetectAndMeasureConfig, 

461 pipelineConnections=DetectAndMeasureScoreConnections): 

462 pass 

463 

464 

465class DetectAndMeasureScoreTask(DetectAndMeasureTask): 

466 """Detect DIA sources using a score image, 

467 and measure the detections on the difference image. 

468 

469 Source detection is run on the supplied score, or maximum likelihood, 

470 image. Note that no additional convolution will be done in this case. 

471 Close positive and negative detections will optionally be merged into 

472 dipole diaSources. 

473 Sky sources, or forced detections in background regions, will optionally 

474 be added, and the configured measurement algorithm will be run on all 

475 detections. 

476 """ 

477 ConfigClass = DetectAndMeasureScoreConfig 

478 _DefaultName = "detectAndMeasureScore" 

479 

480 @timeMethod 

481 def run(self, science, matchedTemplate, difference, scoreExposure, 

482 idFactory=None): 

483 """Detect and measure sources on a score image. 

484 

485 Parameters 

486 ---------- 

487 science : `lsst.afw.image.ExposureF` 

488 Science exposure that the template was subtracted from. 

489 matchedTemplate : `lsst.afw.image.ExposureF` 

490 Warped and PSF-matched template that was used produce the 

491 difference image. 

492 difference : `lsst.afw.image.ExposureF` 

493 Result of subtracting template from the science image. 

494 scoreExposure : `lsst.afw.image.ExposureF` 

495 Score or maximum likelihood difference image 

496 idFactory : `lsst.afw.table.IdFactory`, optional 

497 Generator object to assign ids to detected sources in the difference image. 

498 

499 Returns 

500 ------- 

501 measurementResults : `lsst.pipe.base.Struct` 

502 

503 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

504 Subtracted exposure with detection mask applied. 

505 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

506 The catalog of detected sources. 

507 """ 

508 # Ensure that we start with an empty detection mask. 

509 mask = scoreExposure.mask 

510 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

511 

512 table = afwTable.SourceTable.make(self.schema, idFactory) 

513 table.setMetadata(self.algMetadata) 

514 results = self.detection.run( 

515 table=table, 

516 exposure=scoreExposure, 

517 doSmooth=False, 

518 ) 

519 # Copy the detection mask from the Score image to the difference image 

520 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox()) 

521 

522 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

523 positiveFootprints=results.positive, negativeFootprints=results.negative)