Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 35%

147 statements  

« prev     ^ index     » next       coverage.py v7.2.6, created at 2023-05-24 09:18 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from deprecated.sphinx import deprecated 

23 

24import lsst.afw.table as afwTable 

25import lsst.daf.base as dafBase 

26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig 

28import lsst.meas.extensions.trailedSources # noqa: F401 

29import lsst.meas.extensions.shapeHSM 

30from lsst.obs.base import ExposureIdInfo 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33import lsst.utils 

34from lsst.utils.timer import timeMethod 

35 

36from . import DipoleFitTask 

37 

38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask", 

39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"] 

40 

41 

42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

43 dimensions=("instrument", "visit", "detector"), 

44 defaultTemplates={"coaddName": "deep", 

45 "warpTypeSuffix": "", 

46 "fakesType": ""}): 

47 science = pipeBase.connectionTypes.Input( 

48 doc="Input science exposure.", 

49 dimensions=("instrument", "visit", "detector"), 

50 storageClass="ExposureF", 

51 name="{fakesType}calexp" 

52 ) 

53 matchedTemplate = pipeBase.connectionTypes.Input( 

54 doc="Warped and PSF-matched template used to create the difference image.", 

55 dimensions=("instrument", "visit", "detector"), 

56 storageClass="ExposureF", 

57 name="{fakesType}{coaddName}Diff_matchedExp", 

58 ) 

59 difference = pipeBase.connectionTypes.Input( 

60 doc="Result of subtracting template from science.", 

61 dimensions=("instrument", "visit", "detector"), 

62 storageClass="ExposureF", 

63 name="{fakesType}{coaddName}Diff_differenceTempExp", 

64 ) 

65 outputSchema = pipeBase.connectionTypes.InitOutput( 

66 doc="Schema (as an example catalog) for output DIASource catalog.", 

67 storageClass="SourceCatalog", 

68 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

69 ) 

70 diaSources = pipeBase.connectionTypes.Output( 

71 doc="Detected diaSources on the difference image.", 

72 dimensions=("instrument", "visit", "detector"), 

73 storageClass="SourceCatalog", 

74 name="{fakesType}{coaddName}Diff_diaSrc", 

75 ) 

76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

77 doc="Difference image with detection mask plane filled in.", 

78 dimensions=("instrument", "visit", "detector"), 

79 storageClass="ExposureF", 

80 name="{fakesType}{coaddName}Diff_differenceExp", 

81 ) 

82 

83 

84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

85 pipelineConnections=DetectAndMeasureConnections): 

86 """Config for DetectAndMeasureTask 

87 """ 

88 doMerge = pexConfig.Field( 

89 dtype=bool, 

90 default=True, 

91 doc="Merge positive and negative diaSources with grow radius " 

92 "set by growFootprint" 

93 ) 

94 doForcedMeasurement = pexConfig.Field( 

95 dtype=bool, 

96 default=True, 

97 doc="Force photometer diaSource locations on PVI?") 

98 doAddMetrics = pexConfig.Field( 

99 dtype=bool, 

100 default=False, 

101 doc="Add columns to the source table to hold analysis metrics?" 

102 ) 

103 detection = pexConfig.ConfigurableField( 

104 target=SourceDetectionTask, 

105 doc="Final source detection for diaSource measurement", 

106 ) 

107 measurement = pexConfig.ConfigurableField( 

108 target=DipoleFitTask, 

109 doc="Task to measure sources on the difference image.", 

110 ) 

111 doApCorr = lsst.pex.config.Field( 

112 dtype=bool, 

113 default=True, 

114 doc="Run subtask to apply aperture corrections" 

115 ) 

116 applyApCorr = lsst.pex.config.ConfigurableField( 

117 target=ApplyApCorrTask, 

118 doc="Task to apply aperture corrections" 

119 ) 

120 forcedMeasurement = pexConfig.ConfigurableField( 

121 target=ForcedMeasurementTask, 

122 doc="Task to force photometer science image at diaSource locations.", 

123 ) 

124 growFootprint = pexConfig.Field( 

125 dtype=int, 

126 default=2, 

127 doc="Grow positive and negative footprints by this many pixels before merging" 

128 ) 

129 diaSourceMatchRadius = pexConfig.Field( 

130 dtype=float, 

131 default=0.5, 

132 doc="Match radius (in arcseconds) for DiaSource to Source association" 

133 ) 

134 doSkySources = pexConfig.Field( 

135 dtype=bool, 

136 default=False, 

137 doc="Generate sky sources?", 

138 ) 

139 skySources = pexConfig.ConfigurableField( 

140 target=SkyObjectsTask, 

141 doc="Generate sky sources", 

142 ) 

143 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

144 

145 def setDefaults(self): 

146 # DiaSource Detection 

147 self.detection.thresholdPolarity = "both" 

148 self.detection.thresholdValue = 5.0 

149 self.detection.reEstimateBackground = False 

150 self.detection.thresholdType = "pixel_stdev" 

151 self.detection.excludeMaskPlanes = ["EDGE"] 

152 

153 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

155 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

156 'base_LocalPhotoCalib', 

157 'base_LocalWcs', 

158 'ext_shapeHSM_HsmSourceMoments', 

159 'ext_shapeHSM_HsmPsfMoments', 

160 ] 

161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

163 

164 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

165 self.forcedMeasurement.copyColumns = { 

166 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

167 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

168 self.forcedMeasurement.slots.shape = None 

169 

170 

171class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

172 """Detect and measure sources on a difference image. 

173 """ 

174 ConfigClass = DetectAndMeasureConfig 

175 _DefaultName = "detectAndMeasure" 

176 

177 def __init__(self, **kwargs): 

178 super().__init__(**kwargs) 

179 self.schema = afwTable.SourceTable.makeMinimalSchema() 

180 

181 self.algMetadata = dafBase.PropertyList() 

182 self.makeSubtask("detection", schema=self.schema) 

183 self.makeSubtask("measurement", schema=self.schema, 

184 algMetadata=self.algMetadata) 

185 if self.config.doApCorr: 

186 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

187 if self.config.doForcedMeasurement: 

188 self.schema.addField( 

189 "ip_diffim_forced_PsfFlux_instFlux", "D", 

190 "Forced PSF flux measured on the direct image.", 

191 units="count") 

192 self.schema.addField( 

193 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

194 "Forced PSF flux error measured on the direct image.", 

195 units="count") 

196 self.schema.addField( 

197 "ip_diffim_forced_PsfFlux_area", "F", 

198 "Forced PSF flux effective area of PSF.", 

199 units="pixel") 

200 self.schema.addField( 

201 "ip_diffim_forced_PsfFlux_flag", "Flag", 

202 "Forced PSF flux general failure flag.") 

203 self.schema.addField( 

204 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

205 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

206 self.schema.addField( 

207 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

208 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

209 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

210 

211 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

212 self.schema.addField("srcMatchId", "L", "unique id of source match") 

213 if self.config.doSkySources: 

214 self.makeSubtask("skySources") 

215 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

216 

217 # initialize InitOutputs 

218 self.outputSchema = afwTable.SourceCatalog(self.schema) 

219 self.outputSchema.getTable().setMetadata(self.algMetadata) 

220 

221 @staticmethod 

222 @deprecated( 

223 reason=( 

224 "ID factory construction now depends on configuration; use the " 

225 "idGenerator config field. Will be removed after v27." 

226 ), 

227 version="v26.0", 

228 category=FutureWarning, 

229 ) 

230 def makeIdFactory(expId, expBits): 

231 """Create IdFactory instance for unique 64 bit diaSource id-s. 

232 

233 Parameters 

234 ---------- 

235 expId : `int` 

236 Exposure id. 

237 

238 expBits: `int` 

239 Number of used bits in ``expId``. 

240 

241 Notes 

242 ----- 

243 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

244 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

245 low value end of the integer. 

246 

247 Returns 

248 ------- 

249 idFactory: `lsst.afw.table.IdFactory` 

250 """ 

251 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

252 

253 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

254 inputRefs: pipeBase.InputQuantizedConnection, 

255 outputRefs: pipeBase.OutputQuantizedConnection): 

256 inputs = butlerQC.get(inputRefs) 

257 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

258 idFactory = idGenerator.make_table_id_factory() 

259 outputs = self.run(**inputs, idFactory=idFactory) 

260 butlerQC.put(outputs, outputRefs) 

261 

262 @timeMethod 

263 def run(self, science, matchedTemplate, difference, 

264 idFactory=None): 

265 """Detect and measure sources on a difference image. 

266 

267 The difference image will be convolved with a gaussian approximation of 

268 the PSF to form a maximum likelihood image for detection. 

269 Close positive and negative detections will optionally be merged into 

270 dipole diaSources. 

271 Sky sources, or forced detections in background regions, will optionally 

272 be added, and the configured measurement algorithm will be run on all 

273 detections. 

274 

275 Parameters 

276 ---------- 

277 science : `lsst.afw.image.ExposureF` 

278 Science exposure that the template was subtracted from. 

279 matchedTemplate : `lsst.afw.image.ExposureF` 

280 Warped and PSF-matched template that was used produce the 

281 difference image. 

282 difference : `lsst.afw.image.ExposureF` 

283 Result of subtracting template from the science image. 

284 idFactory : `lsst.afw.table.IdFactory`, optional 

285 Generator object to assign ids to detected sources in the difference image. 

286 

287 Returns 

288 ------- 

289 measurementResults : `lsst.pipe.base.Struct` 

290 

291 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

292 Subtracted exposure with detection mask applied. 

293 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

294 The catalog of detected sources. 

295 """ 

296 # Ensure that we start with an empty detection mask. 

297 mask = difference.mask 

298 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

299 

300 table = afwTable.SourceTable.make(self.schema, idFactory) 

301 table.setMetadata(self.algMetadata) 

302 results = self.detection.run( 

303 table=table, 

304 exposure=difference, 

305 doSmooth=True, 

306 ) 

307 

308 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

309 positiveFootprints=results.positive, negativeFootprints=results.negative) 

310 

311 def processResults(self, science, matchedTemplate, difference, sources, table, 

312 positiveFootprints=None, negativeFootprints=None,): 

313 """Measure and process the results of source detection. 

314 

315 Parameters 

316 ---------- 

317 sources : `lsst.afw.table.SourceCatalog` 

318 Detected sources on the difference exposure. 

319 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional 

320 Positive polarity footprints. 

321 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional 

322 Negative polarity footprints. 

323 table : `lsst.afw.table.SourceTable` 

324 Table object that will be used to create the SourceCatalog. 

325 science : `lsst.afw.image.ExposureF` 

326 Science exposure that the template was subtracted from. 

327 matchedTemplate : `lsst.afw.image.ExposureF` 

328 Warped and PSF-matched template that was used produce the 

329 difference image. 

330 difference : `lsst.afw.image.ExposureF` 

331 Result of subtracting template from the science image. 

332 

333 Returns 

334 ------- 

335 measurementResults : `lsst.pipe.base.Struct` 

336 

337 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

338 Subtracted exposure with detection mask applied. 

339 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

340 The catalog of detected sources. 

341 """ 

342 if self.config.doMerge: 

343 fpSet = positiveFootprints 

344 fpSet.merge(negativeFootprints, self.config.growFootprint, 

345 self.config.growFootprint, False) 

346 diaSources = afwTable.SourceCatalog(table) 

347 fpSet.makeSources(diaSources) 

348 self.log.info("Merging detections into %d sources", len(diaSources)) 

349 else: 

350 diaSources = sources 

351 

352 if self.config.doSkySources: 

353 self.addSkySources(diaSources, difference.mask, difference.info.id) 

354 

355 self.measureDiaSources(diaSources, science, difference, matchedTemplate) 

356 

357 if self.config.doForcedMeasurement: 

358 self.measureForcedSources(diaSources, science, difference.getWcs()) 

359 

360 measurementResults = pipeBase.Struct( 

361 subtractedMeasuredExposure=difference, 

362 diaSources=diaSources, 

363 ) 

364 

365 return measurementResults 

366 

367 def addSkySources(self, diaSources, mask, seed): 

368 """Add sources in empty regions of the difference image 

369 for measuring the background. 

370 

371 Parameters 

372 ---------- 

373 diaSources : `lsst.afw.table.SourceCatalog` 

374 The catalog of detected sources. 

375 mask : `lsst.afw.image.Mask` 

376 Mask plane for determining regions where Sky sources can be added. 

377 seed : `int` 

378 Seed value to initialize the random number generator. 

379 """ 

380 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

381 if skySourceFootprints: 

382 for foot in skySourceFootprints: 

383 s = diaSources.addNew() 

384 s.setFootprint(foot) 

385 s.set(self.skySourceKey, True) 

386 

387 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

388 """Use (matched) template and science image to constrain dipole fitting. 

389 

390 Parameters 

391 ---------- 

392 diaSources : `lsst.afw.table.SourceCatalog` 

393 The catalog of detected sources. 

394 science : `lsst.afw.image.ExposureF` 

395 Science exposure that the template was subtracted from. 

396 difference : `lsst.afw.image.ExposureF` 

397 Result of subtracting template from the science image. 

398 matchedTemplate : `lsst.afw.image.ExposureF` 

399 Warped and PSF-matched template that was used produce the 

400 difference image. 

401 """ 

402 # Note that this may not be correct if we convolved the science image. 

403 # In the future we may wish to persist the matchedScience image. 

404 self.measurement.run(diaSources, difference, science, matchedTemplate) 

405 if self.config.doApCorr: 

406 self.applyApCorr.run( 

407 catalog=diaSources, 

408 apCorrMap=difference.getInfo().getApCorrMap() 

409 ) 

410 

411 def measureForcedSources(self, diaSources, science, wcs): 

412 """Perform forced measurement of the diaSources on the science image. 

413 

414 Parameters 

415 ---------- 

416 diaSources : `lsst.afw.table.SourceCatalog` 

417 The catalog of detected sources. 

418 science : `lsst.afw.image.ExposureF` 

419 Science exposure that the template was subtracted from. 

420 wcs : `lsst.afw.geom.SkyWcs` 

421 Coordinate system definition (wcs) for the exposure. 

422 """ 

423 # Run forced psf photometry on the PVI at the diaSource locations. 

424 # Copy the measured flux and error into the diaSource. 

425 forcedSources = self.forcedMeasurement.generateMeasCat( 

426 science, diaSources, wcs) 

427 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

428 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

429 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

430 "ip_diffim_forced_PsfFlux_instFlux", True) 

431 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

432 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

433 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

434 "ip_diffim_forced_PsfFlux_area", True) 

435 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

436 "ip_diffim_forced_PsfFlux_flag", True) 

437 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

438 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

439 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

440 "ip_diffim_forced_PsfFlux_flag_edge", True) 

441 for diaSource, forcedSource in zip(diaSources, forcedSources): 

442 diaSource.assign(forcedSource, mapper) 

443 

444 

445class DetectAndMeasureScoreConnections(DetectAndMeasureConnections): 

446 scoreExposure = pipeBase.connectionTypes.Input( 

447 doc="Maximum likelihood image for detection.", 

448 dimensions=("instrument", "visit", "detector"), 

449 storageClass="ExposureF", 

450 name="{fakesType}{coaddName}Diff_scoreExp", 

451 ) 

452 

453 

454class DetectAndMeasureScoreConfig(DetectAndMeasureConfig, 

455 pipelineConnections=DetectAndMeasureScoreConnections): 

456 pass 

457 

458 

459class DetectAndMeasureScoreTask(DetectAndMeasureTask): 

460 """Detect DIA sources using a score image, 

461 and measure the detections on the difference image. 

462 

463 Source detection is run on the supplied score, or maximum likelihood, 

464 image. Note that no additional convolution will be done in this case. 

465 Close positive and negative detections will optionally be merged into 

466 dipole diaSources. 

467 Sky sources, or forced detections in background regions, will optionally 

468 be added, and the configured measurement algorithm will be run on all 

469 detections. 

470 """ 

471 ConfigClass = DetectAndMeasureScoreConfig 

472 _DefaultName = "detectAndMeasureScore" 

473 

474 @timeMethod 

475 def run(self, science, matchedTemplate, difference, scoreExposure, 

476 idFactory=None): 

477 """Detect and measure sources on a score image. 

478 

479 Parameters 

480 ---------- 

481 science : `lsst.afw.image.ExposureF` 

482 Science exposure that the template was subtracted from. 

483 matchedTemplate : `lsst.afw.image.ExposureF` 

484 Warped and PSF-matched template that was used produce the 

485 difference image. 

486 difference : `lsst.afw.image.ExposureF` 

487 Result of subtracting template from the science image. 

488 scoreExposure : `lsst.afw.image.ExposureF` 

489 Score or maximum likelihood difference image 

490 idFactory : `lsst.afw.table.IdFactory`, optional 

491 Generator object to assign ids to detected sources in the difference image. 

492 

493 Returns 

494 ------- 

495 measurementResults : `lsst.pipe.base.Struct` 

496 

497 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

498 Subtracted exposure with detection mask applied. 

499 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

500 The catalog of detected sources. 

501 """ 

502 # Ensure that we start with an empty detection mask. 

503 mask = scoreExposure.mask 

504 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

505 

506 table = afwTable.SourceTable.make(self.schema, idFactory) 

507 table.setMetadata(self.algMetadata) 

508 results = self.detection.run( 

509 table=table, 

510 exposure=scoreExposure, 

511 doSmooth=False, 

512 ) 

513 # Copy the detection mask from the Score image to the difference image 

514 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox()) 

515 

516 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

517 positiveFootprints=results.positive, negativeFootprints=results.negative)