Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 35%

149 statements  

« prev     ^ index     » next       coverage.py v7.2.6, created at 2023-05-27 02:48 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from deprecated.sphinx import deprecated 

23 

24import lsst.afw.table as afwTable 

25import lsst.daf.base as dafBase 

26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig 

28import lsst.meas.extensions.trailedSources # noqa: F401 

29import lsst.meas.extensions.shapeHSM 

30from lsst.obs.base import ExposureIdInfo 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33import lsst.utils 

34from lsst.utils.timer import timeMethod 

35 

36from . import DipoleFitTask 

37 

38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask", 

39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"] 

40 

41 

42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

43 dimensions=("instrument", "visit", "detector"), 

44 defaultTemplates={"coaddName": "deep", 

45 "warpTypeSuffix": "", 

46 "fakesType": ""}): 

47 science = pipeBase.connectionTypes.Input( 

48 doc="Input science exposure.", 

49 dimensions=("instrument", "visit", "detector"), 

50 storageClass="ExposureF", 

51 name="{fakesType}calexp" 

52 ) 

53 matchedTemplate = pipeBase.connectionTypes.Input( 

54 doc="Warped and PSF-matched template used to create the difference image.", 

55 dimensions=("instrument", "visit", "detector"), 

56 storageClass="ExposureF", 

57 name="{fakesType}{coaddName}Diff_matchedExp", 

58 ) 

59 difference = pipeBase.connectionTypes.Input( 

60 doc="Result of subtracting template from science.", 

61 dimensions=("instrument", "visit", "detector"), 

62 storageClass="ExposureF", 

63 name="{fakesType}{coaddName}Diff_differenceTempExp", 

64 ) 

65 outputSchema = pipeBase.connectionTypes.InitOutput( 

66 doc="Schema (as an example catalog) for output DIASource catalog.", 

67 storageClass="SourceCatalog", 

68 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

69 ) 

70 diaSources = pipeBase.connectionTypes.Output( 

71 doc="Detected diaSources on the difference image.", 

72 dimensions=("instrument", "visit", "detector"), 

73 storageClass="SourceCatalog", 

74 name="{fakesType}{coaddName}Diff_diaSrc", 

75 ) 

76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

77 doc="Difference image with detection mask plane filled in.", 

78 dimensions=("instrument", "visit", "detector"), 

79 storageClass="ExposureF", 

80 name="{fakesType}{coaddName}Diff_differenceExp", 

81 ) 

82 

83 

84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

85 pipelineConnections=DetectAndMeasureConnections): 

86 """Config for DetectAndMeasureTask 

87 """ 

88 doMerge = pexConfig.Field( 

89 dtype=bool, 

90 default=True, 

91 doc="Merge positive and negative diaSources with grow radius " 

92 "set by growFootprint" 

93 ) 

94 doForcedMeasurement = pexConfig.Field( 

95 dtype=bool, 

96 default=True, 

97 doc="Force photometer diaSource locations on PVI?") 

98 doAddMetrics = pexConfig.Field( 

99 dtype=bool, 

100 default=False, 

101 doc="Add columns to the source table to hold analysis metrics?" 

102 ) 

103 detection = pexConfig.ConfigurableField( 

104 target=SourceDetectionTask, 

105 doc="Final source detection for diaSource measurement", 

106 ) 

107 measurement = pexConfig.ConfigurableField( 

108 target=DipoleFitTask, 

109 doc="Task to measure sources on the difference image.", 

110 ) 

111 doApCorr = lsst.pex.config.Field( 

112 dtype=bool, 

113 default=True, 

114 doc="Run subtask to apply aperture corrections" 

115 ) 

116 applyApCorr = lsst.pex.config.ConfigurableField( 

117 target=ApplyApCorrTask, 

118 doc="Task to apply aperture corrections" 

119 ) 

120 forcedMeasurement = pexConfig.ConfigurableField( 

121 target=ForcedMeasurementTask, 

122 doc="Task to force photometer science image at diaSource locations.", 

123 ) 

124 growFootprint = pexConfig.Field( 

125 dtype=int, 

126 default=2, 

127 doc="Grow positive and negative footprints by this many pixels before merging" 

128 ) 

129 diaSourceMatchRadius = pexConfig.Field( 

130 dtype=float, 

131 default=0.5, 

132 doc="Match radius (in arcseconds) for DiaSource to Source association" 

133 ) 

134 doSkySources = pexConfig.Field( 

135 dtype=bool, 

136 default=False, 

137 doc="Generate sky sources?", 

138 ) 

139 skySources = pexConfig.ConfigurableField( 

140 target=SkyObjectsTask, 

141 doc="Generate sky sources", 

142 ) 

143 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

144 

145 def setDefaults(self): 

146 # DiaSource Detection 

147 self.detection.thresholdPolarity = "both" 

148 self.detection.thresholdValue = 5.0 

149 self.detection.reEstimateBackground = False 

150 self.detection.thresholdType = "pixel_stdev" 

151 self.detection.excludeMaskPlanes = ["EDGE"] 

152 

153 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

155 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

156 'base_LocalPhotoCalib', 

157 'base_LocalWcs', 

158 'ext_shapeHSM_HsmSourceMoments', 

159 'ext_shapeHSM_HsmPsfMoments', 

160 ] 

161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

163 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0 

164 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0 

165 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

166 self.forcedMeasurement.copyColumns = { 

167 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

168 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

169 self.forcedMeasurement.slots.shape = None 

170 

171 

172class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

173 """Detect and measure sources on a difference image. 

174 """ 

175 ConfigClass = DetectAndMeasureConfig 

176 _DefaultName = "detectAndMeasure" 

177 

178 def __init__(self, **kwargs): 

179 super().__init__(**kwargs) 

180 self.schema = afwTable.SourceTable.makeMinimalSchema() 

181 

182 self.algMetadata = dafBase.PropertyList() 

183 self.makeSubtask("detection", schema=self.schema) 

184 self.makeSubtask("measurement", schema=self.schema, 

185 algMetadata=self.algMetadata) 

186 if self.config.doApCorr: 

187 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

188 if self.config.doForcedMeasurement: 

189 self.schema.addField( 

190 "ip_diffim_forced_PsfFlux_instFlux", "D", 

191 "Forced PSF flux measured on the direct image.", 

192 units="count") 

193 self.schema.addField( 

194 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

195 "Forced PSF flux error measured on the direct image.", 

196 units="count") 

197 self.schema.addField( 

198 "ip_diffim_forced_PsfFlux_area", "F", 

199 "Forced PSF flux effective area of PSF.", 

200 units="pixel") 

201 self.schema.addField( 

202 "ip_diffim_forced_PsfFlux_flag", "Flag", 

203 "Forced PSF flux general failure flag.") 

204 self.schema.addField( 

205 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

206 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

207 self.schema.addField( 

208 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

209 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

210 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

211 

212 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

213 self.schema.addField("srcMatchId", "L", "unique id of source match") 

214 if self.config.doSkySources: 

215 self.makeSubtask("skySources") 

216 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

217 

218 # initialize InitOutputs 

219 self.outputSchema = afwTable.SourceCatalog(self.schema) 

220 self.outputSchema.getTable().setMetadata(self.algMetadata) 

221 

222 @staticmethod 

223 @deprecated( 

224 reason=( 

225 "ID factory construction now depends on configuration; use the " 

226 "idGenerator config field. Will be removed after v27." 

227 ), 

228 version="v26.0", 

229 category=FutureWarning, 

230 ) 

231 def makeIdFactory(expId, expBits): 

232 """Create IdFactory instance for unique 64 bit diaSource id-s. 

233 

234 Parameters 

235 ---------- 

236 expId : `int` 

237 Exposure id. 

238 

239 expBits: `int` 

240 Number of used bits in ``expId``. 

241 

242 Notes 

243 ----- 

244 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

245 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

246 low value end of the integer. 

247 

248 Returns 

249 ------- 

250 idFactory: `lsst.afw.table.IdFactory` 

251 """ 

252 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

253 

254 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

255 inputRefs: pipeBase.InputQuantizedConnection, 

256 outputRefs: pipeBase.OutputQuantizedConnection): 

257 inputs = butlerQC.get(inputRefs) 

258 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

259 idFactory = idGenerator.make_table_id_factory() 

260 outputs = self.run(**inputs, idFactory=idFactory) 

261 butlerQC.put(outputs, outputRefs) 

262 

263 @timeMethod 

264 def run(self, science, matchedTemplate, difference, 

265 idFactory=None): 

266 """Detect and measure sources on a difference image. 

267 

268 The difference image will be convolved with a gaussian approximation of 

269 the PSF to form a maximum likelihood image for detection. 

270 Close positive and negative detections will optionally be merged into 

271 dipole diaSources. 

272 Sky sources, or forced detections in background regions, will optionally 

273 be added, and the configured measurement algorithm will be run on all 

274 detections. 

275 

276 Parameters 

277 ---------- 

278 science : `lsst.afw.image.ExposureF` 

279 Science exposure that the template was subtracted from. 

280 matchedTemplate : `lsst.afw.image.ExposureF` 

281 Warped and PSF-matched template that was used produce the 

282 difference image. 

283 difference : `lsst.afw.image.ExposureF` 

284 Result of subtracting template from the science image. 

285 idFactory : `lsst.afw.table.IdFactory`, optional 

286 Generator object to assign ids to detected sources in the difference image. 

287 

288 Returns 

289 ------- 

290 measurementResults : `lsst.pipe.base.Struct` 

291 

292 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

293 Subtracted exposure with detection mask applied. 

294 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

295 The catalog of detected sources. 

296 """ 

297 # Ensure that we start with an empty detection mask. 

298 mask = difference.mask 

299 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

300 

301 table = afwTable.SourceTable.make(self.schema, idFactory) 

302 table.setMetadata(self.algMetadata) 

303 results = self.detection.run( 

304 table=table, 

305 exposure=difference, 

306 doSmooth=True, 

307 ) 

308 

309 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

310 positiveFootprints=results.positive, negativeFootprints=results.negative) 

311 

312 def processResults(self, science, matchedTemplate, difference, sources, table, 

313 positiveFootprints=None, negativeFootprints=None,): 

314 """Measure and process the results of source detection. 

315 

316 Parameters 

317 ---------- 

318 sources : `lsst.afw.table.SourceCatalog` 

319 Detected sources on the difference exposure. 

320 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional 

321 Positive polarity footprints. 

322 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional 

323 Negative polarity footprints. 

324 table : `lsst.afw.table.SourceTable` 

325 Table object that will be used to create the SourceCatalog. 

326 science : `lsst.afw.image.ExposureF` 

327 Science exposure that the template was subtracted from. 

328 matchedTemplate : `lsst.afw.image.ExposureF` 

329 Warped and PSF-matched template that was used produce the 

330 difference image. 

331 difference : `lsst.afw.image.ExposureF` 

332 Result of subtracting template from the science image. 

333 

334 Returns 

335 ------- 

336 measurementResults : `lsst.pipe.base.Struct` 

337 

338 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

339 Subtracted exposure with detection mask applied. 

340 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

341 The catalog of detected sources. 

342 """ 

343 if self.config.doMerge: 

344 fpSet = positiveFootprints 

345 fpSet.merge(negativeFootprints, self.config.growFootprint, 

346 self.config.growFootprint, False) 

347 diaSources = afwTable.SourceCatalog(table) 

348 fpSet.makeSources(diaSources) 

349 self.log.info("Merging detections into %d sources", len(diaSources)) 

350 else: 

351 diaSources = sources 

352 

353 if self.config.doSkySources: 

354 self.addSkySources(diaSources, difference.mask, difference.info.id) 

355 

356 self.measureDiaSources(diaSources, science, difference, matchedTemplate) 

357 

358 if self.config.doForcedMeasurement: 

359 self.measureForcedSources(diaSources, science, difference.getWcs()) 

360 

361 measurementResults = pipeBase.Struct( 

362 subtractedMeasuredExposure=difference, 

363 diaSources=diaSources, 

364 ) 

365 

366 return measurementResults 

367 

368 def addSkySources(self, diaSources, mask, seed): 

369 """Add sources in empty regions of the difference image 

370 for measuring the background. 

371 

372 Parameters 

373 ---------- 

374 diaSources : `lsst.afw.table.SourceCatalog` 

375 The catalog of detected sources. 

376 mask : `lsst.afw.image.Mask` 

377 Mask plane for determining regions where Sky sources can be added. 

378 seed : `int` 

379 Seed value to initialize the random number generator. 

380 """ 

381 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

382 if skySourceFootprints: 

383 for foot in skySourceFootprints: 

384 s = diaSources.addNew() 

385 s.setFootprint(foot) 

386 s.set(self.skySourceKey, True) 

387 

388 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

389 """Use (matched) template and science image to constrain dipole fitting. 

390 

391 Parameters 

392 ---------- 

393 diaSources : `lsst.afw.table.SourceCatalog` 

394 The catalog of detected sources. 

395 science : `lsst.afw.image.ExposureF` 

396 Science exposure that the template was subtracted from. 

397 difference : `lsst.afw.image.ExposureF` 

398 Result of subtracting template from the science image. 

399 matchedTemplate : `lsst.afw.image.ExposureF` 

400 Warped and PSF-matched template that was used produce the 

401 difference image. 

402 """ 

403 # Note that this may not be correct if we convolved the science image. 

404 # In the future we may wish to persist the matchedScience image. 

405 self.measurement.run(diaSources, difference, science, matchedTemplate) 

406 if self.config.doApCorr: 

407 self.applyApCorr.run( 

408 catalog=diaSources, 

409 apCorrMap=difference.getInfo().getApCorrMap() 

410 ) 

411 

412 def measureForcedSources(self, diaSources, science, wcs): 

413 """Perform forced measurement of the diaSources on the science image. 

414 

415 Parameters 

416 ---------- 

417 diaSources : `lsst.afw.table.SourceCatalog` 

418 The catalog of detected sources. 

419 science : `lsst.afw.image.ExposureF` 

420 Science exposure that the template was subtracted from. 

421 wcs : `lsst.afw.geom.SkyWcs` 

422 Coordinate system definition (wcs) for the exposure. 

423 """ 

424 # Run forced psf photometry on the PVI at the diaSource locations. 

425 # Copy the measured flux and error into the diaSource. 

426 forcedSources = self.forcedMeasurement.generateMeasCat( 

427 science, diaSources, wcs) 

428 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

429 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

430 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

431 "ip_diffim_forced_PsfFlux_instFlux", True) 

432 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

433 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

434 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

435 "ip_diffim_forced_PsfFlux_area", True) 

436 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

437 "ip_diffim_forced_PsfFlux_flag", True) 

438 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

439 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

440 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

441 "ip_diffim_forced_PsfFlux_flag_edge", True) 

442 for diaSource, forcedSource in zip(diaSources, forcedSources): 

443 diaSource.assign(forcedSource, mapper) 

444 

445 

446class DetectAndMeasureScoreConnections(DetectAndMeasureConnections): 

447 scoreExposure = pipeBase.connectionTypes.Input( 

448 doc="Maximum likelihood image for detection.", 

449 dimensions=("instrument", "visit", "detector"), 

450 storageClass="ExposureF", 

451 name="{fakesType}{coaddName}Diff_scoreExp", 

452 ) 

453 

454 

455class DetectAndMeasureScoreConfig(DetectAndMeasureConfig, 

456 pipelineConnections=DetectAndMeasureScoreConnections): 

457 pass 

458 

459 

460class DetectAndMeasureScoreTask(DetectAndMeasureTask): 

461 """Detect DIA sources using a score image, 

462 and measure the detections on the difference image. 

463 

464 Source detection is run on the supplied score, or maximum likelihood, 

465 image. Note that no additional convolution will be done in this case. 

466 Close positive and negative detections will optionally be merged into 

467 dipole diaSources. 

468 Sky sources, or forced detections in background regions, will optionally 

469 be added, and the configured measurement algorithm will be run on all 

470 detections. 

471 """ 

472 ConfigClass = DetectAndMeasureScoreConfig 

473 _DefaultName = "detectAndMeasureScore" 

474 

475 @timeMethod 

476 def run(self, science, matchedTemplate, difference, scoreExposure, 

477 idFactory=None): 

478 """Detect and measure sources on a score image. 

479 

480 Parameters 

481 ---------- 

482 science : `lsst.afw.image.ExposureF` 

483 Science exposure that the template was subtracted from. 

484 matchedTemplate : `lsst.afw.image.ExposureF` 

485 Warped and PSF-matched template that was used produce the 

486 difference image. 

487 difference : `lsst.afw.image.ExposureF` 

488 Result of subtracting template from the science image. 

489 scoreExposure : `lsst.afw.image.ExposureF` 

490 Score or maximum likelihood difference image 

491 idFactory : `lsst.afw.table.IdFactory`, optional 

492 Generator object to assign ids to detected sources in the difference image. 

493 

494 Returns 

495 ------- 

496 measurementResults : `lsst.pipe.base.Struct` 

497 

498 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

499 Subtracted exposure with detection mask applied. 

500 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

501 The catalog of detected sources. 

502 """ 

503 # Ensure that we start with an empty detection mask. 

504 mask = scoreExposure.mask 

505 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

506 

507 table = afwTable.SourceTable.make(self.schema, idFactory) 

508 table.setMetadata(self.algMetadata) 

509 results = self.detection.run( 

510 table=table, 

511 exposure=scoreExposure, 

512 doSmooth=False, 

513 ) 

514 # Copy the detection mask from the Score image to the difference image 

515 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox()) 

516 

517 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

518 positiveFootprints=results.positive, negativeFootprints=results.negative)