Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 36%

146 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-18 02:52 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from deprecated.sphinx import deprecated 

23 

24import lsst.afw.table as afwTable 

25import lsst.daf.base as dafBase 

26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig 

28import lsst.meas.extensions.trailedSources # noqa: F401 

29import lsst.meas.extensions.shapeHSM 

30from lsst.obs.base import ExposureIdInfo 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33import lsst.utils 

34from lsst.utils.timer import timeMethod 

35 

36from . import DipoleFitTask 

37 

38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask", 

39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"] 

40 

41 

42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

43 dimensions=("instrument", "visit", "detector"), 

44 defaultTemplates={"coaddName": "deep", 

45 "warpTypeSuffix": "", 

46 "fakesType": ""}): 

47 science = pipeBase.connectionTypes.Input( 

48 doc="Input science exposure.", 

49 dimensions=("instrument", "visit", "detector"), 

50 storageClass="ExposureF", 

51 name="{fakesType}calexp" 

52 ) 

53 matchedTemplate = pipeBase.connectionTypes.Input( 

54 doc="Warped and PSF-matched template used to create the difference image.", 

55 dimensions=("instrument", "visit", "detector"), 

56 storageClass="ExposureF", 

57 name="{fakesType}{coaddName}Diff_matchedExp", 

58 ) 

59 difference = pipeBase.connectionTypes.Input( 

60 doc="Result of subtracting template from science.", 

61 dimensions=("instrument", "visit", "detector"), 

62 storageClass="ExposureF", 

63 name="{fakesType}{coaddName}Diff_differenceTempExp", 

64 ) 

65 outputSchema = pipeBase.connectionTypes.InitOutput( 

66 doc="Schema (as an example catalog) for output DIASource catalog.", 

67 storageClass="SourceCatalog", 

68 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

69 ) 

70 diaSources = pipeBase.connectionTypes.Output( 

71 doc="Detected diaSources on the difference image.", 

72 dimensions=("instrument", "visit", "detector"), 

73 storageClass="SourceCatalog", 

74 name="{fakesType}{coaddName}Diff_diaSrc", 

75 ) 

76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

77 doc="Difference image with detection mask plane filled in.", 

78 dimensions=("instrument", "visit", "detector"), 

79 storageClass="ExposureF", 

80 name="{fakesType}{coaddName}Diff_differenceExp", 

81 ) 

82 

83 

84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

85 pipelineConnections=DetectAndMeasureConnections): 

86 """Config for DetectAndMeasureTask 

87 """ 

88 doMerge = pexConfig.Field( 

89 dtype=bool, 

90 default=True, 

91 doc="Merge positive and negative diaSources with grow radius " 

92 "set by growFootprint" 

93 ) 

94 doForcedMeasurement = pexConfig.Field( 

95 dtype=bool, 

96 default=True, 

97 doc="Force photometer diaSource locations on PVI?") 

98 doAddMetrics = pexConfig.Field( 

99 dtype=bool, 

100 default=False, 

101 doc="Add columns to the source table to hold analysis metrics?" 

102 ) 

103 detection = pexConfig.ConfigurableField( 

104 target=SourceDetectionTask, 

105 doc="Final source detection for diaSource measurement", 

106 ) 

107 measurement = pexConfig.ConfigurableField( 

108 target=DipoleFitTask, 

109 doc="Task to measure sources on the difference image.", 

110 ) 

111 doApCorr = lsst.pex.config.Field( 

112 dtype=bool, 

113 default=True, 

114 doc="Run subtask to apply aperture corrections" 

115 ) 

116 applyApCorr = lsst.pex.config.ConfigurableField( 

117 target=ApplyApCorrTask, 

118 doc="Task to apply aperture corrections" 

119 ) 

120 forcedMeasurement = pexConfig.ConfigurableField( 

121 target=ForcedMeasurementTask, 

122 doc="Task to force photometer science image at diaSource locations.", 

123 ) 

124 growFootprint = pexConfig.Field( 

125 dtype=int, 

126 default=2, 

127 doc="Grow positive and negative footprints by this many pixels before merging" 

128 ) 

129 diaSourceMatchRadius = pexConfig.Field( 

130 dtype=float, 

131 default=0.5, 

132 doc="Match radius (in arcseconds) for DiaSource to Source association" 

133 ) 

134 doSkySources = pexConfig.Field( 

135 dtype=bool, 

136 default=False, 

137 doc="Generate sky sources?", 

138 ) 

139 skySources = pexConfig.ConfigurableField( 

140 target=SkyObjectsTask, 

141 doc="Generate sky sources", 

142 ) 

143 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

144 

145 def setDefaults(self): 

146 # DiaSource Detection 

147 self.detection.thresholdPolarity = "both" 

148 self.detection.thresholdValue = 5.0 

149 self.detection.reEstimateBackground = False 

150 self.detection.thresholdType = "pixel_stdev" 

151 

152 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

153 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

154 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

155 'base_LocalPhotoCalib', 

156 'base_LocalWcs', 

157 'ext_shapeHSM_HsmSourceMoments', 

158 'ext_shapeHSM_HsmPsfMoments', 

159 ] 

160 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

161 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

162 

163 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

164 self.forcedMeasurement.copyColumns = { 

165 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

166 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

167 self.forcedMeasurement.slots.shape = None 

168 

169 

170class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

171 """Detect and measure sources on a difference image. 

172 """ 

173 ConfigClass = DetectAndMeasureConfig 

174 _DefaultName = "detectAndMeasure" 

175 

176 def __init__(self, **kwargs): 

177 super().__init__(**kwargs) 

178 self.schema = afwTable.SourceTable.makeMinimalSchema() 

179 

180 self.algMetadata = dafBase.PropertyList() 

181 self.makeSubtask("detection", schema=self.schema) 

182 self.makeSubtask("measurement", schema=self.schema, 

183 algMetadata=self.algMetadata) 

184 if self.config.doApCorr: 

185 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

186 if self.config.doForcedMeasurement: 

187 self.schema.addField( 

188 "ip_diffim_forced_PsfFlux_instFlux", "D", 

189 "Forced PSF flux measured on the direct image.", 

190 units="count") 

191 self.schema.addField( 

192 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

193 "Forced PSF flux error measured on the direct image.", 

194 units="count") 

195 self.schema.addField( 

196 "ip_diffim_forced_PsfFlux_area", "F", 

197 "Forced PSF flux effective area of PSF.", 

198 units="pixel") 

199 self.schema.addField( 

200 "ip_diffim_forced_PsfFlux_flag", "Flag", 

201 "Forced PSF flux general failure flag.") 

202 self.schema.addField( 

203 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

204 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

205 self.schema.addField( 

206 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

207 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

208 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

209 

210 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

211 self.schema.addField("srcMatchId", "L", "unique id of source match") 

212 if self.config.doSkySources: 

213 self.makeSubtask("skySources") 

214 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

215 

216 # initialize InitOutputs 

217 self.outputSchema = afwTable.SourceCatalog(self.schema) 

218 self.outputSchema.getTable().setMetadata(self.algMetadata) 

219 

220 @staticmethod 

221 @deprecated( 

222 reason=( 

223 "ID factory construction now depends on configuration; use the " 

224 "idGenerator config field. Will be removed after v27." 

225 ), 

226 version="v26.0", 

227 category=FutureWarning, 

228 ) 

229 def makeIdFactory(expId, expBits): 

230 """Create IdFactory instance for unique 64 bit diaSource id-s. 

231 

232 Parameters 

233 ---------- 

234 expId : `int` 

235 Exposure id. 

236 

237 expBits: `int` 

238 Number of used bits in ``expId``. 

239 

240 Notes 

241 ----- 

242 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

243 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

244 low value end of the integer. 

245 

246 Returns 

247 ------- 

248 idFactory: `lsst.afw.table.IdFactory` 

249 """ 

250 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

251 

252 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

253 inputRefs: pipeBase.InputQuantizedConnection, 

254 outputRefs: pipeBase.OutputQuantizedConnection): 

255 inputs = butlerQC.get(inputRefs) 

256 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

257 idFactory = idGenerator.make_table_id_factory() 

258 outputs = self.run(**inputs, idFactory=idFactory) 

259 butlerQC.put(outputs, outputRefs) 

260 

261 @timeMethod 

262 def run(self, science, matchedTemplate, difference, 

263 idFactory=None): 

264 """Detect and measure sources on a difference image. 

265 

266 The difference image will be convolved with a gaussian approximation of 

267 the PSF to form a maximum likelihood image for detection. 

268 Close positive and negative detections will optionally be merged into 

269 dipole diaSources. 

270 Sky sources, or forced detections in background regions, will optionally 

271 be added, and the configured measurement algorithm will be run on all 

272 detections. 

273 

274 Parameters 

275 ---------- 

276 science : `lsst.afw.image.ExposureF` 

277 Science exposure that the template was subtracted from. 

278 matchedTemplate : `lsst.afw.image.ExposureF` 

279 Warped and PSF-matched template that was used produce the 

280 difference image. 

281 difference : `lsst.afw.image.ExposureF` 

282 Result of subtracting template from the science image. 

283 idFactory : `lsst.afw.table.IdFactory`, optional 

284 Generator object to assign ids to detected sources in the difference image. 

285 

286 Returns 

287 ------- 

288 measurementResults : `lsst.pipe.base.Struct` 

289 

290 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

291 Subtracted exposure with detection mask applied. 

292 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

293 The catalog of detected sources. 

294 """ 

295 # Ensure that we start with an empty detection mask. 

296 mask = difference.mask 

297 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

298 

299 table = afwTable.SourceTable.make(self.schema, idFactory) 

300 table.setMetadata(self.algMetadata) 

301 results = self.detection.run( 

302 table=table, 

303 exposure=difference, 

304 doSmooth=True, 

305 ) 

306 

307 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

308 positiveFootprints=results.positive, negativeFootprints=results.negative) 

309 

310 def processResults(self, science, matchedTemplate, difference, sources, table, 

311 positiveFootprints=None, negativeFootprints=None,): 

312 """Measure and process the results of source detection. 

313 

314 Parameters 

315 ---------- 

316 sources : `lsst.afw.table.SourceCatalog` 

317 Detected sources on the difference exposure. 

318 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional 

319 Positive polarity footprints. 

320 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional 

321 Negative polarity footprints. 

322 table : `lsst.afw.table.SourceTable` 

323 Table object that will be used to create the SourceCatalog. 

324 science : `lsst.afw.image.ExposureF` 

325 Science exposure that the template was subtracted from. 

326 matchedTemplate : `lsst.afw.image.ExposureF` 

327 Warped and PSF-matched template that was used produce the 

328 difference image. 

329 difference : `lsst.afw.image.ExposureF` 

330 Result of subtracting template from the science image. 

331 

332 Returns 

333 ------- 

334 measurementResults : `lsst.pipe.base.Struct` 

335 

336 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

337 Subtracted exposure with detection mask applied. 

338 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

339 The catalog of detected sources. 

340 """ 

341 if self.config.doMerge: 

342 fpSet = positiveFootprints 

343 fpSet.merge(negativeFootprints, self.config.growFootprint, 

344 self.config.growFootprint, False) 

345 diaSources = afwTable.SourceCatalog(table) 

346 fpSet.makeSources(diaSources) 

347 self.log.info("Merging detections into %d sources", len(diaSources)) 

348 else: 

349 diaSources = sources 

350 

351 if self.config.doSkySources: 

352 self.addSkySources(diaSources, difference.mask, difference.info.id) 

353 

354 self.measureDiaSources(diaSources, science, difference, matchedTemplate) 

355 

356 if self.config.doForcedMeasurement: 

357 self.measureForcedSources(diaSources, science, difference.getWcs()) 

358 

359 measurementResults = pipeBase.Struct( 

360 subtractedMeasuredExposure=difference, 

361 diaSources=diaSources, 

362 ) 

363 

364 return measurementResults 

365 

366 def addSkySources(self, diaSources, mask, seed): 

367 """Add sources in empty regions of the difference image 

368 for measuring the background. 

369 

370 Parameters 

371 ---------- 

372 diaSources : `lsst.afw.table.SourceCatalog` 

373 The catalog of detected sources. 

374 mask : `lsst.afw.image.Mask` 

375 Mask plane for determining regions where Sky sources can be added. 

376 seed : `int` 

377 Seed value to initialize the random number generator. 

378 """ 

379 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

380 if skySourceFootprints: 

381 for foot in skySourceFootprints: 

382 s = diaSources.addNew() 

383 s.setFootprint(foot) 

384 s.set(self.skySourceKey, True) 

385 

386 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

387 """Use (matched) template and science image to constrain dipole fitting. 

388 

389 Parameters 

390 ---------- 

391 diaSources : `lsst.afw.table.SourceCatalog` 

392 The catalog of detected sources. 

393 science : `lsst.afw.image.ExposureF` 

394 Science exposure that the template was subtracted from. 

395 difference : `lsst.afw.image.ExposureF` 

396 Result of subtracting template from the science image. 

397 matchedTemplate : `lsst.afw.image.ExposureF` 

398 Warped and PSF-matched template that was used produce the 

399 difference image. 

400 """ 

401 # Note that this may not be correct if we convolved the science image. 

402 # In the future we may wish to persist the matchedScience image. 

403 self.measurement.run(diaSources, difference, science, matchedTemplate) 

404 if self.config.doApCorr: 

405 self.applyApCorr.run( 

406 catalog=diaSources, 

407 apCorrMap=difference.getInfo().getApCorrMap() 

408 ) 

409 

410 def measureForcedSources(self, diaSources, science, wcs): 

411 """Perform forced measurement of the diaSources on the science image. 

412 

413 Parameters 

414 ---------- 

415 diaSources : `lsst.afw.table.SourceCatalog` 

416 The catalog of detected sources. 

417 science : `lsst.afw.image.ExposureF` 

418 Science exposure that the template was subtracted from. 

419 wcs : `lsst.afw.geom.SkyWcs` 

420 Coordinate system definition (wcs) for the exposure. 

421 """ 

422 # Run forced psf photometry on the PVI at the diaSource locations. 

423 # Copy the measured flux and error into the diaSource. 

424 forcedSources = self.forcedMeasurement.generateMeasCat( 

425 science, diaSources, wcs) 

426 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

427 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

428 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

429 "ip_diffim_forced_PsfFlux_instFlux", True) 

430 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

431 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

432 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

433 "ip_diffim_forced_PsfFlux_area", True) 

434 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

435 "ip_diffim_forced_PsfFlux_flag", True) 

436 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

437 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

438 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

439 "ip_diffim_forced_PsfFlux_flag_edge", True) 

440 for diaSource, forcedSource in zip(diaSources, forcedSources): 

441 diaSource.assign(forcedSource, mapper) 

442 

443 

444class DetectAndMeasureScoreConnections(DetectAndMeasureConnections): 

445 scoreExposure = pipeBase.connectionTypes.Input( 

446 doc="Maximum likelihood image for detection.", 

447 dimensions=("instrument", "visit", "detector"), 

448 storageClass="ExposureF", 

449 name="{fakesType}{coaddName}Diff_scoreExp", 

450 ) 

451 

452 

453class DetectAndMeasureScoreConfig(DetectAndMeasureConfig, 

454 pipelineConnections=DetectAndMeasureScoreConnections): 

455 pass 

456 

457 

458class DetectAndMeasureScoreTask(DetectAndMeasureTask): 

459 """Detect DIA sources using a score image, 

460 and measure the detections on the difference image. 

461 

462 Source detection is run on the supplied score, or maximum likelihood, 

463 image. Note that no additional convolution will be done in this case. 

464 Close positive and negative detections will optionally be merged into 

465 dipole diaSources. 

466 Sky sources, or forced detections in background regions, will optionally 

467 be added, and the configured measurement algorithm will be run on all 

468 detections. 

469 """ 

470 ConfigClass = DetectAndMeasureScoreConfig 

471 _DefaultName = "detectAndMeasureScore" 

472 

473 @timeMethod 

474 def run(self, science, matchedTemplate, difference, scoreExposure, 

475 idFactory=None): 

476 """Detect and measure sources on a score image. 

477 

478 Parameters 

479 ---------- 

480 science : `lsst.afw.image.ExposureF` 

481 Science exposure that the template was subtracted from. 

482 matchedTemplate : `lsst.afw.image.ExposureF` 

483 Warped and PSF-matched template that was used produce the 

484 difference image. 

485 difference : `lsst.afw.image.ExposureF` 

486 Result of subtracting template from the science image. 

487 scoreExposure : `lsst.afw.image.ExposureF` 

488 Score or maximum likelihood difference image 

489 idFactory : `lsst.afw.table.IdFactory`, optional 

490 Generator object to assign ids to detected sources in the difference image. 

491 

492 Returns 

493 ------- 

494 measurementResults : `lsst.pipe.base.Struct` 

495 

496 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

497 Subtracted exposure with detection mask applied. 

498 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

499 The catalog of detected sources. 

500 """ 

501 # Ensure that we start with an empty detection mask. 

502 mask = scoreExposure.mask 

503 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

504 

505 table = afwTable.SourceTable.make(self.schema, idFactory) 

506 table.setMetadata(self.algMetadata) 

507 results = self.detection.run( 

508 table=table, 

509 exposure=scoreExposure, 

510 doSmooth=False, 

511 ) 

512 # Copy the detection mask from the Score image to the difference image 

513 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox()) 

514 

515 return self.processResults(science, matchedTemplate, difference, results.sources, table, 

516 positiveFootprints=results.positive, negativeFootprints=results.negative)