Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 21%

303 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-11 03:14 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23 

24import lsst.afw.detection as afwDetection 

25import lsst.afw.table as afwTable 

26import lsst.daf.base as dafBase 

27import lsst.geom 

28from lsst.ip.diffim.utils import getPsfFwhm, angleMean 

29from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask, SetPrimaryFlagsTask 

30from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig 

31import lsst.meas.deblender 

32import lsst.meas.extensions.trailedSources # noqa: F401 

33import lsst.meas.extensions.shapeHSM 

34import lsst.pex.config as pexConfig 

35from lsst.pex.exceptions import InvalidParameterError 

36import lsst.pipe.base as pipeBase 

37import lsst.utils 

38from lsst.utils.timer import timeMethod 

39 

40from . import DipoleFitTask 

41 

42__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask", 

43 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"] 

44 

45 

46class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("instrument", "visit", "detector"), 

48 defaultTemplates={"coaddName": "deep", 

49 "warpTypeSuffix": "", 

50 "fakesType": ""}): 

51 science = pipeBase.connectionTypes.Input( 

52 doc="Input science exposure.", 

53 dimensions=("instrument", "visit", "detector"), 

54 storageClass="ExposureF", 

55 name="{fakesType}calexp" 

56 ) 

57 matchedTemplate = pipeBase.connectionTypes.Input( 

58 doc="Warped and PSF-matched template used to create the difference image.", 

59 dimensions=("instrument", "visit", "detector"), 

60 storageClass="ExposureF", 

61 name="{fakesType}{coaddName}Diff_matchedExp", 

62 ) 

63 difference = pipeBase.connectionTypes.Input( 

64 doc="Result of subtracting template from science.", 

65 dimensions=("instrument", "visit", "detector"), 

66 storageClass="ExposureF", 

67 name="{fakesType}{coaddName}Diff_differenceTempExp", 

68 ) 

69 outputSchema = pipeBase.connectionTypes.InitOutput( 

70 doc="Schema (as an example catalog) for output DIASource catalog.", 

71 storageClass="SourceCatalog", 

72 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

73 ) 

74 diaSources = pipeBase.connectionTypes.Output( 

75 doc="Detected diaSources on the difference image.", 

76 dimensions=("instrument", "visit", "detector"), 

77 storageClass="SourceCatalog", 

78 name="{fakesType}{coaddName}Diff_diaSrc", 

79 ) 

80 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

81 doc="Difference image with detection mask plane filled in.", 

82 dimensions=("instrument", "visit", "detector"), 

83 storageClass="ExposureF", 

84 name="{fakesType}{coaddName}Diff_differenceExp", 

85 ) 

86 spatiallySampledMetrics = pipeBase.connectionTypes.Output( 

87 doc="Summary metrics computed at randomized locations.", 

88 dimensions=("instrument", "visit", "detector"), 

89 storageClass="ArrowAstropy", 

90 name="{fakesType}{coaddName}Diff_spatiallySampledMetrics", 

91 ) 

92 

93 def __init__(self, *, config=None): 

94 super().__init__(config=config) 

95 if not config.doWriteMetrics: 

96 self.outputs.remove("spatiallySampledMetrics") 

97 

98 

99class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

100 pipelineConnections=DetectAndMeasureConnections): 

101 """Config for DetectAndMeasureTask 

102 """ 

103 doMerge = pexConfig.Field( 

104 dtype=bool, 

105 default=True, 

106 doc="Merge positive and negative diaSources with grow radius " 

107 "set by growFootprint" 

108 ) 

109 doForcedMeasurement = pexConfig.Field( 

110 dtype=bool, 

111 default=True, 

112 doc="Force photometer diaSource locations on PVI?") 

113 doAddMetrics = pexConfig.Field( 

114 dtype=bool, 

115 default=False, 

116 doc="Add columns to the source table to hold analysis metrics?" 

117 ) 

118 detection = pexConfig.ConfigurableField( 

119 target=SourceDetectionTask, 

120 doc="Final source detection for diaSource measurement", 

121 ) 

122 deblend = pexConfig.ConfigurableField( 

123 target=lsst.meas.deblender.SourceDeblendTask, 

124 doc="Task to split blended sources into their components." 

125 ) 

126 measurement = pexConfig.ConfigurableField( 

127 target=DipoleFitTask, 

128 doc="Task to measure sources on the difference image.", 

129 ) 

130 doApCorr = lsst.pex.config.Field( 

131 dtype=bool, 

132 default=True, 

133 doc="Run subtask to apply aperture corrections" 

134 ) 

135 applyApCorr = lsst.pex.config.ConfigurableField( 

136 target=ApplyApCorrTask, 

137 doc="Task to apply aperture corrections" 

138 ) 

139 forcedMeasurement = pexConfig.ConfigurableField( 

140 target=ForcedMeasurementTask, 

141 doc="Task to force photometer science image at diaSource locations.", 

142 ) 

143 growFootprint = pexConfig.Field( 

144 dtype=int, 

145 default=2, 

146 doc="Grow positive and negative footprints by this many pixels before merging" 

147 ) 

148 diaSourceMatchRadius = pexConfig.Field( 

149 dtype=float, 

150 default=0.5, 

151 doc="Match radius (in arcseconds) for DiaSource to Source association" 

152 ) 

153 doSkySources = pexConfig.Field( 

154 dtype=bool, 

155 default=False, 

156 doc="Generate sky sources?", 

157 ) 

158 skySources = pexConfig.ConfigurableField( 

159 target=SkyObjectsTask, 

160 doc="Generate sky sources", 

161 ) 

162 setPrimaryFlags = pexConfig.ConfigurableField( 

163 target=SetPrimaryFlagsTask, 

164 doc="Task to add isPrimary and deblending-related flags to the catalog." 

165 ) 

166 badSourceFlags = lsst.pex.config.ListField( 

167 dtype=str, 

168 doc="Sources with any of these flags set are removed before writing the output catalog.", 

169 default=("base_PixelFlags_flag_offimage", 

170 "base_PixelFlags_flag_interpolatedCenterAll", 

171 "base_PixelFlags_flag_badCenterAll", 

172 "base_PixelFlags_flag_edgeCenterAll", 

173 "base_PixelFlags_flag_saturatedCenterAll", 

174 ), 

175 ) 

176 metricsMaskPlanes = lsst.pex.config.ListField( 

177 dtype=str, 

178 doc="List of mask planes to include in metrics", 

179 default=('BAD', 'CLIPPED', 'CR', 'DETECTED', 'DETECTED_NEGATIVE', 'EDGE', 

180 'INEXACT_PSF', 'INJECTED', 'INJECTED_TEMPLATE', 'INTRP', 'NOT_DEBLENDED', 

181 'NO_DATA', 'REJECTED', 'SAT', 'SAT_TEMPLATE', 'SENSOR_EDGE', 'STREAK', 'SUSPECT', 

182 'UNMASKEDNAN', 

183 ), 

184 ) 

185 metricSources = pexConfig.ConfigurableField( 

186 target=SkyObjectsTask, 

187 doc="Generate QA metric sources", 

188 ) 

189 doWriteMetrics = lsst.pex.config.Field( 

190 dtype=bool, 

191 default=True, 

192 doc="Compute and write summary metrics." 

193 ) 

194 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

195 

196 def setDefaults(self): 

197 # DiaSource Detection 

198 self.detection.thresholdPolarity = "both" 

199 self.detection.thresholdValue = 5.0 

200 self.detection.reEstimateBackground = False 

201 self.detection.thresholdType = "pixel_stdev" 

202 self.detection.excludeMaskPlanes = ["EDGE"] 

203 

204 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

205 self.measurement.algorithms.names.add("base_PeakLikelihoodFlux") 

206 self.measurement.plugins.names |= ["ext_trailedSources_Naive", 

207 "base_LocalPhotoCalib", 

208 "base_LocalWcs", 

209 "ext_shapeHSM_HsmSourceMoments", 

210 "ext_shapeHSM_HsmPsfMoments", 

211 ] 

212 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

213 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

214 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0 

215 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0 

216 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

217 self.forcedMeasurement.copyColumns = { 

218 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

219 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

220 self.forcedMeasurement.slots.shape = None 

221 

222 # Keep track of which footprints contain streaks 

223 self.measurement.plugins["base_PixelFlags"].masksFpAnywhere = [ 

224 "STREAK", "INJECTED", "INJECTED_TEMPLATE"] 

225 self.measurement.plugins["base_PixelFlags"].masksFpCenter = [ 

226 "STREAK", "INJECTED", "INJECTED_TEMPLATE"] 

227 self.skySources.avoidMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "NO_DATA", "EDGE"] 

228 self.metricSources.avoidMask = ["NO_DATA", "EDGE"] 

229 

230 

231class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

232 """Detect and measure sources on a difference image. 

233 """ 

234 ConfigClass = DetectAndMeasureConfig 

235 _DefaultName = "detectAndMeasure" 

236 

237 def __init__(self, **kwargs): 

238 super().__init__(**kwargs) 

239 self.schema = afwTable.SourceTable.makeMinimalSchema() 

240 # Add coordinate error fields: 

241 afwTable.CoordKey.addErrorFields(self.schema) 

242 

243 self.algMetadata = dafBase.PropertyList() 

244 self.makeSubtask("detection", schema=self.schema) 

245 self.makeSubtask("deblend", schema=self.schema) 

246 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

247 self.makeSubtask("measurement", schema=self.schema, 

248 algMetadata=self.algMetadata) 

249 if self.config.doApCorr: 

250 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

251 if self.config.doForcedMeasurement: 

252 self.schema.addField( 

253 "ip_diffim_forced_PsfFlux_instFlux", "D", 

254 "Forced PSF flux measured on the direct image.", 

255 units="count") 

256 self.schema.addField( 

257 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

258 "Forced PSF flux error measured on the direct image.", 

259 units="count") 

260 self.schema.addField( 

261 "ip_diffim_forced_PsfFlux_area", "F", 

262 "Forced PSF flux effective area of PSF.", 

263 units="pixel") 

264 self.schema.addField( 

265 "ip_diffim_forced_PsfFlux_flag", "Flag", 

266 "Forced PSF flux general failure flag.") 

267 self.schema.addField( 

268 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

269 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

270 self.schema.addField( 

271 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

272 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

273 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

274 

275 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

276 self.schema.addField("srcMatchId", "L", "unique id of source match") 

277 if self.config.doSkySources: 

278 self.makeSubtask("skySources", schema=self.schema) 

279 

280 # Check that the schema and config are consistent 

281 for flag in self.config.badSourceFlags: 

282 if flag not in self.schema: 

283 raise pipeBase.InvalidQuantumError("Field %s not in schema" % flag) 

284 

285 if self.config.doWriteMetrics: 

286 self.makeSubtask("metricSources") 

287 self.metricSchema = afwTable.SourceTable.makeMinimalSchema() 

288 self.metricSchema.addField( 

289 "x", "F", 

290 "X location of the metric evaluation.", 

291 units="pixel") 

292 self.metricSchema.addField( 

293 "y", "F", 

294 "Y location of the metric evaluation.", 

295 units="pixel") 

296 self.metricSources.skySourceKey = self.metricSchema.addField("sky_source", type="Flag", 

297 doc="Metric evaluation objects.") 

298 self.metricSchema.addField( 

299 "source_density", "F", 

300 "Density of diaSources at location.", 

301 units="count/degree^2") 

302 self.metricSchema.addField( 

303 "dipole_density", "F", 

304 "Density of dipoles at location.", 

305 units="count/degree^2") 

306 self.metricSchema.addField( 

307 "dipole_direction", "F", 

308 "Mean dipole orientation.", 

309 units="radian") 

310 self.metricSchema.addField( 

311 "dipole_separation", "F", 

312 "Mean dipole separation.", 

313 units="pixel") 

314 self.metricSchema.addField( 

315 "template_value", "F", 

316 "Median of template at location.", 

317 units="nJy") 

318 self.metricSchema.addField( 

319 "science_value", "F", 

320 "Median of science at location.", 

321 units="nJy") 

322 self.metricSchema.addField( 

323 "diffim_value", "F", 

324 "Median of diffim at location.", 

325 units="nJy") 

326 self.metricSchema.addField( 

327 "science_psfSize", "F", 

328 "Width of the science image PSF at location.", 

329 units="pixel") 

330 self.metricSchema.addField( 

331 "template_psfSize", "F", 

332 "Width of the template image PSF at location.", 

333 units="pixel") 

334 for maskPlane in self.config.metricsMaskPlanes: 

335 self.metricSchema.addField( 

336 "%s_mask_fraction"%maskPlane.lower(), "F", 

337 "Fraction of pixels with %s mask"%maskPlane 

338 ) 

339 

340 # initialize InitOutputs 

341 self.outputSchema = afwTable.SourceCatalog(self.schema) 

342 self.outputSchema.getTable().setMetadata(self.algMetadata) 

343 

344 def runQuantum(self, butlerQC: pipeBase.QuantumContext, 

345 inputRefs: pipeBase.InputQuantizedConnection, 

346 outputRefs: pipeBase.OutputQuantizedConnection): 

347 inputs = butlerQC.get(inputRefs) 

348 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

349 idFactory = idGenerator.make_table_id_factory() 

350 outputs = self.run(**inputs, idFactory=idFactory) 

351 butlerQC.put(outputs, outputRefs) 

352 

353 @timeMethod 

354 def run(self, science, matchedTemplate, difference, 

355 idFactory=None): 

356 """Detect and measure sources on a difference image. 

357 

358 The difference image will be convolved with a gaussian approximation of 

359 the PSF to form a maximum likelihood image for detection. 

360 Close positive and negative detections will optionally be merged into 

361 dipole diaSources. 

362 Sky sources, or forced detections in background regions, will optionally 

363 be added, and the configured measurement algorithm will be run on all 

364 detections. 

365 

366 Parameters 

367 ---------- 

368 science : `lsst.afw.image.ExposureF` 

369 Science exposure that the template was subtracted from. 

370 matchedTemplate : `lsst.afw.image.ExposureF` 

371 Warped and PSF-matched template that was used produce the 

372 difference image. 

373 difference : `lsst.afw.image.ExposureF` 

374 Result of subtracting template from the science image. 

375 idFactory : `lsst.afw.table.IdFactory`, optional 

376 Generator object used to assign ids to detected sources in the 

377 difference image. Ids from this generator are not set until after 

378 deblending and merging positive/negative peaks. 

379 

380 Returns 

381 ------- 

382 measurementResults : `lsst.pipe.base.Struct` 

383 

384 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

385 Subtracted exposure with detection mask applied. 

386 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

387 The catalog of detected sources. 

388 """ 

389 if idFactory is None: 

390 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory() 

391 

392 # Ensure that we start with an empty detection mask. 

393 mask = difference.mask 

394 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

395 

396 # Don't use the idFactory until after deblend+merge, so that we aren't 

397 # generating ids that just get thrown away (footprint merge doesn't 

398 # know about past ids). 

399 table = afwTable.SourceTable.make(self.schema) 

400 results = self.detection.run( 

401 table=table, 

402 exposure=difference, 

403 doSmooth=True, 

404 ) 

405 

406 sources, positives, negatives = self._deblend(difference, 

407 results.positive, 

408 results.negative) 

409 

410 return self.processResults(science, matchedTemplate, difference, sources, idFactory, 

411 positiveFootprints=positives, 

412 negativeFootprints=negatives) 

413 

414 def processResults(self, science, matchedTemplate, difference, sources, idFactory, 

415 positiveFootprints=None, negativeFootprints=None,): 

416 """Measure and process the results of source detection. 

417 

418 Parameters 

419 ---------- 

420 science : `lsst.afw.image.ExposureF` 

421 Science exposure that the template was subtracted from. 

422 matchedTemplate : `lsst.afw.image.ExposureF` 

423 Warped and PSF-matched template that was used produce the 

424 difference image. 

425 difference : `lsst.afw.image.ExposureF` 

426 Result of subtracting template from the science image. 

427 sources : `lsst.afw.table.SourceCatalog` 

428 Detected sources on the difference exposure. 

429 idFactory : `lsst.afw.table.IdFactory` 

430 Generator object used to assign ids to detected sources in the 

431 difference image. 

432 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional 

433 Positive polarity footprints. 

434 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional 

435 Negative polarity footprints. 

436 

437 Returns 

438 ------- 

439 measurementResults : `lsst.pipe.base.Struct` 

440 

441 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

442 Subtracted exposure with detection mask applied. 

443 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

444 The catalog of detected sources. 

445 """ 

446 self.metadata.add("nUnmergedDiaSources", len(sources)) 

447 if self.config.doMerge: 

448 fpSet = positiveFootprints 

449 fpSet.merge(negativeFootprints, self.config.growFootprint, 

450 self.config.growFootprint, False) 

451 initialDiaSources = afwTable.SourceCatalog(self.schema) 

452 fpSet.makeSources(initialDiaSources) 

453 self.log.info("Merging detections into %d sources", len(initialDiaSources)) 

454 else: 

455 initialDiaSources = sources 

456 

457 # Assign source ids at the end: deblend/merge mean that we don't keep 

458 # track of parents and children, we only care about the final ids. 

459 for source in initialDiaSources: 

460 source.setId(idFactory()) 

461 # Ensure sources added after this get correct ids. 

462 initialDiaSources.getTable().setIdFactory(idFactory) 

463 initialDiaSources.setMetadata(self.algMetadata) 

464 

465 self.metadata.add("nMergedDiaSources", len(initialDiaSources)) 

466 

467 if self.config.doSkySources: 

468 self.addSkySources(initialDiaSources, difference.mask, difference.info.id) 

469 

470 if not initialDiaSources.isContiguous(): 

471 initialDiaSources = initialDiaSources.copy(deep=True) 

472 

473 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate) 

474 diaSources = self._removeBadSources(initialDiaSources) 

475 

476 if self.config.doForcedMeasurement: 

477 self.measureForcedSources(diaSources, science, difference.getWcs()) 

478 

479 spatiallySampledMetrics = self.calculateMetrics(difference, diaSources, science, matchedTemplate, 

480 idFactory) 

481 

482 measurementResults = pipeBase.Struct( 

483 subtractedMeasuredExposure=difference, 

484 diaSources=diaSources, 

485 spatiallySampledMetrics=spatiallySampledMetrics, 

486 ) 

487 

488 return measurementResults 

489 

490 def _deblend(self, difference, positiveFootprints, negativeFootprints): 

491 """Deblend the positive and negative footprints and return a catalog 

492 containing just the children, and the deblended footprints. 

493 

494 Parameters 

495 ---------- 

496 difference : `lsst.afw.image.Exposure` 

497 Result of subtracting template from the science image. 

498 positiveFootprints, negativeFootprints : `lsst.afw.detection.FootprintSet` 

499 Positive and negative polarity footprints measured on 

500 ``difference`` to be deblended separately. 

501 

502 Returns 

503 ------- 

504 sources : `lsst.afw.table.SourceCatalog` 

505 Positive and negative deblended children. 

506 positives, negatives : `lsst.afw.detection.FootprintSet` 

507 Deblended positive and negative polarity footprints measured on 

508 ``difference``. 

509 """ 

510 def makeFootprints(sources): 

511 footprints = afwDetection.FootprintSet(difference.getBBox()) 

512 footprints.setFootprints([src.getFootprint() for src in sources]) 

513 return footprints 

514 

515 def deblend(footprints): 

516 """Deblend a positive or negative footprint set, 

517 and return the deblended children. 

518 """ 

519 sources = afwTable.SourceCatalog(self.schema) 

520 footprints.makeSources(sources) 

521 self.deblend.run(exposure=difference, sources=sources) 

522 self.setPrimaryFlags.run(sources) 

523 children = sources["detect_isDeblendedSource"] == 1 

524 sources = sources[children].copy(deep=True) 

525 # Clear parents, so that measurement plugins behave correctly. 

526 sources['parent'] = 0 

527 return sources.copy(deep=True) 

528 

529 positives = deblend(positiveFootprints) 

530 negatives = deblend(negativeFootprints) 

531 

532 sources = afwTable.SourceCatalog(self.schema) 

533 sources.reserve(len(positives) + len(negatives)) 

534 sources.extend(positives, deep=True) 

535 sources.extend(negatives, deep=True) 

536 return sources, makeFootprints(positives), makeFootprints(negatives) 

537 

538 def _removeBadSources(self, diaSources): 

539 """Remove unphysical diaSources from the catalog. 

540 

541 Parameters 

542 ---------- 

543 diaSources : `lsst.afw.table.SourceCatalog` 

544 The catalog of detected sources. 

545 

546 Returns 

547 ------- 

548 diaSources : `lsst.afw.table.SourceCatalog` 

549 The updated catalog of detected sources, with any source that has a 

550 flag in ``config.badSourceFlags`` set removed. 

551 """ 

552 selector = np.ones(len(diaSources), dtype=bool) 

553 for flag in self.config.badSourceFlags: 

554 flags = diaSources[flag] 

555 nBad = np.count_nonzero(flags) 

556 if nBad > 0: 

557 self.log.debug("Found %d unphysical sources with flag %s.", nBad, flag) 

558 selector &= ~flags 

559 nBadTotal = np.count_nonzero(~selector) 

560 self.metadata.add("nRemovedBadFlaggedSources", nBadTotal) 

561 self.log.info("Removed %d unphysical sources.", nBadTotal) 

562 return diaSources[selector].copy(deep=True) 

563 

564 def addSkySources(self, diaSources, mask, seed, 

565 subtask=None): 

566 """Add sources in empty regions of the difference image 

567 for measuring the background. 

568 

569 Parameters 

570 ---------- 

571 diaSources : `lsst.afw.table.SourceCatalog` 

572 The catalog of detected sources. 

573 mask : `lsst.afw.image.Mask` 

574 Mask plane for determining regions where Sky sources can be added. 

575 seed : `int` 

576 Seed value to initialize the random number generator. 

577 """ 

578 if subtask is None: 

579 subtask = self.skySources 

580 skySourceFootprints = subtask.run(mask=mask, seed=seed, catalog=diaSources) 

581 self.metadata.add(f"n_{subtask.getName()}", len(skySourceFootprints)) 

582 

583 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

584 """Use (matched) template and science image to constrain dipole fitting. 

585 

586 Parameters 

587 ---------- 

588 diaSources : `lsst.afw.table.SourceCatalog` 

589 The catalog of detected sources. 

590 science : `lsst.afw.image.ExposureF` 

591 Science exposure that the template was subtracted from. 

592 difference : `lsst.afw.image.ExposureF` 

593 Result of subtracting template from the science image. 

594 matchedTemplate : `lsst.afw.image.ExposureF` 

595 Warped and PSF-matched template that was used produce the 

596 difference image. 

597 """ 

598 # Ensure that the required mask planes are present 

599 for mp in self.config.measurement.plugins["base_PixelFlags"].masksFpAnywhere: 

600 difference.mask.addMaskPlane(mp) 

601 # Note that this may not be correct if we convolved the science image. 

602 # In the future we may wish to persist the matchedScience image. 

603 self.measurement.run(diaSources, difference, science, matchedTemplate) 

604 if self.config.doApCorr: 

605 apCorrMap = difference.getInfo().getApCorrMap() 

606 if apCorrMap is None: 

607 self.log.warning("Difference image does not have valid aperture correction; skipping.") 

608 else: 

609 self.applyApCorr.run( 

610 catalog=diaSources, 

611 apCorrMap=apCorrMap, 

612 ) 

613 

614 def measureForcedSources(self, diaSources, science, wcs): 

615 """Perform forced measurement of the diaSources on the science image. 

616 

617 Parameters 

618 ---------- 

619 diaSources : `lsst.afw.table.SourceCatalog` 

620 The catalog of detected sources. 

621 science : `lsst.afw.image.ExposureF` 

622 Science exposure that the template was subtracted from. 

623 wcs : `lsst.afw.geom.SkyWcs` 

624 Coordinate system definition (wcs) for the exposure. 

625 """ 

626 # Run forced psf photometry on the PVI at the diaSource locations. 

627 # Copy the measured flux and error into the diaSource. 

628 forcedSources = self.forcedMeasurement.generateMeasCat(science, diaSources, wcs) 

629 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

630 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

631 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

632 "ip_diffim_forced_PsfFlux_instFlux", True) 

633 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

634 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

635 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

636 "ip_diffim_forced_PsfFlux_area", True) 

637 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

638 "ip_diffim_forced_PsfFlux_flag", True) 

639 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

640 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

641 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

642 "ip_diffim_forced_PsfFlux_flag_edge", True) 

643 for diaSource, forcedSource in zip(diaSources, forcedSources): 

644 diaSource.assign(forcedSource, mapper) 

645 

646 def calculateMetrics(self, difference, diaSources, science, matchedTemplate, idFactory): 

647 """Add image QA metrics to the Task metadata. 

648 

649 Parameters 

650 ---------- 

651 difference : `lsst.afw.image.Exposure` 

652 The target image to calculate metrics for. 

653 diaSources : `lsst.afw.table.SourceCatalog` 

654 The catalog of detected sources. 

655 science : `lsst.afw.image.Exposure` 

656 The science image. 

657 matchedTemplate : `lsst.afw.image.Exposure` 

658 The reference image, warped and psf-matched to the science image. 

659 idFactory : `lsst.afw.table.IdFactory`, optional 

660 Generator object used to assign ids to detected sources in the 

661 difference image. 

662 

663 Returns 

664 ------- 

665 spatiallySampledMetrics : `lsst.afw.table.SourceCatalog`, or `None` 

666 A catalog of randomized locations containing locally evaluated 

667 metric results 

668 """ 

669 mask = difference.mask 

670 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0 

671 self.metadata.add("nGoodPixels", np.sum(~badPix)) 

672 self.metadata.add("nBadPixels", np.sum(badPix)) 

673 detPosPix = (mask.array & mask.getPlaneBitMask("DETECTED")) > 0 

674 detNegPix = (mask.array & mask.getPlaneBitMask("DETECTED_NEGATIVE")) > 0 

675 self.metadata.add("nPixelsDetectedPositive", np.sum(detPosPix)) 

676 self.metadata.add("nPixelsDetectedNegative", np.sum(detNegPix)) 

677 detPosPix &= badPix 

678 detNegPix &= badPix 

679 self.metadata.add("nBadPixelsDetectedPositive", np.sum(detPosPix)) 

680 self.metadata.add("nBadPixelsDetectedNegative", np.sum(detNegPix)) 

681 metricsMaskPlanes = [] 

682 for maskPlane in self.config.metricsMaskPlanes: 

683 try: 

684 self.metadata.add("%s_mask_fraction"%maskPlane.lower(), evaluateMaskFraction(mask, maskPlane)) 

685 metricsMaskPlanes.append(maskPlane) 

686 except InvalidParameterError: 

687 self.metadata.add("%s_mask_fraction"%maskPlane.lower(), -1) 

688 self.log.info("Unable to calculate metrics for mask plane %s: not in image"%maskPlane) 

689 

690 if self.config.doWriteMetrics: 

691 spatiallySampledMetrics = afwTable.SourceCatalog(self.metricSchema) 

692 spatiallySampledMetrics.getTable().setIdFactory(idFactory) 

693 self.addSkySources(spatiallySampledMetrics, science.mask, difference.info.id, 

694 subtask=self.metricSources) 

695 for src in spatiallySampledMetrics: 

696 self._evaluateLocalMetric(src, diaSources, science, matchedTemplate, difference, 

697 metricsMaskPlanes=metricsMaskPlanes) 

698 

699 return spatiallySampledMetrics.asAstropy() 

700 

701 def _evaluateLocalMetric(self, src, diaSources, science, matchedTemplate, difference, 

702 metricsMaskPlanes): 

703 """Calculate image quality metrics at spatially sampled locations. 

704 

705 Parameters 

706 ---------- 

707 src : `lsst.afw.table.SourceRecord` 

708 The source record to be updated with metric calculations. 

709 diaSources : `lsst.afw.table.SourceCatalog` 

710 The catalog of detected sources. 

711 science : `lsst.afw.image.Exposure` 

712 The science image. 

713 matchedTemplate : `lsst.afw.image.Exposure` 

714 The reference image, warped and psf-matched to the science image. 

715 difference : `lsst.afw.image.Exposure` 

716 Result of subtracting template from the science image. 

717 metricsMaskPlanes : `list` of `str` 

718 Mask planes to calculate metrics from. 

719 """ 

720 bbox = src.getFootprint().getBBox() 

721 pix = bbox.getCenter() 

722 src.set('science_psfSize', getPsfFwhm(science.psf, position=pix)) 

723 src.set('template_psfSize', getPsfFwhm(matchedTemplate.psf, position=pix)) 

724 

725 metricRegionSize = 100 

726 bbox.grow(metricRegionSize) 

727 bbox = bbox.clippedTo(science.getBBox()) 

728 nPix = bbox.getArea() 

729 pixScale = science.wcs.getPixelScale() 

730 area = nPix*pixScale.asDegrees()**2 

731 peak = src.getFootprint().getPeaks()[0] 

732 src.set('x', peak['i_x']) 

733 src.set('y', peak['i_y']) 

734 src.setCoord(science.wcs.pixelToSky(peak['i_x'], peak['i_y'])) 

735 selectSources = diaSources[bbox.contains(diaSources.getX(), diaSources.getY())] 

736 if self.config.doSkySources: 

737 selectSources = selectSources[~selectSources["sky_source"]] 

738 sourceDensity = len(selectSources)/area 

739 dipoleSources = selectSources[selectSources["ip_diffim_DipoleFit_flag_classification"]] 

740 dipoleDensity = len(dipoleSources)/area 

741 if dipoleSources: 

742 meanDipoleOrientation = angleMean(dipoleSources["ip_diffim_DipoleFit_orientation"]) 

743 src.set('dipole_direction', meanDipoleOrientation) 

744 meanDipoleSeparation = np.mean(dipoleSources["ip_diffim_DipoleFit_separation"]) 

745 src.set('dipole_separation', meanDipoleSeparation) 

746 templateVal = np.median(matchedTemplate[bbox].image.array) 

747 scienceVal = np.median(science[bbox].image.array) 

748 diffimVal = np.median(difference[bbox].image.array) 

749 src.set('source_density', sourceDensity) 

750 src.set('dipole_density', dipoleDensity) 

751 src.set('template_value', templateVal) 

752 src.set('science_value', scienceVal) 

753 src.set('diffim_value', diffimVal) 

754 for maskPlane in metricsMaskPlanes: 

755 src.set("%s_mask_fraction"%maskPlane.lower(), 

756 evaluateMaskFraction(difference.mask[bbox], maskPlane) 

757 ) 

758 

759 

760class DetectAndMeasureScoreConnections(DetectAndMeasureConnections): 

761 scoreExposure = pipeBase.connectionTypes.Input( 

762 doc="Maximum likelihood image for detection.", 

763 dimensions=("instrument", "visit", "detector"), 

764 storageClass="ExposureF", 

765 name="{fakesType}{coaddName}Diff_scoreExp", 

766 ) 

767 

768 

769class DetectAndMeasureScoreConfig(DetectAndMeasureConfig, 

770 pipelineConnections=DetectAndMeasureScoreConnections): 

771 pass 

772 

773 

774class DetectAndMeasureScoreTask(DetectAndMeasureTask): 

775 """Detect DIA sources using a score image, 

776 and measure the detections on the difference image. 

777 

778 Source detection is run on the supplied score, or maximum likelihood, 

779 image. Note that no additional convolution will be done in this case. 

780 Close positive and negative detections will optionally be merged into 

781 dipole diaSources. 

782 Sky sources, or forced detections in background regions, will optionally 

783 be added, and the configured measurement algorithm will be run on all 

784 detections. 

785 """ 

786 ConfigClass = DetectAndMeasureScoreConfig 

787 _DefaultName = "detectAndMeasureScore" 

788 

789 @timeMethod 

790 def run(self, science, matchedTemplate, difference, scoreExposure, 

791 idFactory=None): 

792 """Detect and measure sources on a score image. 

793 

794 Parameters 

795 ---------- 

796 science : `lsst.afw.image.ExposureF` 

797 Science exposure that the template was subtracted from. 

798 matchedTemplate : `lsst.afw.image.ExposureF` 

799 Warped and PSF-matched template that was used produce the 

800 difference image. 

801 difference : `lsst.afw.image.ExposureF` 

802 Result of subtracting template from the science image. 

803 scoreExposure : `lsst.afw.image.ExposureF` 

804 Score or maximum likelihood difference image 

805 idFactory : `lsst.afw.table.IdFactory`, optional 

806 Generator object used to assign ids to detected sources in the 

807 difference image. Ids from this generator are not set until after 

808 deblending and merging positive/negative peaks. 

809 

810 Returns 

811 ------- 

812 measurementResults : `lsst.pipe.base.Struct` 

813 

814 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

815 Subtracted exposure with detection mask applied. 

816 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

817 The catalog of detected sources. 

818 """ 

819 if idFactory is None: 

820 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory() 

821 

822 # Ensure that we start with an empty detection mask. 

823 mask = scoreExposure.mask 

824 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

825 

826 # Don't use the idFactory until after deblend+merge, so that we aren't 

827 # generating ids that just get thrown away (footprint merge doesn't 

828 # know about past ids). 

829 table = afwTable.SourceTable.make(self.schema) 

830 results = self.detection.run( 

831 table=table, 

832 exposure=scoreExposure, 

833 doSmooth=False, 

834 ) 

835 # Copy the detection mask from the Score image to the difference image 

836 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox()) 

837 

838 sources, positives, negatives = self._deblend(difference, 

839 results.positive, 

840 results.negative) 

841 

842 return self.processResults(science, matchedTemplate, difference, sources, idFactory, 

843 positiveFootprints=positives, negativeFootprints=negatives) 

844 

845 

846def evaluateMaskFraction(mask, maskPlane): 

847 nMaskSet = np.count_nonzero((mask.array & mask.getPlaneBitMask(maskPlane))) 

848 return nMaskSet/mask.array.size