Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 37%

1# This file is part of ip_diffim.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <https://www.gnu.org/licenses/>.

22from deprecated.sphinx import deprecated

24import lsst.afw.table as afwTable

25import lsst.daf.base as dafBase

26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask

27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig

28import lsst.meas.extensions.trailedSources # noqa: F401

29import lsst.meas.extensions.shapeHSM

30from lsst.obs.base import ExposureIdInfo

31import lsst.pex.config as pexConfig

32import lsst.pipe.base as pipeBase

33import lsst.utils

34from lsst.utils.timer import timeMethod

36from . import DipoleFitTask

38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",

39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]

42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,

43 dimensions=("instrument", "visit", "detector"),

44 defaultTemplates={"coaddName": "deep",

45 "warpTypeSuffix": "",

46 "fakesType": ""}):

47 science = pipeBase.connectionTypes.Input(

48 doc="Input science exposure.",

49 dimensions=("instrument", "visit", "detector"),

50 storageClass="ExposureF",

51 name="{fakesType}calexp"

52 )

53 matchedTemplate = pipeBase.connectionTypes.Input(

54 doc="Warped and PSF-matched template used to create the difference image.",

55 dimensions=("instrument", "visit", "detector"),

56 storageClass="ExposureF",

57 name="{fakesType}{coaddName}Diff_matchedExp",

58 )

59 difference = pipeBase.connectionTypes.Input(

60 doc="Result of subtracting template from science.",

61 dimensions=("instrument", "visit", "detector"),

62 storageClass="ExposureF",

63 name="{fakesType}{coaddName}Diff_differenceTempExp",

64 )

65 outputSchema = pipeBase.connectionTypes.InitOutput(

66 doc="Schema (as an example catalog) for output DIASource catalog.",

67 storageClass="SourceCatalog",

68 name="{fakesType}{coaddName}Diff_diaSrc_schema",

69 )

70 diaSources = pipeBase.connectionTypes.Output(

71 doc="Detected diaSources on the difference image.",

72 dimensions=("instrument", "visit", "detector"),

73 storageClass="SourceCatalog",

74 name="{fakesType}{coaddName}Diff_diaSrc",

75 )

76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(

77 doc="Difference image with detection mask plane filled in.",

78 dimensions=("instrument", "visit", "detector"),

79 storageClass="ExposureF",

80 name="{fakesType}{coaddName}Diff_differenceExp",

81 )

84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,

85 pipelineConnections=DetectAndMeasureConnections):

86 """Config for DetectAndMeasureTask

87 """

88 doMerge = pexConfig.Field(

89 dtype=bool,

90 default=True,

91 doc="Merge positive and negative diaSources with grow radius "

92 "set by growFootprint"

93 )

94 doForcedMeasurement = pexConfig.Field(

95 dtype=bool,

96 default=True,

97 doc="Force photometer diaSource locations on PVI?")

98 doAddMetrics = pexConfig.Field(

99 dtype=bool,

100 default=False,

101 doc="Add columns to the source table to hold analysis metrics?"

102 )

103 detection = pexConfig.ConfigurableField(

104 target=SourceDetectionTask,

105 doc="Final source detection for diaSource measurement",

106 )

107 measurement = pexConfig.ConfigurableField(

108 target=DipoleFitTask,

109 doc="Task to measure sources on the difference image.",

110 )

111 doApCorr = lsst.pex.config.Field(

112 dtype=bool,

113 default=True,

114 doc="Run subtask to apply aperture corrections"

115 )

116 applyApCorr = lsst.pex.config.ConfigurableField(

117 target=ApplyApCorrTask,

118 doc="Task to apply aperture corrections"

119 )

120 forcedMeasurement = pexConfig.ConfigurableField(

121 target=ForcedMeasurementTask,

122 doc="Task to force photometer science image at diaSource locations.",

123 )

124 growFootprint = pexConfig.Field(

125 dtype=int,

126 default=2,

127 doc="Grow positive and negative footprints by this many pixels before merging"

128 )

129 diaSourceMatchRadius = pexConfig.Field(

130 dtype=float,

131 default=0.5,

132 doc="Match radius (in arcseconds) for DiaSource to Source association"

133 )

134 doSkySources = pexConfig.Field(

135 dtype=bool,

136 default=False,

137 doc="Generate sky sources?",

138 )

139 skySources = pexConfig.ConfigurableField(

140 target=SkyObjectsTask,

141 doc="Generate sky sources",

142 )

143 idGenerator = DetectorVisitIdGeneratorConfig.make_field()

144

145 def setDefaults(self):

146 # DiaSource Detection

147 self.detection.thresholdPolarity = "both"

148 self.detection.thresholdValue = 5.0

149 self.detection.reEstimateBackground = False

150 self.detection.thresholdType = "pixel_stdev"

151 self.detection.excludeMaskPlanes = ["EDGE"]

152

153 # Add filtered flux measurement, the correct measurement for pre-convolved images.

154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')

155 self.measurement.plugins.names |= ['ext_trailedSources_Naive',

156 'base_LocalPhotoCalib',

157 'base_LocalWcs',

158 'ext_shapeHSM_HsmSourceMoments',

159 'ext_shapeHSM_HsmPsfMoments',

160 ]

161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"

162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"

163 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0

164 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0

165 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]

166 self.forcedMeasurement.copyColumns = {

167 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}

168 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"

169 self.forcedMeasurement.slots.shape = None

170

171 # Keep track of which footprints contain streaks

172 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK']

173 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK']

174

175

176class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):

177 """Detect and measure sources on a difference image.

178 """

179 ConfigClass = DetectAndMeasureConfig

180 _DefaultName = "detectAndMeasure"

181

182 def __init__(self, **kwargs):

183 super().__init__(**kwargs)

184 self.schema = afwTable.SourceTable.makeMinimalSchema()

185 # Add coordinate error fields:

186 afwTable.CoordKey.addErrorFields(self.schema)

187

188 self.algMetadata = dafBase.PropertyList()

189 self.makeSubtask("detection", schema=self.schema)

190 self.makeSubtask("measurement", schema=self.schema,

191 algMetadata=self.algMetadata)

192 if self.config.doApCorr:

193 self.makeSubtask("applyApCorr", schema=self.measurement.schema)

194 if self.config.doForcedMeasurement:

195 self.schema.addField(

196 "ip_diffim_forced_PsfFlux_instFlux", "D",

197 "Forced PSF flux measured on the direct image.",

198 units="count")

199 self.schema.addField(

200 "ip_diffim_forced_PsfFlux_instFluxErr", "D",

201 "Forced PSF flux error measured on the direct image.",

202 units="count")

203 self.schema.addField(

204 "ip_diffim_forced_PsfFlux_area", "F",

205 "Forced PSF flux effective area of PSF.",

206 units="pixel")

207 self.schema.addField(

208 "ip_diffim_forced_PsfFlux_flag", "Flag",

209 "Forced PSF flux general failure flag.")

210 self.schema.addField(

211 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",

212 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")

213 self.schema.addField(

214 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",

215 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")

216 self.makeSubtask("forcedMeasurement", refSchema=self.schema)

217

218 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")

219 self.schema.addField("srcMatchId", "L", "unique id of source match")

220 if self.config.doSkySources:

221 self.makeSubtask("skySources")

222 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")

223

224 # initialize InitOutputs

225 self.outputSchema = afwTable.SourceCatalog(self.schema)

226 self.outputSchema.getTable().setMetadata(self.algMetadata)

227

228 # TODO: remove on DM-38687.

229 @staticmethod

230 @deprecated(

231 reason=(

232 "ID factory construction now depends on configuration; use the "

233 "idGenerator config field. Will be removed after v26."

234 ),

235 version="v26.0",

236 category=FutureWarning,

237 )

238 def makeIdFactory(expId, expBits):

239 """Create IdFactory instance for unique 64 bit diaSource id-s.

240

241 Parameters

242 ----------

243 expId : `int`

244 Exposure id.

245

246 expBits: `int`

247 Number of used bits in ``expId``.

248

249 Notes

250 -----

251 The diasource id-s consists of the ``expId`` stored fixed in the highest value

252 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the

253 low value end of the integer.

254

255 Returns

256 -------

257 idFactory: `lsst.afw.table.IdFactory`

258 """

259 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()

260

261 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,

262 inputRefs: pipeBase.InputQuantizedConnection,

263 outputRefs: pipeBase.OutputQuantizedConnection):

264 inputs = butlerQC.get(inputRefs)

265 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)

266 idFactory = idGenerator.make_table_id_factory()

267 outputs = self.run(**inputs, idFactory=idFactory)

268 butlerQC.put(outputs, outputRefs)

269

270 @timeMethod

271 def run(self, science, matchedTemplate, difference,

272 idFactory=None):

273 """Detect and measure sources on a difference image.

274

275 The difference image will be convolved with a gaussian approximation of

276 the PSF to form a maximum likelihood image for detection.

277 Close positive and negative detections will optionally be merged into

278 dipole diaSources.

279 Sky sources, or forced detections in background regions, will optionally

280 be added, and the configured measurement algorithm will be run on all

281 detections.

282

283 Parameters

284 ----------

285 science : `lsst.afw.image.ExposureF`

286 Science exposure that the template was subtracted from.

287 matchedTemplate : `lsst.afw.image.ExposureF`

288 Warped and PSF-matched template that was used produce the

289 difference image.

290 difference : `lsst.afw.image.ExposureF`

291 Result of subtracting template from the science image.

292 idFactory : `lsst.afw.table.IdFactory`, optional

293 Generator object to assign ids to detected sources in the difference image.

294

295 Returns

296 -------

297 measurementResults : `lsst.pipe.base.Struct`

298

299 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`

300 Subtracted exposure with detection mask applied.

301 ``diaSources`` : `lsst.afw.table.SourceCatalog`

302 The catalog of detected sources.

303 """

304 # Ensure that we start with an empty detection mask.

305 mask = difference.mask

306 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))

307

308 table = afwTable.SourceTable.make(self.schema, idFactory)

309 table.setMetadata(self.algMetadata)

310 results = self.detection.run(

311 table=table,

312 exposure=difference,

313 doSmooth=True,

314 )

315

316 return self.processResults(science, matchedTemplate, difference, results.sources, table,

317 positiveFootprints=results.positive, negativeFootprints=results.negative)

318

319 def processResults(self, science, matchedTemplate, difference, sources, table,

320 positiveFootprints=None, negativeFootprints=None,):

321 """Measure and process the results of source detection.

322

323 Parameters

324 ----------

325 sources : `lsst.afw.table.SourceCatalog`

326 Detected sources on the difference exposure.

327 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional

328 Positive polarity footprints.

329 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional

330 Negative polarity footprints.

331 table : `lsst.afw.table.SourceTable`

332 Table object that will be used to create the SourceCatalog.

333 science : `lsst.afw.image.ExposureF`

334 Science exposure that the template was subtracted from.

335 matchedTemplate : `lsst.afw.image.ExposureF`

336 Warped and PSF-matched template that was used produce the

337 difference image.

338 difference : `lsst.afw.image.ExposureF`

339 Result of subtracting template from the science image.

340

341 Returns

342 -------

343 measurementResults : `lsst.pipe.base.Struct`

344

345 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`

346 Subtracted exposure with detection mask applied.

347 ``diaSources`` : `lsst.afw.table.SourceCatalog`

348 The catalog of detected sources.

349 """

350 if self.config.doMerge:

351 fpSet = positiveFootprints

352 fpSet.merge(negativeFootprints, self.config.growFootprint,

353 self.config.growFootprint, False)

354 diaSources = afwTable.SourceCatalog(table)

355 fpSet.makeSources(diaSources)

356 self.log.info("Merging detections into %d sources", len(diaSources))

357 else:

358 diaSources = sources

359

360 if self.config.doSkySources:

361 self.addSkySources(diaSources, difference.mask, difference.info.id)

362

363 self.measureDiaSources(diaSources, science, difference, matchedTemplate)

364

365 if self.config.doForcedMeasurement:

366 self.measureForcedSources(diaSources, science, difference.getWcs())

367

368 measurementResults = pipeBase.Struct(

369 subtractedMeasuredExposure=difference,

370 diaSources=diaSources,

371 )

372

373 return measurementResults

374

375 def addSkySources(self, diaSources, mask, seed):

376 """Add sources in empty regions of the difference image

377 for measuring the background.

378

379 Parameters

380 ----------

381 diaSources : `lsst.afw.table.SourceCatalog`

382 The catalog of detected sources.

383 mask : `lsst.afw.image.Mask`

384 Mask plane for determining regions where Sky sources can be added.

385 seed : `int`

386 Seed value to initialize the random number generator.

387 """

388 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)

389 if skySourceFootprints:

390 for foot in skySourceFootprints:

391 s = diaSources.addNew()

392 s.setFootprint(foot)

393 s.set(self.skySourceKey, True)

394

395 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):

396 """Use (matched) template and science image to constrain dipole fitting.

397

398 Parameters

399 ----------

400 diaSources : `lsst.afw.table.SourceCatalog`

401 The catalog of detected sources.

402 science : `lsst.afw.image.ExposureF`

403 Science exposure that the template was subtracted from.

404 difference : `lsst.afw.image.ExposureF`

405 Result of subtracting template from the science image.

406 matchedTemplate : `lsst.afw.image.ExposureF`

407 Warped and PSF-matched template that was used produce the

408 difference image.

409 """

410 # Note that this may not be correct if we convolved the science image.

411 # In the future we may wish to persist the matchedScience image.

412 self.measurement.run(diaSources, difference, science, matchedTemplate)

413 if self.config.doApCorr:

414 apCorrMap = difference.getInfo().getApCorrMap()

415 if apCorrMap is None:

416 self.log.warning("Difference image does not have valid aperture correction; skipping.")

417 else:

418 self.applyApCorr.run(

419 catalog=diaSources,

420 apCorrMap=apCorrMap,

421 )

422

423 def measureForcedSources(self, diaSources, science, wcs):

424 """Perform forced measurement of the diaSources on the science image.

425

426 Parameters

427 ----------

428 diaSources : `lsst.afw.table.SourceCatalog`

429 The catalog of detected sources.

430 science : `lsst.afw.image.ExposureF`

431 Science exposure that the template was subtracted from.

432 wcs : `lsst.afw.geom.SkyWcs`

433 Coordinate system definition (wcs) for the exposure.

434 """

435 # Run forced psf photometry on the PVI at the diaSource locations.

436 # Copy the measured flux and error into the diaSource.

437 forcedSources = self.forcedMeasurement.generateMeasCat(

438 science, diaSources, wcs)

439 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)

440 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)

441 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],

442 "ip_diffim_forced_PsfFlux_instFlux", True)

443 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],

444 "ip_diffim_forced_PsfFlux_instFluxErr", True)

445 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],

446 "ip_diffim_forced_PsfFlux_area", True)

447 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],

448 "ip_diffim_forced_PsfFlux_flag", True)

449 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],

450 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)

451 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],

452 "ip_diffim_forced_PsfFlux_flag_edge", True)

453 for diaSource, forcedSource in zip(diaSources, forcedSources):

454 diaSource.assign(forcedSource, mapper)

455

456

457class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):

458 scoreExposure = pipeBase.connectionTypes.Input(

459 doc="Maximum likelihood image for detection.",

460 dimensions=("instrument", "visit", "detector"),

461 storageClass="ExposureF",

462 name="{fakesType}{coaddName}Diff_scoreExp",

463 )

464

465

466class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,

467 pipelineConnections=DetectAndMeasureScoreConnections):

468 pass

469

470

471class DetectAndMeasureScoreTask(DetectAndMeasureTask):

472 """Detect DIA sources using a score image,

473 and measure the detections on the difference image.

474

475 Source detection is run on the supplied score, or maximum likelihood,

476 image. Note that no additional convolution will be done in this case.

477 Close positive and negative detections will optionally be merged into

478 dipole diaSources.

479 Sky sources, or forced detections in background regions, will optionally

480 be added, and the configured measurement algorithm will be run on all

481 detections.

482 """

483 ConfigClass = DetectAndMeasureScoreConfig

484 _DefaultName = "detectAndMeasureScore"

485

486 @timeMethod

487 def run(self, science, matchedTemplate, difference, scoreExposure,

488 idFactory=None):

489 """Detect and measure sources on a score image.

490

491 Parameters

492 ----------

493 science : `lsst.afw.image.ExposureF`

494 Science exposure that the template was subtracted from.

495 matchedTemplate : `lsst.afw.image.ExposureF`

496 Warped and PSF-matched template that was used produce the

497 difference image.

498 difference : `lsst.afw.image.ExposureF`

499 Result of subtracting template from the science image.

500 scoreExposure : `lsst.afw.image.ExposureF`

501 Score or maximum likelihood difference image

502 idFactory : `lsst.afw.table.IdFactory`, optional

503 Generator object to assign ids to detected sources in the difference image.

504

505 Returns

506 -------

507 measurementResults : `lsst.pipe.base.Struct`

508

509 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`

510 Subtracted exposure with detection mask applied.

511 ``diaSources`` : `lsst.afw.table.SourceCatalog`

512 The catalog of detected sources.

513 """

514 # Ensure that we start with an empty detection mask.

515 mask = scoreExposure.mask

516 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))

517

518 table = afwTable.SourceTable.make(self.schema, idFactory)

519 table.setMetadata(self.algMetadata)

520 results = self.detection.run(

521 table=table,

522 exposure=scoreExposure,

523 doSmooth=False,

524 )

525 # Copy the detection mask from the Score image to the difference image

526 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())

527

528 return self.processResults(science, matchedTemplate, difference, results.sources, table,

529 positiveFootprints=results.positive, negativeFootprints=results.negative)