Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 32%

124 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-05 02:45 -0800

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import lsst.afw.table as afwTable 

23import lsst.daf.base as dafBase 

24from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask 

25from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask 

26import lsst.meas.extensions.trailedSources # noqa: F401 

27import lsst.meas.extensions.shapeHSM 

28from lsst.obs.base import ExposureIdInfo 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.utils 

32from lsst.utils.timer import timeMethod 

33 

34from . import DipoleFitTask 

35 

36__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask"] 

37 

38 

39class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections, 

40 dimensions=("instrument", "visit", "detector"), 

41 defaultTemplates={"coaddName": "deep", 

42 "warpTypeSuffix": "", 

43 "fakesType": ""}): 

44 science = pipeBase.connectionTypes.Input( 

45 doc="Input science exposure.", 

46 dimensions=("instrument", "visit", "detector"), 

47 storageClass="ExposureF", 

48 name="{fakesType}calexp" 

49 ) 

50 matchedTemplate = pipeBase.connectionTypes.Input( 

51 doc="Warped and PSF-matched template used to create the difference image.", 

52 dimensions=("instrument", "visit", "detector"), 

53 storageClass="ExposureF", 

54 name="{fakesType}{coaddName}Diff_matchedExp", 

55 ) 

56 difference = pipeBase.connectionTypes.Input( 

57 doc="Result of subtracting template from science.", 

58 dimensions=("instrument", "visit", "detector"), 

59 storageClass="ExposureF", 

60 name="{fakesType}{coaddName}Diff_differenceTempExp", 

61 ) 

62 outputSchema = pipeBase.connectionTypes.InitOutput( 

63 doc="Schema (as an example catalog) for output DIASource catalog.", 

64 storageClass="SourceCatalog", 

65 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

66 ) 

67 diaSources = pipeBase.connectionTypes.Output( 

68 doc="Detected diaSources on the difference image.", 

69 dimensions=("instrument", "visit", "detector"), 

70 storageClass="SourceCatalog", 

71 name="{fakesType}{coaddName}Diff_diaSrc", 

72 ) 

73 subtractedMeasuredExposure = pipeBase.connectionTypes.Output( 

74 doc="Difference image with detection mask plane filled in.", 

75 dimensions=("instrument", "visit", "detector"), 

76 storageClass="ExposureF", 

77 name="{fakesType}{coaddName}Diff_differenceExp", 

78 ) 

79 

80 

81class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, 

82 pipelineConnections=DetectAndMeasureConnections): 

83 """Config for DetectAndMeasureTask 

84 """ 

85 doMerge = pexConfig.Field( 

86 dtype=bool, 

87 default=True, 

88 doc="Merge positive and negative diaSources with grow radius " 

89 "set by growFootprint" 

90 ) 

91 doForcedMeasurement = pexConfig.Field( 

92 dtype=bool, 

93 default=True, 

94 doc="Force photometer diaSource locations on PVI?") 

95 doAddMetrics = pexConfig.Field( 

96 dtype=bool, 

97 default=False, 

98 doc="Add columns to the source table to hold analysis metrics?" 

99 ) 

100 detection = pexConfig.ConfigurableField( 

101 target=SourceDetectionTask, 

102 doc="Final source detection for diaSource measurement", 

103 ) 

104 measurement = pexConfig.ConfigurableField( 

105 target=DipoleFitTask, 

106 doc="Task to measure sources on the difference image.", 

107 ) 

108 doApCorr = lsst.pex.config.Field( 

109 dtype=bool, 

110 default=True, 

111 doc="Run subtask to apply aperture corrections" 

112 ) 

113 applyApCorr = lsst.pex.config.ConfigurableField( 

114 target=ApplyApCorrTask, 

115 doc="Task to apply aperture corrections" 

116 ) 

117 forcedMeasurement = pexConfig.ConfigurableField( 

118 target=ForcedMeasurementTask, 

119 doc="Task to force photometer science image at diaSource locations.", 

120 ) 

121 growFootprint = pexConfig.Field( 

122 dtype=int, 

123 default=2, 

124 doc="Grow positive and negative footprints by this many pixels before merging" 

125 ) 

126 diaSourceMatchRadius = pexConfig.Field( 

127 dtype=float, 

128 default=0.5, 

129 doc="Match radius (in arcseconds) for DiaSource to Source association" 

130 ) 

131 doSkySources = pexConfig.Field( 

132 dtype=bool, 

133 default=False, 

134 doc="Generate sky sources?", 

135 ) 

136 skySources = pexConfig.ConfigurableField( 

137 target=SkyObjectsTask, 

138 doc="Generate sky sources", 

139 ) 

140 

141 def setDefaults(self): 

142 # DiaSource Detection 

143 self.detection.thresholdPolarity = "both" 

144 self.detection.thresholdValue = 5.0 

145 self.detection.reEstimateBackground = False 

146 self.detection.thresholdType = "pixel_stdev" 

147 

148 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

149 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

150 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

151 'base_LocalPhotoCalib', 

152 'base_LocalWcs', 

153 'ext_shapeHSM_HsmSourceMoments', 

154 'ext_shapeHSM_HsmPsfMoments', 

155 ] 

156 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

157 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

158 

159 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

160 self.forcedMeasurement.copyColumns = { 

161 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

162 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

163 self.forcedMeasurement.slots.shape = None 

164 

165 

166class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): 

167 """Detect and measure sources on a difference image. 

168 """ 

169 ConfigClass = DetectAndMeasureConfig 

170 _DefaultName = "detectAndMeasure" 

171 

172 def __init__(self, **kwargs): 

173 super().__init__(**kwargs) 

174 self.schema = afwTable.SourceTable.makeMinimalSchema() 

175 

176 self.algMetadata = dafBase.PropertyList() 

177 self.makeSubtask("detection", schema=self.schema) 

178 self.makeSubtask("measurement", schema=self.schema, 

179 algMetadata=self.algMetadata) 

180 if self.config.doApCorr: 

181 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

182 if self.config.doForcedMeasurement: 

183 self.schema.addField( 

184 "ip_diffim_forced_PsfFlux_instFlux", "D", 

185 "Forced PSF flux measured on the direct image.", 

186 units="count") 

187 self.schema.addField( 

188 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

189 "Forced PSF flux error measured on the direct image.", 

190 units="count") 

191 self.schema.addField( 

192 "ip_diffim_forced_PsfFlux_area", "F", 

193 "Forced PSF flux effective area of PSF.", 

194 units="pixel") 

195 self.schema.addField( 

196 "ip_diffim_forced_PsfFlux_flag", "Flag", 

197 "Forced PSF flux general failure flag.") 

198 self.schema.addField( 

199 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

200 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

201 self.schema.addField( 

202 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

203 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

204 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

205 

206 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

207 self.schema.addField("srcMatchId", "L", "unique id of source match") 

208 if self.config.doSkySources: 

209 self.makeSubtask("skySources") 

210 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

211 

212 # initialize InitOutputs 

213 self.outputSchema = afwTable.SourceCatalog(self.schema) 

214 self.outputSchema.getTable().setMetadata(self.algMetadata) 

215 

216 @staticmethod 

217 def makeIdFactory(expId, expBits): 

218 """Create IdFactory instance for unique 64 bit diaSource id-s. 

219 

220 Parameters 

221 ---------- 

222 expId : `int` 

223 Exposure id. 

224 

225 expBits: `int` 

226 Number of used bits in ``expId``. 

227 

228 Notes 

229 ----- 

230 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

231 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

232 low value end of the integer. 

233 

234 Returns 

235 ------- 

236 idFactory: `lsst.afw.table.IdFactory` 

237 """ 

238 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

239 

240 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

241 inputRefs: pipeBase.InputQuantizedConnection, 

242 outputRefs: pipeBase.OutputQuantizedConnection): 

243 inputs = butlerQC.get(inputRefs) 

244 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

245 returnMaxBits=True) 

246 idFactory = self.makeIdFactory(expId=expId, expBits=expBits) 

247 

248 outputs = self.run(inputs['science'], 

249 inputs['matchedTemplate'], 

250 inputs['difference'], 

251 idFactory=idFactory) 

252 butlerQC.put(outputs, outputRefs) 

253 

254 @timeMethod 

255 def run(self, science, matchedTemplate, difference, 

256 idFactory=None): 

257 """Detect and measure sources on a difference image. 

258 

259 Parameters 

260 ---------- 

261 science : `lsst.afw.image.ExposureF` 

262 Science exposure that the template was subtracted from. 

263 matchedTemplate : `lsst.afw.image.ExposureF` 

264 Warped and PSF-matched template that was used produce the 

265 difference image. 

266 difference : `lsst.afw.image.ExposureF` 

267 Result of subtracting template from the science image. 

268 idFactory : `lsst.afw.table.IdFactory`, optional 

269 Generator object to assign ids to detected sources in the difference image. 

270 

271 Returns 

272 ------- 

273 results : `lsst.pipe.base.Struct` 

274 

275 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

276 Subtracted exposure with detection mask applied. 

277 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

278 The catalog of detected sources. 

279 """ 

280 # Ensure that we start with an empty detection mask. 

281 mask = difference.mask 

282 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

283 

284 table = afwTable.SourceTable.make(self.schema, idFactory) 

285 table.setMetadata(self.algMetadata) 

286 results = self.detection.run( 

287 table=table, 

288 exposure=difference, 

289 doSmooth=True, 

290 ) 

291 

292 if self.config.doMerge: 

293 fpSet = results.fpSets.positive 

294 fpSet.merge(results.fpSets.negative, self.config.growFootprint, 

295 self.config.growFootprint, False) 

296 diaSources = afwTable.SourceCatalog(table) 

297 fpSet.makeSources(diaSources) 

298 self.log.info("Merging detections into %d sources", len(diaSources)) 

299 else: 

300 diaSources = results.sources 

301 

302 if self.config.doSkySources: 

303 self.addSkySources(diaSources, difference.mask, difference.info.id) 

304 

305 self.measureDiaSources(diaSources, science, difference, matchedTemplate) 

306 

307 if self.config.doForcedMeasurement: 

308 self.measureForcedSources(diaSources, science, difference.getWcs()) 

309 

310 return pipeBase.Struct( 

311 subtractedMeasuredExposure=difference, 

312 diaSources=diaSources, 

313 ) 

314 

315 def addSkySources(self, diaSources, mask, seed): 

316 """Add sources in empty regions of the difference image 

317 for measuring the background. 

318 

319 Parameters 

320 ---------- 

321 diaSources : `lsst.afw.table.SourceCatalog` 

322 The catalog of detected sources. 

323 mask : `lsst.afw.image.Mask` 

324 Mask plane for determining regions where Sky sources can be added. 

325 seed : `int` 

326 Seed value to initialize the random number generator. 

327 """ 

328 skySourceFootprints = self.skySources.run(mask=mask, seed=seed) 

329 if skySourceFootprints: 

330 for foot in skySourceFootprints: 

331 s = diaSources.addNew() 

332 s.setFootprint(foot) 

333 s.set(self.skySourceKey, True) 

334 

335 def measureDiaSources(self, diaSources, science, difference, matchedTemplate): 

336 """Use (matched) template and science image to constrain dipole fitting. 

337 

338 Parameters 

339 ---------- 

340 diaSources : `lsst.afw.table.SourceCatalog` 

341 The catalog of detected sources. 

342 science : `lsst.afw.image.ExposureF` 

343 Science exposure that the template was subtracted from. 

344 difference : `lsst.afw.image.ExposureF` 

345 Result of subtracting template from the science image. 

346 matchedTemplate : `lsst.afw.image.ExposureF` 

347 Warped and PSF-matched template that was used produce the 

348 difference image. 

349 """ 

350 # Note that this may not be correct if we convolved the science image. 

351 # In the future we may wish to persist the matchedScience image. 

352 self.measurement.run(diaSources, difference, science, matchedTemplate) 

353 if self.config.doApCorr: 

354 self.applyApCorr.run( 

355 catalog=diaSources, 

356 apCorrMap=difference.getInfo().getApCorrMap() 

357 ) 

358 

359 def measureForcedSources(self, diaSources, science, wcs): 

360 """Perform forced measurement of the diaSources on the science image. 

361 

362 Parameters 

363 ---------- 

364 diaSources : `lsst.afw.table.SourceCatalog` 

365 The catalog of detected sources. 

366 science : `lsst.afw.image.ExposureF` 

367 Science exposure that the template was subtracted from. 

368 wcs : `lsst.afw.geom.SkyWcs` 

369 Coordinate system definition (wcs) for the exposure. 

370 """ 

371 # Run forced psf photometry on the PVI at the diaSource locations. 

372 # Copy the measured flux and error into the diaSource. 

373 forcedSources = self.forcedMeasurement.generateMeasCat( 

374 science, diaSources, wcs) 

375 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs) 

376 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

377 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

378 "ip_diffim_forced_PsfFlux_instFlux", True) 

379 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

380 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

381 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

382 "ip_diffim_forced_PsfFlux_area", True) 

383 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

384 "ip_diffim_forced_PsfFlux_flag", True) 

385 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

386 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

387 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

388 "ip_diffim_forced_PsfFlux_flag_edge", True) 

389 for diaSource, forcedSource in zip(diaSources, forcedSources): 

390 diaSource.assign(forcedSource, mapper)