Coverage for python/lsst/pipe/tasks/imageDifference.py: 15%

551 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-06 02:41 -0800

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"] 

23 

24import math 

25import random 

26import numpy 

27 

28import lsst.utils 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.daf.base as dafBase 

32import lsst.geom as geom 

33import lsst.afw.math as afwMath 

34import lsst.afw.table as afwTable 

35import lsst.meas.extensions.trailedSources # noqa: F401 

36from lsst.meas.algorithms import (SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask, 

37 LoadReferenceObjectsConfig, SkyObjectsTask, 

38 ScaleVarianceTask) 

39from lsst.meas.astrom import AstrometryConfig, AstrometryTask 

40from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask 

41from lsst.pipe.tasks.registerImage import RegisterTask 

42from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList, 

43 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, 

44 GetCoaddAsTemplateTask, DipoleFitTask, 

45 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry) 

46import lsst.ip.diffim.diffimTools as diffimTools 

47import lsst.ip.diffim.utils as diUtils 

48import lsst.afw.display as afwDisplay 

49from lsst.skymap import BaseSkyMap 

50from lsst.obs.base import ExposureIdInfo 

51from lsst.utils.timer import timeMethod 

52 

53from deprecated.sphinx import deprecated 

54 

55FwhmPerSigma = 2*math.sqrt(2*math.log(2)) 

56IqrToSigma = 0.741 

57 

58 

59class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections, 

60 dimensions=("instrument", "visit", "detector", "skymap"), 

61 defaultTemplates={"coaddName": "deep", 

62 "skyMapName": "deep", 

63 "warpTypeSuffix": "", 

64 "fakesType": ""}): 

65 

66 exposure = pipeBase.connectionTypes.Input( 

67 doc="Input science exposure to subtract from.", 

68 dimensions=("instrument", "visit", "detector"), 

69 storageClass="ExposureF", 

70 name="{fakesType}calexp" 

71 ) 

72 

73 # TODO DM-22953 

74 # kernelSources = pipeBase.connectionTypes.Input( 

75 # doc="Source catalog produced in calibrate task for kernel candidate sources", 

76 # name="src", 

77 # storageClass="SourceCatalog", 

78 # dimensions=("instrument", "visit", "detector"), 

79 # ) 

80 

81 skyMap = pipeBase.connectionTypes.Input( 

82 doc="Input definition of geometry/bbox and projection/wcs for template exposures", 

83 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

84 dimensions=("skymap", ), 

85 storageClass="SkyMap", 

86 ) 

87 coaddExposures = pipeBase.connectionTypes.Input( 

88 doc="Input template to match and subtract from the exposure", 

89 dimensions=("tract", "patch", "skymap", "band"), 

90 storageClass="ExposureF", 

91 name="{fakesType}{coaddName}Coadd{warpTypeSuffix}", 

92 multiple=True, 

93 deferLoad=True 

94 ) 

95 dcrCoadds = pipeBase.connectionTypes.Input( 

96 doc="Input DCR template to match and subtract from the exposure", 

97 name="{fakesType}dcrCoadd{warpTypeSuffix}", 

98 storageClass="ExposureF", 

99 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

100 multiple=True, 

101 deferLoad=True 

102 ) 

103 finalizedPsfApCorrCatalog = pipeBase.connectionTypes.Input( 

104 doc=("Per-visit finalized psf models and aperture correction maps. " 

105 "These catalogs use the detector id for the catalog id, " 

106 "sorted on id for fast lookup."), 

107 name="finalized_psf_ap_corr_catalog", 

108 storageClass="ExposureCatalog", 

109 dimensions=("instrument", "visit"), 

110 ) 

111 outputSchema = pipeBase.connectionTypes.InitOutput( 

112 doc="Schema (as an example catalog) for output DIASource catalog.", 

113 storageClass="SourceCatalog", 

114 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

115 ) 

116 subtractedExposure = pipeBase.connectionTypes.Output( 

117 doc="Output AL difference or Zogy proper difference image", 

118 dimensions=("instrument", "visit", "detector"), 

119 storageClass="ExposureF", 

120 name="{fakesType}{coaddName}Diff_differenceExp", 

121 ) 

122 scoreExposure = pipeBase.connectionTypes.Output( 

123 doc="Output AL likelihood or Zogy score image", 

124 dimensions=("instrument", "visit", "detector"), 

125 storageClass="ExposureF", 

126 name="{fakesType}{coaddName}Diff_scoreExp", 

127 ) 

128 warpedExposure = pipeBase.connectionTypes.Output( 

129 doc="Warped template used to create `subtractedExposure`.", 

130 dimensions=("instrument", "visit", "detector"), 

131 storageClass="ExposureF", 

132 name="{fakesType}{coaddName}Diff_warpedExp", 

133 ) 

134 matchedExposure = pipeBase.connectionTypes.Output( 

135 doc="Warped template used to create `subtractedExposure`.", 

136 dimensions=("instrument", "visit", "detector"), 

137 storageClass="ExposureF", 

138 name="{fakesType}{coaddName}Diff_matchedExp", 

139 ) 

140 diaSources = pipeBase.connectionTypes.Output( 

141 doc="Output detected diaSources on the difference image", 

142 dimensions=("instrument", "visit", "detector"), 

143 storageClass="SourceCatalog", 

144 name="{fakesType}{coaddName}Diff_diaSrc", 

145 ) 

146 

147 def __init__(self, *, config=None): 

148 super().__init__(config=config) 

149 if config.coaddName == 'dcr': 

150 self.inputs.remove("coaddExposures") 

151 else: 

152 self.inputs.remove("dcrCoadds") 

153 if not config.doWriteSubtractedExp: 

154 self.outputs.remove("subtractedExposure") 

155 if not config.doWriteScoreExp: 

156 self.outputs.remove("scoreExposure") 

157 if not config.doWriteWarpedExp: 

158 self.outputs.remove("warpedExposure") 

159 if not config.doWriteMatchedExp: 

160 self.outputs.remove("matchedExposure") 

161 if not config.doWriteSources: 

162 self.outputs.remove("diaSources") 

163 if not config.doApplyFinalizedPsf: 

164 self.inputs.remove("finalizedPsfApCorrCatalog") 

165 

166 # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef) 

167 # Make kernelSources optional 

168 

169 

170class ImageDifferenceConfig(pipeBase.PipelineTaskConfig, 

171 pipelineConnections=ImageDifferenceTaskConnections): 

172 """Config for ImageDifferenceTask. 

173 """ 

174 

175 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False, 

176 doc="Add background to calexp before processing it. " 

177 "Useful as ipDiffim does background matching.") 

178 doUseRegister = pexConfig.Field(dtype=bool, default=False, 

179 doc="Re-compute astrometry on the template. " 

180 "Use image-to-image registration to align template with " 

181 "science image (AL only).") 

182 doDebugRegister = pexConfig.Field(dtype=bool, default=False, 

183 doc="Writing debugging data for doUseRegister") 

184 doSelectSources = pexConfig.Field(dtype=bool, default=False, 

185 doc="Select stars to use for kernel fitting (AL only)") 

186 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False, 

187 doc="Select stars of extreme color as part " 

188 "of the control sample (AL only)") 

189 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False, 

190 doc="Select stars that are variable to be part " 

191 "of the control sample (AL only)") 

192 doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?") 

193 doPreConvolve = pexConfig.Field(dtype=bool, default=False, 

194 doc="Not in use. Superseded by useScoreImageDetection.", 

195 deprecated="This option superseded by useScoreImageDetection." 

196 " Will be removed after v22.") 

197 useScoreImageDetection = pexConfig.Field( 

198 dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or " 

199 "the Zogy score image. Use it for source detection (if doDetection=True).") 

200 doWriteScoreExp = pexConfig.Field( 

201 dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?") 

202 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False, 

203 doc="Scale variance of the template before PSF matching") 

204 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True, 

205 doc="Scale variance of the diffim before PSF matching. " 

206 "You may do either this or template variance scaling, " 

207 "or neither. (Doing both is a waste of CPU.)") 

208 useGaussianForPreConvolution = pexConfig.Field( 

209 dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution " 

210 "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)") 

211 doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?") 

212 doDecorrelation = pexConfig.Field(dtype=bool, default=True, 

213 doc="Perform diffim decorrelation to undo pixel correlation due to A&L " 

214 "kernel convolution (AL only)? If True, also update the diffim PSF.") 

215 doMerge = pexConfig.Field(dtype=bool, default=True, 

216 doc="Merge positive and negative diaSources with grow radius " 

217 "set by growFootprint") 

218 doMatchSources = pexConfig.Field(dtype=bool, default=False, 

219 doc="Match diaSources with input calexp sources and ref catalog sources") 

220 doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?") 

221 doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?") 

222 doForcedMeasurement = pexConfig.Field( 

223 dtype=bool, 

224 default=True, 

225 doc="Force photometer diaSource locations on PVI?") 

226 doWriteSubtractedExp = pexConfig.Field( 

227 dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?") 

228 doWriteWarpedExp = pexConfig.Field( 

229 dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?") 

230 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False, 

231 doc="Write warped and PSF-matched template coadd exposure?") 

232 doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?") 

233 doAddMetrics = pexConfig.Field(dtype=bool, default=False, 

234 doc="Add columns to the source table to hold analysis metrics?") 

235 doApplyFinalizedPsf = pexConfig.Field( 

236 doc="Whether to apply finalized psf models and aperture correction map.", 

237 dtype=bool, 

238 default=False, 

239 ) 

240 

241 coaddName = pexConfig.Field( 

242 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

243 dtype=str, 

244 default="deep", 

245 ) 

246 convolveTemplate = pexConfig.Field( 

247 doc="Which image gets convolved (default = template)", 

248 dtype=bool, 

249 default=True 

250 ) 

251 refObjLoader = pexConfig.ConfigField( 

252 dtype=LoadReferenceObjectsConfig, 

253 doc="reference object loader", 

254 ) 

255 astrometer = pexConfig.ConfigurableField( 

256 target=AstrometryTask, 

257 doc="astrometry task; used to match sources to reference objects, but not to fit a WCS", 

258 ) 

259 sourceSelector = pexConfig.ConfigurableField( 

260 target=ObjectSizeStarSelectorTask, 

261 doc="Source selection algorithm", 

262 ) 

263 subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al") 

264 decorrelate = pexConfig.ConfigurableField( 

265 target=DecorrelateALKernelSpatialTask, 

266 doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. " 

267 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the " 

268 "default of 5.5).", 

269 ) 

270 # Old style ImageMapper grid. ZogyTask has its own grid option 

271 doSpatiallyVarying = pexConfig.Field( 

272 dtype=bool, 

273 default=False, 

274 doc="Perform A&L decorrelation on a grid across the " 

275 "image in order to allow for spatial variations. Zogy does not use this option." 

276 ) 

277 detection = pexConfig.ConfigurableField( 

278 target=SourceDetectionTask, 

279 doc="Low-threshold detection for final measurement", 

280 ) 

281 measurement = pexConfig.ConfigurableField( 

282 target=DipoleFitTask, 

283 doc="Enable updated dipole fitting method", 

284 ) 

285 doApCorr = lsst.pex.config.Field( 

286 dtype=bool, 

287 default=True, 

288 doc="Run subtask to apply aperture corrections" 

289 ) 

290 applyApCorr = lsst.pex.config.ConfigurableField( 

291 target=ApplyApCorrTask, 

292 doc="Subtask to apply aperture corrections" 

293 ) 

294 forcedMeasurement = pexConfig.ConfigurableField( 

295 target=ForcedMeasurementTask, 

296 doc="Subtask to force photometer PVI at diaSource location.", 

297 ) 

298 getTemplate = pexConfig.ConfigurableField( 

299 target=GetCoaddAsTemplateTask, 

300 doc="Subtask to retrieve template exposure and sources", 

301 ) 

302 scaleVariance = pexConfig.ConfigurableField( 

303 target=ScaleVarianceTask, 

304 doc="Subtask to rescale the variance of the template " 

305 "to the statistically expected level" 

306 ) 

307 controlStepSize = pexConfig.Field( 

308 doc="What step size (every Nth one) to select a control sample from the kernelSources", 

309 dtype=int, 

310 default=5 

311 ) 

312 controlRandomSeed = pexConfig.Field( 

313 doc="Random seed for shuffing the control sample", 

314 dtype=int, 

315 default=10 

316 ) 

317 register = pexConfig.ConfigurableField( 

318 target=RegisterTask, 

319 doc="Task to enable image-to-image image registration (warping)", 

320 ) 

321 kernelSourcesFromRef = pexConfig.Field( 

322 doc="Select sources to measure kernel from reference catalog if True, template if false", 

323 dtype=bool, 

324 default=False 

325 ) 

326 templateSipOrder = pexConfig.Field( 

327 dtype=int, default=2, 

328 doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)" 

329 ) 

330 growFootprint = pexConfig.Field( 

331 dtype=int, default=2, 

332 doc="Grow positive and negative footprints by this amount before merging" 

333 ) 

334 diaSourceMatchRadius = pexConfig.Field( 

335 dtype=float, default=0.5, 

336 doc="Match radius (in arcseconds) for DiaSource to Source association" 

337 ) 

338 requiredTemplateFraction = pexConfig.Field( 

339 dtype=float, default=0.1, 

340 doc="Do not attempt to run task if template covers less than this fraction of pixels." 

341 "Setting to 0 will always attempt image subtraction" 

342 ) 

343 doSkySources = pexConfig.Field( 

344 dtype=bool, 

345 default=False, 

346 doc="Generate sky sources?", 

347 ) 

348 skySources = pexConfig.ConfigurableField( 

349 target=SkyObjectsTask, 

350 doc="Generate sky sources", 

351 ) 

352 

353 def setDefaults(self): 

354 # defaults are OK for catalog and diacatalog 

355 

356 self.subtract['al'].kernel.name = "AL" 

357 self.subtract['al'].kernel.active.fitForBackground = True 

358 self.subtract['al'].kernel.active.spatialKernelOrder = 1 

359 self.subtract['al'].kernel.active.spatialBgOrder = 2 

360 

361 # DiaSource Detection 

362 self.detection.thresholdPolarity = "both" 

363 self.detection.thresholdValue = 5.0 

364 self.detection.reEstimateBackground = False 

365 self.detection.thresholdType = "pixel_stdev" 

366 

367 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

368 # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier. 

369 # To change that you must modify algorithms.names in the task's applyOverrides method, 

370 # after the user has set doPreConvolve. 

371 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

372 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

373 'base_LocalPhotoCalib', 

374 'base_LocalWcs'] 

375 

376 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

377 self.forcedMeasurement.copyColumns = { 

378 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

379 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

380 self.forcedMeasurement.slots.shape = None 

381 

382 # For shuffling the control sample 

383 random.seed(self.controlRandomSeed) 

384 

385 def validate(self): 

386 pexConfig.Config.validate(self) 

387 if not self.doSubtract and not self.doDetection: 

388 raise ValueError("Either doSubtract or doDetection must be enabled.") 

389 if self.doMeasurement and not self.doDetection: 

390 raise ValueError("Cannot run source measurement without source detection.") 

391 if self.doMerge and not self.doDetection: 

392 raise ValueError("Cannot run source merging without source detection.") 

393 if self.doSkySources and not self.doDetection: 

394 raise ValueError("Cannot run sky source creation without source detection.") 

395 if self.doUseRegister and not self.doSelectSources: 

396 raise ValueError("doUseRegister=True and doSelectSources=False. " 

397 "Cannot run RegisterTask without selecting sources.") 

398 if self.doScaleDiffimVariance and self.doScaleTemplateVariance: 

399 raise ValueError("Scaling the diffim variance and scaling the template variance " 

400 "are both set. Please choose one or the other.") 

401 # We cannot allow inconsistencies that would lead to None or not available output products 

402 if self.subtract.name == 'zogy': 

403 if self.doWriteMatchedExp: 

404 raise ValueError("doWriteMatchedExp=True Matched exposure is not " 

405 "calculated in zogy subtraction.") 

406 if self.doAddMetrics: 

407 raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.") 

408 if self.doDecorrelation: 

409 raise ValueError( 

410 "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.") 

411 if self.doSelectSources: 

412 raise ValueError( 

413 "doSelectSources=True Selecting sources for PSF matching is not a zogy option.") 

414 if self.useGaussianForPreConvolution: 

415 raise ValueError( 

416 "useGaussianForPreConvolution=True This is an AL subtraction only option.") 

417 else: 

418 # AL only consistency checks 

419 if self.useScoreImageDetection and not self.convolveTemplate: 

420 raise ValueError( 

421 "convolveTemplate=False and useScoreImageDetection=True " 

422 "Pre-convolution and matching of the science image is not a supported operation.") 

423 if self.doWriteSubtractedExp and self.useScoreImageDetection: 

424 raise ValueError( 

425 "doWriteSubtractedExp=True and useScoreImageDetection=True " 

426 "Regular difference image is not calculated. " 

427 "AL subtraction calculates either the regular difference image or the score image.") 

428 if self.doWriteScoreExp and not self.useScoreImageDetection: 

429 raise ValueError( 

430 "doWriteScoreExp=True and useScoreImageDetection=False " 

431 "Score image is not calculated. " 

432 "AL subtraction calculates either the regular difference image or the score image.") 

433 if self.doAddMetrics and not self.doSubtract: 

434 raise ValueError("Subtraction must be enabled for kernel metrics calculation.") 

435 if self.useGaussianForPreConvolution and not self.useScoreImageDetection: 

436 raise ValueError( 

437 "useGaussianForPreConvolution=True and useScoreImageDetection=False " 

438 "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.") 

439 

440 

441@deprecated(reason="This Task has been replaced with lsst.ip.diffim.subtractImages" 

442 " and lsst.ip.diffim.detectAndMeasure. Will be removed after v25.", 

443 version="v24.0", category=FutureWarning) 

444class ImageDifferenceTask(pipeBase.PipelineTask): 

445 """Subtract an image from a template and measure the result. 

446 

447 Parameters 

448 ---------- 

449 butler : `lsst.daf.butler.Butler` or `None`, optional 

450 Butler object to use in constructing reference object loaders. 

451 **kwargs 

452 Additional keyword arguments. 

453 """ 

454 ConfigClass = ImageDifferenceConfig 

455 _DefaultName = "imageDifference" 

456 

457 def __init__(self, butler=None, **kwargs): 

458 super().__init__(**kwargs) 

459 self.makeSubtask("getTemplate") 

460 

461 self.makeSubtask("subtract") 

462 

463 if self.config.subtract.name == 'al' and self.config.doDecorrelation: 

464 self.makeSubtask("decorrelate") 

465 

466 if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance: 

467 self.makeSubtask("scaleVariance") 

468 

469 if self.config.doUseRegister: 

470 self.makeSubtask("register") 

471 self.schema = afwTable.SourceTable.makeMinimalSchema() 

472 

473 if self.config.doSelectSources: 

474 self.makeSubtask("sourceSelector") 

475 if self.config.kernelSourcesFromRef: 

476 self.makeSubtask('refObjLoader', butler=butler) 

477 self.makeSubtask("astrometer", refObjLoader=self.refObjLoader) 

478 

479 self.algMetadata = dafBase.PropertyList() 

480 if self.config.doDetection: 

481 self.makeSubtask("detection", schema=self.schema) 

482 if self.config.doMeasurement: 

483 self.makeSubtask("measurement", schema=self.schema, 

484 algMetadata=self.algMetadata) 

485 if self.config.doApCorr: 

486 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

487 if self.config.doForcedMeasurement: 

488 self.schema.addField( 

489 "ip_diffim_forced_PsfFlux_instFlux", "D", 

490 "Forced PSF flux measured on the direct image.", 

491 units="count") 

492 self.schema.addField( 

493 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

494 "Forced PSF flux error measured on the direct image.", 

495 units="count") 

496 self.schema.addField( 

497 "ip_diffim_forced_PsfFlux_area", "F", 

498 "Forced PSF flux effective area of PSF.", 

499 units="pixel") 

500 self.schema.addField( 

501 "ip_diffim_forced_PsfFlux_flag", "Flag", 

502 "Forced PSF flux general failure flag.") 

503 self.schema.addField( 

504 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

505 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

506 self.schema.addField( 

507 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

508 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

509 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

510 if self.config.doMatchSources: 

511 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

512 self.schema.addField("srcMatchId", "L", "unique id of source match") 

513 if self.config.doSkySources: 

514 self.makeSubtask("skySources") 

515 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

516 

517 # initialize InitOutputs 

518 self.outputSchema = afwTable.SourceCatalog(self.schema) 

519 self.outputSchema.getTable().setMetadata(self.algMetadata) 

520 

521 @staticmethod 

522 def makeIdFactory(expId, expBits): 

523 """Create IdFactory instance for unique 64 bit diaSource id-s. 

524 

525 Parameters 

526 ---------- 

527 expId : `int` 

528 Exposure ID. 

529 

530 expBits : `int` 

531 Number of used bits in ``expId``. 

532 

533 Returns 

534 ------- 

535 idFactory : `lsst.afw.table.IdFactory` 

536 Generator object to assign ids to detected sources in the difference image. 

537 

538 Notes 

539 ----- 

540 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

541 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

542 low value end of the integer. 

543 """ 

544 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

545 

546 @lsst.utils.inheritDoc(pipeBase.PipelineTask) 

547 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

548 inputRefs: pipeBase.InputQuantizedConnection, 

549 outputRefs: pipeBase.OutputQuantizedConnection): 

550 inputs = butlerQC.get(inputRefs) 

551 self.log.info("Processing %s", butlerQC.quantum.dataId) 

552 

553 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None) 

554 exposure = self.prepareCalibratedExposure( 

555 inputs["exposure"], 

556 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog 

557 ) 

558 

559 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

560 returnMaxBits=True) 

561 idFactory = self.makeIdFactory(expId=expId, expBits=expBits) 

562 if self.config.coaddName == 'dcr': 

563 templateExposures = inputRefs.dcrCoadds 

564 else: 

565 templateExposures = inputRefs.coaddExposures 

566 templateStruct = self.getTemplate.runQuantum( 

567 exposure, butlerQC, inputRefs.skyMap, templateExposures 

568 ) 

569 

570 self.checkTemplateIsSufficient(templateStruct.exposure) 

571 

572 outputs = self.run(exposure=exposure, 

573 templateExposure=templateStruct.exposure, 

574 idFactory=idFactory) 

575 # Consistency with runDataref gen2 handling 

576 if outputs.diaSources is None: 

577 del outputs.diaSources 

578 butlerQC.put(outputs, outputRefs) 

579 

580 def prepareCalibratedExposure(self, exposure, finalizedPsfApCorrCatalog=None): 

581 """Prepare a calibrated exposure and apply finalized psf if so configured. 

582 

583 Parameters 

584 ---------- 

585 exposure : `lsst.afw.image.exposure.Exposure` 

586 Input exposure to adjust calibrations. 

587 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

588 Exposure catalog with finalized psf models and aperture correction 

589 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses 

590 the detector id for the catalog id, sorted on id for fast lookup. 

591 

592 Returns 

593 ------- 

594 exposure : `lsst.afw.image.exposure.Exposure` 

595 Exposure with adjusted calibrations. 

596 """ 

597 detectorId = exposure.getInfo().getDetector().getId() 

598 

599 if finalizedPsfApCorrCatalog is not None: 

600 row = finalizedPsfApCorrCatalog.find(detectorId) 

601 if row is None: 

602 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; " 

603 "Using original psf.", detectorId) 

604 else: 

605 psf = row.getPsf() 

606 apCorrMap = row.getApCorrMap() 

607 if psf is None or apCorrMap is None: 

608 self.log.warning("Detector id %s has None for psf/apCorrMap in " 

609 "finalizedPsfApCorrCatalog; Using original psf.", detectorId) 

610 else: 

611 exposure.setPsf(psf) 

612 exposure.info.setApCorrMap(apCorrMap) 

613 

614 return exposure 

615 

616 @timeMethod 

617 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None, 

618 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None): 

619 """PSF matches, subtract two images and perform detection on the difference image. 

620 

621 Parameters 

622 ---------- 

623 exposure : `lsst.afw.image.ExposureF`, optional 

624 The science exposure, the minuend in the image subtraction. 

625 Can be None only if ``config.doSubtract==False``. 

626 selectSources : `lsst.afw.table.SourceCatalog`, optional 

627 Identified sources on the science exposure. This catalog is used to 

628 select sources in order to perform the AL PSF matching on stamp images 

629 around them. The selection steps depend on config options and whether 

630 ``templateSources`` and ``matchingSources`` specified. 

631 templateExposure : `lsst.afw.image.ExposureF`, optional 

632 The template to be subtracted from ``exposure`` in the image subtraction. 

633 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``. 

634 The template exposure should cover the same sky area as the science exposure. 

635 It is either a stich of patches of a coadd skymap image or a calexp 

636 of the same pointing as the science exposure. Can be None only 

637 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None. 

638 templateSources : `lsst.afw.table.SourceCatalog`, optional 

639 Identified sources on the template exposure. 

640 idFactory : `lsst.afw.table.IdFactory` 

641 Generator object to assign ids to detected sources in the difference image. 

642 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional 

643 Background exposure to be added back to the science exposure 

644 if ``config.doAddCalexpBackground==True``. 

645 subtractedExposure : `lsst.afw.image.ExposureF`, optional 

646 If ``config.doSubtract==False`` and ``config.doDetection==True``, 

647 performs the post subtraction source detection only on this exposure. 

648 Otherwise should be None. 

649 

650 Returns 

651 ------- 

652 results : `lsst.pipe.base.Struct` 

653 Results as a struct with attributes: 

654 

655 ``subtractedExposure`` 

656 Difference image (`lsst.afw.image.ExposureF`). 

657 ``scoreExposure`` 

658 The zogy score exposure, if calculated (`lsst.afw.image.ExposureF` or `None`). 

659 ``matchedExposure`` 

660 The matched PSF exposure (`lsst.afw.image.ExposureF`). 

661 ``subtractRes`` 

662 The returned result structure of the ImagePsfMatchTask subtask (`lsst.pipe.base.Struct`). 

663 ``diaSources`` 

664 The catalog of detected sources (`lsst.afw.table.SourceCatalog`). 

665 ``selectSources`` 

666 The input source catalog with optionally added Qa information 

667 (`lsst.afw.table.SourceCatalog`). 

668 

669 Notes 

670 ----- 

671 The following major steps are included: 

672 

673 - warp template coadd to match WCS of image 

674 - PSF match image to warped template 

675 - subtract image from PSF-matched, warped template 

676 - detect sources 

677 - measure sources 

678 

679 For details about the image subtraction configuration modes 

680 see `lsst.ip.diffim`. 

681 """ 

682 subtractRes = None 

683 controlSources = None 

684 subtractedExposure = None 

685 scoreExposure = None 

686 diaSources = None 

687 kernelSources = None 

688 # We'll clone exposure if modified but will still need the original 

689 exposureOrig = exposure 

690 

691 if self.config.doAddCalexpBackground: 

692 mi = exposure.getMaskedImage() 

693 mi += calexpBackgroundExposure.getImage() 

694 

695 if not exposure.hasPsf(): 

696 raise pipeBase.TaskError("Exposure has no psf") 

697 sciencePsf = exposure.getPsf() 

698 

699 if self.config.doSubtract: 

700 if self.config.doScaleTemplateVariance: 

701 self.log.info("Rescaling template variance") 

702 templateVarFactor = self.scaleVariance.run( 

703 templateExposure.getMaskedImage()) 

704 self.log.info("Template variance scaling factor: %.2f", templateVarFactor) 

705 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor) 

706 self.metadata.add("psfMatchingAlgorithm", self.config.subtract.name) 

707 

708 if self.config.subtract.name == 'zogy': 

709 subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True) 

710 scoreExposure = subtractRes.scoreExp 

711 subtractedExposure = subtractRes.diffExp 

712 subtractRes.subtractedExposure = subtractedExposure 

713 subtractRes.matchedExposure = None 

714 

715 elif self.config.subtract.name == 'al': 

716 # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution 

717 # Just need a rough estimate; average positions are fine 

718 sciAvgPos = sciencePsf.getAveragePosition() 

719 scienceSigmaOrig = sciencePsf.computeShape(sciAvgPos).getDeterminantRadius() 

720 

721 templatePsf = templateExposure.getPsf() 

722 templateAvgPos = templatePsf.getAveragePosition() 

723 templateSigma = templatePsf.computeShape(templateAvgPos).getDeterminantRadius() 

724 

725 # if requested, convolve the science exposure with its PSF 

726 # (properly, this should be a cross-correlation, but our code does not yet support that) 

727 # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, 

728 # else sigma of original science exposure 

729 # TODO: DM-22762 This functional block should be moved into its own method 

730 preConvPsf = None 

731 if self.config.useScoreImageDetection: 

732 self.log.warning("AL likelihood image: pre-convolution of PSF is not implemented.") 

733 convControl = afwMath.ConvolutionControl() 

734 # cannot convolve in place, so need a new image anyway 

735 srcMI = exposure.maskedImage 

736 exposure = exposure.clone() # New deep copy 

737 srcPsf = sciencePsf 

738 if self.config.useGaussianForPreConvolution: 

739 self.log.info( 

740 "AL likelihood image: Using Gaussian (sigma=%.2f) PSF estimation " 

741 "for science image pre-convolution", scienceSigmaOrig) 

742 # convolve with a simplified PSF model: a double Gaussian 

743 kWidth, kHeight = sciencePsf.getLocalKernel( 

744 sciencePsf.getAveragePosition() 

745 ).getDimensions() 

746 preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) 

747 else: 

748 # convolve with science exposure's PSF model 

749 self.log.info( 

750 "AL likelihood image: Using the science image PSF for pre-convolution.") 

751 preConvPsf = srcPsf 

752 afwMath.convolve( 

753 exposure.maskedImage, 

754 srcMI, 

755 preConvPsf.getLocalKernel(preConvPsf.getAveragePosition()), 

756 convControl 

757 ) 

758 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2) 

759 else: 

760 scienceSigmaPost = scienceSigmaOrig 

761 

762 # If requested, find and select sources from the image 

763 # else, AL subtraction will do its own source detection 

764 # TODO: DM-22762 This functional block should be moved into its own method 

765 if self.config.doSelectSources: 

766 if selectSources is None: 

767 self.log.warning("Src product does not exist; running detection, measurement," 

768 " selection") 

769 # Run own detection and measurement; necessary in nightly processing 

770 selectSources = self.subtract.getSelectSources( 

771 exposure, 

772 sigma=scienceSigmaPost, 

773 doSmooth=not self.config.useScoreImageDetection, 

774 idFactory=idFactory, 

775 ) 

776 

777 if self.config.doAddMetrics: 

778 # Number of basis functions 

779 

780 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active, 

781 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma, 

782 targetFwhmPix=templateSigma*FwhmPerSigma)) 

783 # Modify the schema of all Sources 

784 # DEPRECATED: This is a data dependent (nparam) output product schema 

785 # outside the task constructor. 

786 # NOTE: The pre-determination of nparam at this point 

787 # may be incorrect as the template psf is warped later in 

788 # ImagePsfMatchTask.matchExposures() 

789 kcQa = KernelCandidateQa(nparam) 

790 selectSources = kcQa.addToSchema(selectSources) 

791 if self.config.kernelSourcesFromRef: 

792 # match exposure sources to reference catalog 

793 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources) 

794 matches = astromRet.matches 

795 elif templateSources: 

796 # match exposure sources to template sources 

797 mc = afwTable.MatchControl() 

798 mc.findOnlyClosest = False 

799 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds, 

800 mc) 

801 else: 

802 raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," 

803 "but template sources not available. Cannot match science " 

804 "sources with template sources. Run process* on data from " 

805 "which templates are built.") 

806 

807 kernelSources = self.sourceSelector.run(selectSources, exposure=exposure, 

808 matches=matches).sourceCat 

809 random.shuffle(kernelSources, random.random) 

810 controlSources = kernelSources[::self.config.controlStepSize] 

811 kernelSources = [k for i, k in enumerate(kernelSources) 

812 if i % self.config.controlStepSize] 

813 

814 if self.config.doSelectDcrCatalog: 

815 redSelector = DiaCatalogSourceSelectorTask( 

816 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, 

817 grMax=99.999)) 

818 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat 

819 controlSources.extend(redSources) 

820 

821 blueSelector = DiaCatalogSourceSelectorTask( 

822 DiaCatalogSourceSelectorConfig(grMin=-99.999, 

823 grMax=self.sourceSelector.config.grMin)) 

824 blueSources = blueSelector.selectStars(exposure, selectSources, 

825 matches=matches).starCat 

826 controlSources.extend(blueSources) 

827 

828 if self.config.doSelectVariableCatalog: 

829 varSelector = DiaCatalogSourceSelectorTask( 

830 DiaCatalogSourceSelectorConfig(includeVariable=True)) 

831 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat 

832 controlSources.extend(varSources) 

833 

834 self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)", 

835 len(kernelSources), len(selectSources), len(controlSources)) 

836 

837 allresids = {} 

838 # TODO: DM-22762 This functional block should be moved into its own method 

839 if self.config.doUseRegister: 

840 self.log.info("Registering images") 

841 

842 if templateSources is None: 

843 # Run detection on the template, which is 

844 # temporarily background-subtracted 

845 # sigma of PSF of template image before warping 

846 templateSources = self.subtract.getSelectSources( 

847 templateExposure, 

848 sigma=templateSigma, 

849 doSmooth=True, 

850 idFactory=idFactory 

851 ) 

852 

853 # Third step: we need to fit the relative astrometry. 

854 # 

855 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources) 

856 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs, 

857 exposure.getWcs(), exposure.getBBox()) 

858 templateExposure = warpedExp 

859 

860 # Create debugging outputs on the astrometric 

861 # residuals as a function of position. Persistence 

862 # not yet implemented; expected on (I believe) #2636. 

863 if self.config.doDebugRegister: 

864 # Grab matches to reference catalog 

865 srcToMatch = {x.second.getId(): x.first for x in matches} 

866 

867 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey() 

868 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey() 

869 sids = [m.first.getId() for m in wcsResults.matches] 

870 positions = [m.first.get(refCoordKey) for m in wcsResults.matches] 

871 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky( 

872 m.second.get(inCentroidKey))) for m in wcsResults.matches] 

873 allresids = dict(zip(sids, zip(positions, residuals))) 

874 

875 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset( 

876 wcsResults.wcs.pixelToSky( 

877 m.second.get(inCentroidKey))) for m in wcsResults.matches] 

878 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) 

879 + 2.5*numpy.log10(srcToMatch[x].get("r")) 

880 for x in sids if x in srcToMatch.keys()]) 

881 dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals) 

882 if s in srcToMatch.keys()]) 

883 dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals) 

884 if s in srcToMatch.keys()]) 

885 idx1 = numpy.where(colors < self.sourceSelector.config.grMin) 

886 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) 

887 & (colors <= self.sourceSelector.config.grMax)) 

888 idx3 = numpy.where(colors > self.sourceSelector.config.grMax) 

889 rms1Long = IqrToSigma*( 

890 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25))) 

891 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) 

892 - numpy.percentile(dlat[idx1], 25)) 

893 rms2Long = IqrToSigma*( 

894 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25))) 

895 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) 

896 - numpy.percentile(dlat[idx2], 25)) 

897 rms3Long = IqrToSigma*( 

898 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25))) 

899 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) 

900 - numpy.percentile(dlat[idx3], 25)) 

901 self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f", 

902 numpy.median(dlong[idx1]), rms1Long, 

903 numpy.median(dlat[idx1]), rms1Lat) 

904 self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f", 

905 numpy.median(dlong[idx2]), rms2Long, 

906 numpy.median(dlat[idx2]), rms2Lat) 

907 self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f", 

908 numpy.median(dlong[idx3]), rms3Long, 

909 numpy.median(dlat[idx3]), rms3Lat) 

910 

911 self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1])) 

912 self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2])) 

913 self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3])) 

914 self.metadata.add("RegisterBlueLongOffsetStd", rms1Long) 

915 self.metadata.add("RegisterGreenLongOffsetStd", rms2Long) 

916 self.metadata.add("RegisterRedLongOffsetStd", rms3Long) 

917 

918 self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1])) 

919 self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2])) 

920 self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3])) 

921 self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat) 

922 self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat) 

923 self.metadata.add("RegisterRedLatOffsetStd", rms3Lat) 

924 

925 # warp template exposure to match exposure, 

926 # PSF match template exposure to exposure, 

927 # then return the difference 

928 

929 # Return warped template... Construct sourceKernelCand list after subtract 

930 self.log.info("Subtracting images") 

931 subtractRes = self.subtract.subtractExposures( 

932 templateExposure=templateExposure, 

933 scienceExposure=exposure, 

934 candidateList=kernelSources, 

935 convolveTemplate=self.config.convolveTemplate, 

936 doWarping=not self.config.doUseRegister 

937 ) 

938 if self.config.useScoreImageDetection: 

939 scoreExposure = subtractRes.subtractedExposure 

940 else: 

941 subtractedExposure = subtractRes.subtractedExposure 

942 

943 if self.config.doDetection: 

944 self.log.info("Computing diffim PSF") 

945 

946 # Get Psf from the appropriate input image if it doesn't exist 

947 if subtractedExposure is not None and not subtractedExposure.hasPsf(): 

948 if self.config.convolveTemplate: 

949 subtractedExposure.setPsf(exposure.getPsf()) 

950 else: 

951 subtractedExposure.setPsf(templateExposure.getPsf()) 

952 

953 # If doSubtract is False, then subtractedExposure was fetched from disk (above), 

954 # thus it may have already been decorrelated. Thus, we do not decorrelate if 

955 # doSubtract is False. 

956 

957 # NOTE: At this point doSubtract == True 

958 if self.config.doDecorrelation and self.config.doSubtract: 

959 preConvKernel = None 

960 if self.config.useGaussianForPreConvolution: 

961 preConvKernel = preConvPsf.getLocalKernel(preConvPsf.getAveragePosition()) 

962 if self.config.useScoreImageDetection: 

963 scoreExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure, 

964 scoreExposure, 

965 subtractRes.psfMatchingKernel, 

966 spatiallyVarying=self.config.doSpatiallyVarying, 

967 preConvKernel=preConvKernel, 

968 templateMatched=True, 

969 preConvMode=True).correctedExposure 

970 # Note that the subtracted exposure is always decorrelated, 

971 # even if the score image is used for detection 

972 subtractedExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure, 

973 subtractedExposure, 

974 subtractRes.psfMatchingKernel, 

975 spatiallyVarying=self.config.doSpatiallyVarying, 

976 preConvKernel=None, 

977 templateMatched=self.config.convolveTemplate, 

978 preConvMode=False).correctedExposure 

979 # END (if subtractAlgorithm == 'AL') 

980 # END (if self.config.doSubtract) 

981 if self.config.doDetection: 

982 self.log.info("Running diaSource detection") 

983 

984 # subtractedExposure - reserved for task return value 

985 # in zogy, it is always the proper difference image 

986 # in AL, it may be (yet) pre-convolved and/or decorrelated 

987 # 

988 # detectionExposure - controls which exposure to use for detection 

989 # in-place modifications will appear in task return 

990 if self.config.useScoreImageDetection: 

991 # zogy with score image detection enabled 

992 self.log.info("Detection, diffim rescaling and measurements are " 

993 "on AL likelihood or Zogy score image.") 

994 detectionExposure = scoreExposure 

995 else: 

996 # AL or zogy with no score image detection 

997 detectionExposure = subtractedExposure 

998 

999 # Rescale difference image variance plane 

1000 if self.config.doScaleDiffimVariance: 

1001 self.log.info("Rescaling diffim variance") 

1002 diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage()) 

1003 self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor) 

1004 self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor) 

1005 

1006 # Erase existing detection mask planes 

1007 mask = detectionExposure.getMaskedImage().getMask() 

1008 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

1009 

1010 table = afwTable.SourceTable.make(self.schema, idFactory) 

1011 table.setMetadata(self.algMetadata) 

1012 results = self.detection.run( 

1013 table=table, 

1014 exposure=detectionExposure, 

1015 doSmooth=not self.config.useScoreImageDetection 

1016 ) 

1017 

1018 if self.config.doMerge: 

1019 fpSet = results.fpSets.positive 

1020 fpSet.merge(results.fpSets.negative, self.config.growFootprint, 

1021 self.config.growFootprint, False) 

1022 diaSources = afwTable.SourceCatalog(table) 

1023 fpSet.makeSources(diaSources) 

1024 self.log.info("Merging detections into %d sources", len(diaSources)) 

1025 else: 

1026 diaSources = results.sources 

1027 # Inject skySources before measurement. 

1028 if self.config.doSkySources: 

1029 skySourceFootprints = self.skySources.run( 

1030 mask=detectionExposure.mask, 

1031 seed=detectionExposure.info.id) 

1032 if skySourceFootprints: 

1033 for foot in skySourceFootprints: 

1034 s = diaSources.addNew() 

1035 s.setFootprint(foot) 

1036 s.set(self.skySourceKey, True) 

1037 

1038 if self.config.doMeasurement: 

1039 newDipoleFitting = self.config.doDipoleFitting 

1040 self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting) 

1041 if not newDipoleFitting: 

1042 # Just fit dipole in diffim 

1043 self.measurement.run(diaSources, detectionExposure) 

1044 else: 

1045 # Use (matched) template and science image (if avail.) to constrain dipole fitting 

1046 if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict(): 

1047 self.measurement.run(diaSources, detectionExposure, exposure, 

1048 subtractRes.matchedExposure) 

1049 else: 

1050 self.measurement.run(diaSources, detectionExposure, exposure) 

1051 if self.config.doApCorr: 

1052 self.applyApCorr.run( 

1053 catalog=diaSources, 

1054 apCorrMap=detectionExposure.getInfo().getApCorrMap() 

1055 ) 

1056 

1057 if self.config.doForcedMeasurement: 

1058 # Run forced psf photometry on the PVI at the diaSource locations. 

1059 # Copy the measured flux and error into the diaSource. 

1060 forcedSources = self.forcedMeasurement.generateMeasCat( 

1061 exposure, diaSources, detectionExposure.getWcs()) 

1062 self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs()) 

1063 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

1064 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

1065 "ip_diffim_forced_PsfFlux_instFlux", True) 

1066 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

1067 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

1068 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

1069 "ip_diffim_forced_PsfFlux_area", True) 

1070 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

1071 "ip_diffim_forced_PsfFlux_flag", True) 

1072 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

1073 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

1074 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

1075 "ip_diffim_forced_PsfFlux_flag_edge", True) 

1076 for diaSource, forcedSource in zip(diaSources, forcedSources): 

1077 diaSource.assign(forcedSource, mapper) 

1078 

1079 # Match with the calexp sources if possible 

1080 if self.config.doMatchSources: 

1081 if selectSources is not None: 

1082 # Create key,val pair where key=diaSourceId and val=sourceId 

1083 matchRadAsec = self.config.diaSourceMatchRadius 

1084 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds() 

1085 

1086 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel) 

1087 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for 

1088 srcMatch in srcMatches]) 

1089 self.log.info("Matched %d / %d diaSources to sources", 

1090 len(srcMatchDict), len(diaSources)) 

1091 else: 

1092 self.log.warning("Src product does not exist; cannot match with diaSources") 

1093 srcMatchDict = {} 

1094 

1095 # Create key,val pair where key=diaSourceId and val=refId 

1096 refAstromConfig = AstrometryConfig() 

1097 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec 

1098 refAstrometer = AstrometryTask(self.refObjLoader, config=refAstromConfig) 

1099 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources) 

1100 refMatches = astromRet.matches 

1101 if refMatches is None: 

1102 self.log.warning("No diaSource matches with reference catalog") 

1103 refMatchDict = {} 

1104 else: 

1105 self.log.info("Matched %d / %d diaSources to reference catalog", 

1106 len(refMatches), len(diaSources)) 

1107 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for 

1108 refMatch in refMatches]) 

1109 

1110 # Assign source Ids 

1111 for diaSource in diaSources: 

1112 sid = diaSource.getId() 

1113 if sid in srcMatchDict: 

1114 diaSource.set("srcMatchId", srcMatchDict[sid]) 

1115 if sid in refMatchDict: 

1116 diaSource.set("refMatchId", refMatchDict[sid]) 

1117 

1118 if self.config.doAddMetrics and self.config.doSelectSources: 

1119 self.log.info("Evaluating metrics and control sample") 

1120 

1121 kernelCandList = [] 

1122 for cell in subtractRes.kernelCellSet.getCellList(): 

1123 for cand in cell.begin(False): # include bad candidates 

1124 kernelCandList.append(cand) 

1125 

1126 # Get basis list to build control sample kernels 

1127 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList() 

1128 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters()) 

1129 

1130 controlCandList = ( 

1131 diffimTools.sourceTableToCandidateList(controlSources, 

1132 subtractRes.warpedExposure, exposure, 

1133 self.config.subtract.kernel.active, 

1134 self.config.subtract.kernel.active.detectionConfig, 

1135 self.log, doBuild=True, basisList=basisList)) 

1136 

1137 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel, 

1138 subtractRes.backgroundModel, dof=nparam) 

1139 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel, 

1140 subtractRes.backgroundModel) 

1141 

1142 if self.config.doDetection: 

1143 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources) 

1144 else: 

1145 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids) 

1146 

1147 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) 

1148 return pipeBase.Struct( 

1149 subtractedExposure=subtractedExposure, 

1150 scoreExposure=scoreExposure, 

1151 warpedExposure=subtractRes.warpedExposure, 

1152 matchedExposure=subtractRes.matchedExposure, 

1153 subtractRes=subtractRes, 

1154 diaSources=diaSources, 

1155 selectSources=selectSources 

1156 ) 

1157 

1158 def fitAstrometry(self, templateSources, templateExposure, selectSources): 

1159 """Fit the relative astrometry between templateSources and selectSources 

1160 

1161 Notes 

1162 ----- 

1163 TODO: Remove this method. It originally fit a new WCS to the template before calling register.run 

1164 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed. 

1165 It remains because a subtask overrides it. 

1166 """ 

1167 results = self.register.run(templateSources, templateExposure.getWcs(), 

1168 templateExposure.getBBox(), selectSources) 

1169 return results 

1170 

1171 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources): 

1172 """Make debug plots and displays. 

1173 

1174 Parameters 

1175 ---------- 

1176 exposure : `lsst.afw.image.exposure.Exposure` 

1177 Input exposure. 

1178 subtractRes : `lsst.pipe.base.Struct` 

1179 Returned result structure of the ImagePsfMatchTask subtask. 

1180 selectSources : `lsst.afw.table.SourceCatalog` 

1181 Input source catalog. 

1182 kernelSources : `lsst.afw.table.SourceCatalog` 

1183 unknown 

1184 diaSources : `lsst.afw.table.SourceCatalog` 

1185 The catalog of detected sources. 

1186 

1187 Notes 

1188 ----- 

1189 TODO: Test and update for current debug display and slot names. 

1190 """ 

1191 import lsstDebug 

1192 display = lsstDebug.Info(__name__).display 

1193 showSubtracted = lsstDebug.Info(__name__).showSubtracted 

1194 showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals 

1195 showDiaSources = lsstDebug.Info(__name__).showDiaSources 

1196 showDipoles = lsstDebug.Info(__name__).showDipoles 

1197 maskTransparency = lsstDebug.Info(__name__).maskTransparency 

1198 if display: 

1199 disp = afwDisplay.getDisplay(frame=lsstDebug.frame) 

1200 if not maskTransparency: 

1201 maskTransparency = 0 

1202 disp.setMaskTransparency(maskTransparency) 

1203 

1204 if display and showSubtracted: 

1205 disp.mtv(subtractRes.subtractedExposure, title="Subtracted image") 

1206 mi = subtractRes.subtractedExposure.getMaskedImage() 

1207 x0, y0 = mi.getX0(), mi.getY0() 

1208 with disp.Buffering(): 

1209 for s in diaSources: 

1210 x, y = s.getX() - x0, s.getY() - y0 

1211 ctype = "red" if s.get("flags_negative") else "yellow" 

1212 if (s.get("base_PixelFlags_flag_interpolatedCenter") 

1213 or s.get("base_PixelFlags_flag_saturatedCenter") 

1214 or s.get("base_PixelFlags_flag_crCenter")): 

1215 ptype = "x" 

1216 elif (s.get("base_PixelFlags_flag_interpolated") 

1217 or s.get("base_PixelFlags_flag_saturated") 

1218 or s.get("base_PixelFlags_flag_cr")): 

1219 ptype = "+" 

1220 else: 

1221 ptype = "o" 

1222 disp.dot(ptype, x, y, size=4, ctype=ctype) 

1223 lsstDebug.frame += 1 

1224 

1225 if display and showPixelResiduals and selectSources: 

1226 nonKernelSources = [] 

1227 for source in selectSources: 

1228 if source not in kernelSources: 

1229 nonKernelSources.append(source) 

1230 

1231 diUtils.plotPixelResiduals(exposure, 

1232 subtractRes.warpedExposure, 

1233 subtractRes.subtractedExposure, 

1234 subtractRes.kernelCellSet, 

1235 subtractRes.psfMatchingKernel, 

1236 subtractRes.backgroundModel, 

1237 nonKernelSources, 

1238 self.subtract.config.kernel.active.detectionConfig, 

1239 origVariance=False) 

1240 diUtils.plotPixelResiduals(exposure, 

1241 subtractRes.warpedExposure, 

1242 subtractRes.subtractedExposure, 

1243 subtractRes.kernelCellSet, 

1244 subtractRes.psfMatchingKernel, 

1245 subtractRes.backgroundModel, 

1246 nonKernelSources, 

1247 self.subtract.config.kernel.active.detectionConfig, 

1248 origVariance=True) 

1249 if display and showDiaSources: 

1250 flagChecker = SourceFlagChecker(diaSources) 

1251 isFlagged = [flagChecker(x) for x in diaSources] 

1252 isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources] 

1253 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole, 

1254 frame=lsstDebug.frame) 

1255 lsstDebug.frame += 1 

1256 

1257 if display and showDipoles: 

1258 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources, 

1259 frame=lsstDebug.frame) 

1260 lsstDebug.frame += 1 

1261 

1262 def checkTemplateIsSufficient(self, templateExposure): 

1263 """Raise NoWorkFound if template coverage < requiredTemplateFraction. 

1264 

1265 Parameters 

1266 ---------- 

1267 templateExposure : `lsst.afw.image.ExposureF` 

1268 The template exposure to check. 

1269 

1270 Raises 

1271 ------ 

1272 NoWorkFound 

1273 Raised if fraction of good pixels, defined as not having NO_DATA 

1274 set, is less then the configured requiredTemplateFraction. 

1275 """ 

1276 # Count the number of pixels with the NO_DATA mask bit set 

1277 # counting NaN pixels is insufficient because pixels without data are often intepolated over) 

1278 pixNoData = numpy.count_nonzero(templateExposure.mask.array 

1279 & templateExposure.mask.getPlaneBitMask('NO_DATA')) 

1280 pixGood = templateExposure.getBBox().getArea() - pixNoData 

1281 self.log.info("template has %d good pixels (%.1f%%)", pixGood, 

1282 100*pixGood/templateExposure.getBBox().getArea()) 

1283 

1284 if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction: 

1285 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. " 

1286 "To force subtraction, set config requiredTemplateFraction=0." % ( 

1287 100*pixGood/templateExposure.getBBox().getArea(), 

1288 100*self.config.requiredTemplateFraction)) 

1289 raise pipeBase.NoWorkFound(message) 

1290 

1291 

1292class ImageDifferenceFromTemplateConnections(ImageDifferenceTaskConnections, 

1293 defaultTemplates={"coaddName": "goodSeeing"} 

1294 ): 

1295 inputTemplate = pipeBase.connectionTypes.Input( 

1296 doc=("Warped template produced by GetMultiTractCoaddTemplate"), 

1297 dimensions=("instrument", "visit", "detector"), 

1298 storageClass="ExposureF", 

1299 name="{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}", 

1300 ) 

1301 

1302 def __init__(self, *, config=None): 

1303 super().__init__(config=config) 

1304 # ImageDifferenceConnections will have removed one of these. 

1305 # Make sure they're both gone, because no coadds are needed. 

1306 if "coaddExposures" in self.inputs: 

1307 self.inputs.remove("coaddExposures") 

1308 if "dcrCoadds" in self.inputs: 

1309 self.inputs.remove("dcrCoadds") 

1310 

1311 

1312class ImageDifferenceFromTemplateConfig(ImageDifferenceConfig, 

1313 pipelineConnections=ImageDifferenceFromTemplateConnections): 

1314 pass 

1315 

1316 

1317class ImageDifferenceFromTemplateTask(ImageDifferenceTask): 

1318 ConfigClass = ImageDifferenceFromTemplateConfig 

1319 _DefaultName = "imageDifference" 

1320 

1321 @lsst.utils.inheritDoc(pipeBase.PipelineTask) 

1322 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1323 inputs = butlerQC.get(inputRefs) 

1324 self.log.info("Processing %s", butlerQC.quantum.dataId) 

1325 self.checkTemplateIsSufficient(inputs['inputTemplate']) 

1326 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

1327 returnMaxBits=True) 

1328 idFactory = self.makeIdFactory(expId=expId, expBits=expBits) 

1329 

1330 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None) 

1331 exposure = self.prepareCalibratedExposure( 

1332 inputs["exposure"], 

1333 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog 

1334 ) 

1335 

1336 outputs = self.run(exposure=exposure, 

1337 templateExposure=inputs['inputTemplate'], 

1338 idFactory=idFactory) 

1339 

1340 # Consistency with runDataref gen2 handling 

1341 if outputs.diaSources is None: 

1342 del outputs.diaSources 

1343 butlerQC.put(outputs, outputRefs)