Coverage for python/lsst/pipe/tasks/imageDifference.py: 16%

551 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-12 01:26 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import random 

24import numpy 

25 

26import lsst.utils 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29import lsst.daf.base as dafBase 

30import lsst.geom as geom 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.meas.extensions.trailedSources # noqa: F401 

34from lsst.meas.algorithms import (SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask, 

35 LoadReferenceObjectsConfig, SkyObjectsTask, 

36 ScaleVarianceTask) 

37from lsst.meas.astrom import AstrometryConfig, AstrometryTask 

38from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask 

39from lsst.pipe.tasks.registerImage import RegisterTask 

40from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList, 

41 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, 

42 GetCoaddAsTemplateTask, DipoleFitTask, 

43 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry) 

44import lsst.ip.diffim.diffimTools as diffimTools 

45import lsst.ip.diffim.utils as diUtils 

46import lsst.afw.display as afwDisplay 

47from lsst.skymap import BaseSkyMap 

48from lsst.obs.base import ExposureIdInfo 

49from lsst.utils.timer import timeMethod 

50 

51from deprecated.sphinx import deprecated 

52 

53__all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"] 

54FwhmPerSigma = 2*math.sqrt(2*math.log(2)) 

55IqrToSigma = 0.741 

56 

57 

58class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections, 

59 dimensions=("instrument", "visit", "detector", "skymap"), 

60 defaultTemplates={"coaddName": "deep", 

61 "skyMapName": "deep", 

62 "warpTypeSuffix": "", 

63 "fakesType": ""}): 

64 

65 exposure = pipeBase.connectionTypes.Input( 

66 doc="Input science exposure to subtract from.", 

67 dimensions=("instrument", "visit", "detector"), 

68 storageClass="ExposureF", 

69 name="{fakesType}calexp" 

70 ) 

71 

72 # TODO DM-22953 

73 # kernelSources = pipeBase.connectionTypes.Input( 

74 # doc="Source catalog produced in calibrate task for kernel candidate sources", 

75 # name="src", 

76 # storageClass="SourceCatalog", 

77 # dimensions=("instrument", "visit", "detector"), 

78 # ) 

79 

80 skyMap = pipeBase.connectionTypes.Input( 

81 doc="Input definition of geometry/bbox and projection/wcs for template exposures", 

82 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

83 dimensions=("skymap", ), 

84 storageClass="SkyMap", 

85 ) 

86 coaddExposures = pipeBase.connectionTypes.Input( 

87 doc="Input template to match and subtract from the exposure", 

88 dimensions=("tract", "patch", "skymap", "band"), 

89 storageClass="ExposureF", 

90 name="{fakesType}{coaddName}Coadd{warpTypeSuffix}", 

91 multiple=True, 

92 deferLoad=True 

93 ) 

94 dcrCoadds = pipeBase.connectionTypes.Input( 

95 doc="Input DCR template to match and subtract from the exposure", 

96 name="{fakesType}dcrCoadd{warpTypeSuffix}", 

97 storageClass="ExposureF", 

98 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

99 multiple=True, 

100 deferLoad=True 

101 ) 

102 finalizedPsfApCorrCatalog = pipeBase.connectionTypes.Input( 

103 doc=("Per-visit finalized psf models and aperture correction maps. " 

104 "These catalogs use the detector id for the catalog id, " 

105 "sorted on id for fast lookup."), 

106 name="finalized_psf_ap_corr_catalog", 

107 storageClass="ExposureCatalog", 

108 dimensions=("instrument", "visit"), 

109 ) 

110 outputSchema = pipeBase.connectionTypes.InitOutput( 

111 doc="Schema (as an example catalog) for output DIASource catalog.", 

112 storageClass="SourceCatalog", 

113 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

114 ) 

115 subtractedExposure = pipeBase.connectionTypes.Output( 

116 doc="Output AL difference or Zogy proper difference image", 

117 dimensions=("instrument", "visit", "detector"), 

118 storageClass="ExposureF", 

119 name="{fakesType}{coaddName}Diff_differenceExp", 

120 ) 

121 scoreExposure = pipeBase.connectionTypes.Output( 

122 doc="Output AL likelihood or Zogy score image", 

123 dimensions=("instrument", "visit", "detector"), 

124 storageClass="ExposureF", 

125 name="{fakesType}{coaddName}Diff_scoreExp", 

126 ) 

127 warpedExposure = pipeBase.connectionTypes.Output( 

128 doc="Warped template used to create `subtractedExposure`.", 

129 dimensions=("instrument", "visit", "detector"), 

130 storageClass="ExposureF", 

131 name="{fakesType}{coaddName}Diff_warpedExp", 

132 ) 

133 matchedExposure = pipeBase.connectionTypes.Output( 

134 doc="Warped template used to create `subtractedExposure`.", 

135 dimensions=("instrument", "visit", "detector"), 

136 storageClass="ExposureF", 

137 name="{fakesType}{coaddName}Diff_matchedExp", 

138 ) 

139 diaSources = pipeBase.connectionTypes.Output( 

140 doc="Output detected diaSources on the difference image", 

141 dimensions=("instrument", "visit", "detector"), 

142 storageClass="SourceCatalog", 

143 name="{fakesType}{coaddName}Diff_diaSrc", 

144 ) 

145 

146 def __init__(self, *, config=None): 

147 super().__init__(config=config) 

148 if config.coaddName == 'dcr': 

149 self.inputs.remove("coaddExposures") 

150 else: 

151 self.inputs.remove("dcrCoadds") 

152 if not config.doWriteSubtractedExp: 

153 self.outputs.remove("subtractedExposure") 

154 if not config.doWriteScoreExp: 

155 self.outputs.remove("scoreExposure") 

156 if not config.doWriteWarpedExp: 

157 self.outputs.remove("warpedExposure") 

158 if not config.doWriteMatchedExp: 

159 self.outputs.remove("matchedExposure") 

160 if not config.doWriteSources: 

161 self.outputs.remove("diaSources") 

162 if not config.doApplyFinalizedPsf: 

163 self.inputs.remove("finalizedPsfApCorrCatalog") 

164 

165 # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef) 

166 # Make kernelSources optional 

167 

168 

169class ImageDifferenceConfig(pipeBase.PipelineTaskConfig, 

170 pipelineConnections=ImageDifferenceTaskConnections): 

171 """Config for ImageDifferenceTask 

172 """ 

173 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False, 

174 doc="Add background to calexp before processing it. " 

175 "Useful as ipDiffim does background matching.") 

176 doUseRegister = pexConfig.Field(dtype=bool, default=False, 

177 doc="Re-compute astrometry on the template. " 

178 "Use image-to-image registration to align template with " 

179 "science image (AL only).") 

180 doDebugRegister = pexConfig.Field(dtype=bool, default=False, 

181 doc="Writing debugging data for doUseRegister") 

182 doSelectSources = pexConfig.Field(dtype=bool, default=False, 

183 doc="Select stars to use for kernel fitting (AL only)") 

184 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False, 

185 doc="Select stars of extreme color as part " 

186 "of the control sample (AL only)") 

187 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False, 

188 doc="Select stars that are variable to be part " 

189 "of the control sample (AL only)") 

190 doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?") 

191 doPreConvolve = pexConfig.Field(dtype=bool, default=False, 

192 doc="Not in use. Superseded by useScoreImageDetection.", 

193 deprecated="This option superseded by useScoreImageDetection." 

194 " Will be removed after v22.") 

195 useScoreImageDetection = pexConfig.Field( 

196 dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or " 

197 "the Zogy score image. Use it for source detection (if doDetection=True).") 

198 doWriteScoreExp = pexConfig.Field( 

199 dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?") 

200 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False, 

201 doc="Scale variance of the template before PSF matching") 

202 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True, 

203 doc="Scale variance of the diffim before PSF matching. " 

204 "You may do either this or template variance scaling, " 

205 "or neither. (Doing both is a waste of CPU.)") 

206 useGaussianForPreConvolution = pexConfig.Field( 

207 dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution " 

208 "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)") 

209 doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?") 

210 doDecorrelation = pexConfig.Field(dtype=bool, default=True, 

211 doc="Perform diffim decorrelation to undo pixel correlation due to A&L " 

212 "kernel convolution (AL only)? If True, also update the diffim PSF.") 

213 doMerge = pexConfig.Field(dtype=bool, default=True, 

214 doc="Merge positive and negative diaSources with grow radius " 

215 "set by growFootprint") 

216 doMatchSources = pexConfig.Field(dtype=bool, default=False, 

217 doc="Match diaSources with input calexp sources and ref catalog sources") 

218 doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?") 

219 doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?") 

220 doForcedMeasurement = pexConfig.Field( 

221 dtype=bool, 

222 default=True, 

223 doc="Force photometer diaSource locations on PVI?") 

224 doWriteSubtractedExp = pexConfig.Field( 

225 dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?") 

226 doWriteWarpedExp = pexConfig.Field( 

227 dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?") 

228 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False, 

229 doc="Write warped and PSF-matched template coadd exposure?") 

230 doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?") 

231 doAddMetrics = pexConfig.Field(dtype=bool, default=False, 

232 doc="Add columns to the source table to hold analysis metrics?") 

233 doApplyFinalizedPsf = pexConfig.Field( 

234 doc="Whether to apply finalized psf models and aperture correction map.", 

235 dtype=bool, 

236 default=False, 

237 ) 

238 

239 coaddName = pexConfig.Field( 

240 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

241 dtype=str, 

242 default="deep", 

243 ) 

244 convolveTemplate = pexConfig.Field( 

245 doc="Which image gets convolved (default = template)", 

246 dtype=bool, 

247 default=True 

248 ) 

249 refObjLoader = pexConfig.ConfigField( 

250 dtype=LoadReferenceObjectsConfig, 

251 doc="reference object loader", 

252 ) 

253 astrometer = pexConfig.ConfigurableField( 

254 target=AstrometryTask, 

255 doc="astrometry task; used to match sources to reference objects, but not to fit a WCS", 

256 ) 

257 sourceSelector = pexConfig.ConfigurableField( 

258 target=ObjectSizeStarSelectorTask, 

259 doc="Source selection algorithm", 

260 ) 

261 subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al") 

262 decorrelate = pexConfig.ConfigurableField( 

263 target=DecorrelateALKernelSpatialTask, 

264 doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. " 

265 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the " 

266 "default of 5.5).", 

267 ) 

268 # Old style ImageMapper grid. ZogyTask has its own grid option 

269 doSpatiallyVarying = pexConfig.Field( 

270 dtype=bool, 

271 default=False, 

272 doc="Perform A&L decorrelation on a grid across the " 

273 "image in order to allow for spatial variations. Zogy does not use this option." 

274 ) 

275 detection = pexConfig.ConfigurableField( 

276 target=SourceDetectionTask, 

277 doc="Low-threshold detection for final measurement", 

278 ) 

279 measurement = pexConfig.ConfigurableField( 

280 target=DipoleFitTask, 

281 doc="Enable updated dipole fitting method", 

282 ) 

283 doApCorr = lsst.pex.config.Field( 

284 dtype=bool, 

285 default=True, 

286 doc="Run subtask to apply aperture corrections" 

287 ) 

288 applyApCorr = lsst.pex.config.ConfigurableField( 

289 target=ApplyApCorrTask, 

290 doc="Subtask to apply aperture corrections" 

291 ) 

292 forcedMeasurement = pexConfig.ConfigurableField( 

293 target=ForcedMeasurementTask, 

294 doc="Subtask to force photometer PVI at diaSource location.", 

295 ) 

296 getTemplate = pexConfig.ConfigurableField( 

297 target=GetCoaddAsTemplateTask, 

298 doc="Subtask to retrieve template exposure and sources", 

299 ) 

300 scaleVariance = pexConfig.ConfigurableField( 

301 target=ScaleVarianceTask, 

302 doc="Subtask to rescale the variance of the template " 

303 "to the statistically expected level" 

304 ) 

305 controlStepSize = pexConfig.Field( 

306 doc="What step size (every Nth one) to select a control sample from the kernelSources", 

307 dtype=int, 

308 default=5 

309 ) 

310 controlRandomSeed = pexConfig.Field( 

311 doc="Random seed for shuffing the control sample", 

312 dtype=int, 

313 default=10 

314 ) 

315 register = pexConfig.ConfigurableField( 

316 target=RegisterTask, 

317 doc="Task to enable image-to-image image registration (warping)", 

318 ) 

319 kernelSourcesFromRef = pexConfig.Field( 

320 doc="Select sources to measure kernel from reference catalog if True, template if false", 

321 dtype=bool, 

322 default=False 

323 ) 

324 templateSipOrder = pexConfig.Field( 

325 dtype=int, default=2, 

326 doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)" 

327 ) 

328 growFootprint = pexConfig.Field( 

329 dtype=int, default=2, 

330 doc="Grow positive and negative footprints by this amount before merging" 

331 ) 

332 diaSourceMatchRadius = pexConfig.Field( 

333 dtype=float, default=0.5, 

334 doc="Match radius (in arcseconds) for DiaSource to Source association" 

335 ) 

336 requiredTemplateFraction = pexConfig.Field( 

337 dtype=float, default=0.1, 

338 doc="Do not attempt to run task if template covers less than this fraction of pixels." 

339 "Setting to 0 will always attempt image subtraction" 

340 ) 

341 doSkySources = pexConfig.Field( 

342 dtype=bool, 

343 default=False, 

344 doc="Generate sky sources?", 

345 ) 

346 skySources = pexConfig.ConfigurableField( 

347 target=SkyObjectsTask, 

348 doc="Generate sky sources", 

349 ) 

350 

351 def setDefaults(self): 

352 # defaults are OK for catalog and diacatalog 

353 

354 self.subtract['al'].kernel.name = "AL" 

355 self.subtract['al'].kernel.active.fitForBackground = True 

356 self.subtract['al'].kernel.active.spatialKernelOrder = 1 

357 self.subtract['al'].kernel.active.spatialBgOrder = 2 

358 

359 # DiaSource Detection 

360 self.detection.thresholdPolarity = "both" 

361 self.detection.thresholdValue = 5.0 

362 self.detection.reEstimateBackground = False 

363 self.detection.thresholdType = "pixel_stdev" 

364 

365 # Add filtered flux measurement, the correct measurement for pre-convolved images. 

366 # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier. 

367 # To change that you must modify algorithms.names in the task's applyOverrides method, 

368 # after the user has set doPreConvolve. 

369 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux') 

370 self.measurement.plugins.names |= ['ext_trailedSources_Naive', 

371 'base_LocalPhotoCalib', 

372 'base_LocalWcs'] 

373 

374 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"] 

375 self.forcedMeasurement.copyColumns = { 

376 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"} 

377 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid" 

378 self.forcedMeasurement.slots.shape = None 

379 

380 # For shuffling the control sample 

381 random.seed(self.controlRandomSeed) 

382 

383 def validate(self): 

384 pexConfig.Config.validate(self) 

385 if not self.doSubtract and not self.doDetection: 

386 raise ValueError("Either doSubtract or doDetection must be enabled.") 

387 if self.doMeasurement and not self.doDetection: 

388 raise ValueError("Cannot run source measurement without source detection.") 

389 if self.doMerge and not self.doDetection: 

390 raise ValueError("Cannot run source merging without source detection.") 

391 if self.doSkySources and not self.doDetection: 

392 raise ValueError("Cannot run sky source creation without source detection.") 

393 if self.doUseRegister and not self.doSelectSources: 

394 raise ValueError("doUseRegister=True and doSelectSources=False. " 

395 "Cannot run RegisterTask without selecting sources.") 

396 if self.doScaleDiffimVariance and self.doScaleTemplateVariance: 

397 raise ValueError("Scaling the diffim variance and scaling the template variance " 

398 "are both set. Please choose one or the other.") 

399 # We cannot allow inconsistencies that would lead to None or not available output products 

400 if self.subtract.name == 'zogy': 

401 if self.doWriteMatchedExp: 

402 raise ValueError("doWriteMatchedExp=True Matched exposure is not " 

403 "calculated in zogy subtraction.") 

404 if self.doAddMetrics: 

405 raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.") 

406 if self.doDecorrelation: 

407 raise ValueError( 

408 "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.") 

409 if self.doSelectSources: 

410 raise ValueError( 

411 "doSelectSources=True Selecting sources for PSF matching is not a zogy option.") 

412 if self.useGaussianForPreConvolution: 

413 raise ValueError( 

414 "useGaussianForPreConvolution=True This is an AL subtraction only option.") 

415 else: 

416 # AL only consistency checks 

417 if self.useScoreImageDetection and not self.convolveTemplate: 

418 raise ValueError( 

419 "convolveTemplate=False and useScoreImageDetection=True " 

420 "Pre-convolution and matching of the science image is not a supported operation.") 

421 if self.doWriteSubtractedExp and self.useScoreImageDetection: 

422 raise ValueError( 

423 "doWriteSubtractedExp=True and useScoreImageDetection=True " 

424 "Regular difference image is not calculated. " 

425 "AL subtraction calculates either the regular difference image or the score image.") 

426 if self.doWriteScoreExp and not self.useScoreImageDetection: 

427 raise ValueError( 

428 "doWriteScoreExp=True and useScoreImageDetection=False " 

429 "Score image is not calculated. " 

430 "AL subtraction calculates either the regular difference image or the score image.") 

431 if self.doAddMetrics and not self.doSubtract: 

432 raise ValueError("Subtraction must be enabled for kernel metrics calculation.") 

433 if self.useGaussianForPreConvolution and not self.useScoreImageDetection: 

434 raise ValueError( 

435 "useGaussianForPreConvolution=True and useScoreImageDetection=False " 

436 "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.") 

437 

438 

439@deprecated(reason="This Task has been replaced with lsst.ip.diffim.subtractImages" 

440 " and lsst.ip.diffim.detectAndMeasure. Will be removed after v25.", 

441 version="v24.0", category=FutureWarning) 

442class ImageDifferenceTask(pipeBase.PipelineTask): 

443 """Subtract an image from a template and measure the result 

444 """ 

445 ConfigClass = ImageDifferenceConfig 

446 _DefaultName = "imageDifference" 

447 

448 def __init__(self, butler=None, **kwargs): 

449 """!Construct an ImageDifference Task 

450 

451 @param[in] butler Butler object to use in constructing reference object loaders 

452 """ 

453 super().__init__(**kwargs) 

454 self.makeSubtask("getTemplate") 

455 

456 self.makeSubtask("subtract") 

457 

458 if self.config.subtract.name == 'al' and self.config.doDecorrelation: 

459 self.makeSubtask("decorrelate") 

460 

461 if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance: 

462 self.makeSubtask("scaleVariance") 

463 

464 if self.config.doUseRegister: 

465 self.makeSubtask("register") 

466 self.schema = afwTable.SourceTable.makeMinimalSchema() 

467 

468 if self.config.doSelectSources: 

469 self.makeSubtask("sourceSelector") 

470 if self.config.kernelSourcesFromRef: 

471 self.makeSubtask('refObjLoader', butler=butler) 

472 self.makeSubtask("astrometer", refObjLoader=self.refObjLoader) 

473 

474 self.algMetadata = dafBase.PropertyList() 

475 if self.config.doDetection: 

476 self.makeSubtask("detection", schema=self.schema) 

477 if self.config.doMeasurement: 

478 self.makeSubtask("measurement", schema=self.schema, 

479 algMetadata=self.algMetadata) 

480 if self.config.doApCorr: 

481 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

482 if self.config.doForcedMeasurement: 

483 self.schema.addField( 

484 "ip_diffim_forced_PsfFlux_instFlux", "D", 

485 "Forced PSF flux measured on the direct image.", 

486 units="count") 

487 self.schema.addField( 

488 "ip_diffim_forced_PsfFlux_instFluxErr", "D", 

489 "Forced PSF flux error measured on the direct image.", 

490 units="count") 

491 self.schema.addField( 

492 "ip_diffim_forced_PsfFlux_area", "F", 

493 "Forced PSF flux effective area of PSF.", 

494 units="pixel") 

495 self.schema.addField( 

496 "ip_diffim_forced_PsfFlux_flag", "Flag", 

497 "Forced PSF flux general failure flag.") 

498 self.schema.addField( 

499 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag", 

500 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.") 

501 self.schema.addField( 

502 "ip_diffim_forced_PsfFlux_flag_edge", "Flag", 

503 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.") 

504 self.makeSubtask("forcedMeasurement", refSchema=self.schema) 

505 if self.config.doMatchSources: 

506 self.schema.addField("refMatchId", "L", "unique id of reference catalog match") 

507 self.schema.addField("srcMatchId", "L", "unique id of source match") 

508 if self.config.doSkySources: 

509 self.makeSubtask("skySources") 

510 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

511 

512 # initialize InitOutputs 

513 self.outputSchema = afwTable.SourceCatalog(self.schema) 

514 self.outputSchema.getTable().setMetadata(self.algMetadata) 

515 

516 @staticmethod 

517 def makeIdFactory(expId, expBits): 

518 """Create IdFactory instance for unique 64 bit diaSource id-s. 

519 

520 Parameters 

521 ---------- 

522 expId : `int` 

523 Exposure id. 

524 

525 expBits: `int` 

526 Number of used bits in ``expId``. 

527 

528 Notes 

529 ----- 

530 The diasource id-s consists of the ``expId`` stored fixed in the highest value 

531 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 

532 low value end of the integer. 

533 

534 Returns 

535 ------- 

536 idFactory: `lsst.afw.table.IdFactory` 

537 """ 

538 return ExposureIdInfo(expId, expBits).makeSourceIdFactory() 

539 

540 @lsst.utils.inheritDoc(pipeBase.PipelineTask) 

541 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, 

542 inputRefs: pipeBase.InputQuantizedConnection, 

543 outputRefs: pipeBase.OutputQuantizedConnection): 

544 inputs = butlerQC.get(inputRefs) 

545 self.log.info("Processing %s", butlerQC.quantum.dataId) 

546 

547 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None) 

548 exposure = self.prepareCalibratedExposure( 

549 inputs["exposure"], 

550 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog 

551 ) 

552 

553 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

554 returnMaxBits=True) 

555 idFactory = self.makeIdFactory(expId=expId, expBits=expBits) 

556 if self.config.coaddName == 'dcr': 

557 templateExposures = inputRefs.dcrCoadds 

558 else: 

559 templateExposures = inputRefs.coaddExposures 

560 templateStruct = self.getTemplate.runQuantum( 

561 exposure, butlerQC, inputRefs.skyMap, templateExposures 

562 ) 

563 

564 self.checkTemplateIsSufficient(templateStruct.exposure) 

565 

566 outputs = self.run(exposure=exposure, 

567 templateExposure=templateStruct.exposure, 

568 idFactory=idFactory) 

569 # Consistency with runDataref gen2 handling 

570 if outputs.diaSources is None: 

571 del outputs.diaSources 

572 butlerQC.put(outputs, outputRefs) 

573 

574 def prepareCalibratedExposure(self, exposure, finalizedPsfApCorrCatalog=None): 

575 """Prepare a calibrated exposure and apply finalized psf if so configured. 

576 

577 Parameters 

578 ---------- 

579 exposure : `lsst.afw.image.exposure.Exposure` 

580 Input exposure to adjust calibrations. 

581 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

582 Exposure catalog with finalized psf models and aperture correction 

583 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses 

584 the detector id for the catalog id, sorted on id for fast lookup. 

585 

586 Returns 

587 ------- 

588 exposure : `lsst.afw.image.exposure.Exposure` 

589 Exposure with adjusted calibrations. 

590 """ 

591 detectorId = exposure.getInfo().getDetector().getId() 

592 

593 if finalizedPsfApCorrCatalog is not None: 

594 row = finalizedPsfApCorrCatalog.find(detectorId) 

595 if row is None: 

596 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; " 

597 "Using original psf.", detectorId) 

598 else: 

599 psf = row.getPsf() 

600 apCorrMap = row.getApCorrMap() 

601 if psf is None or apCorrMap is None: 

602 self.log.warning("Detector id %s has None for psf/apCorrMap in " 

603 "finalizedPsfApCorrCatalog; Using original psf.", detectorId) 

604 else: 

605 exposure.setPsf(psf) 

606 exposure.info.setApCorrMap(apCorrMap) 

607 

608 return exposure 

609 

610 @timeMethod 

611 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None, 

612 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None): 

613 """PSF matches, subtract two images and perform detection on the difference image. 

614 

615 Parameters 

616 ---------- 

617 exposure : `lsst.afw.image.ExposureF`, optional 

618 The science exposure, the minuend in the image subtraction. 

619 Can be None only if ``config.doSubtract==False``. 

620 selectSources : `lsst.afw.table.SourceCatalog`, optional 

621 Identified sources on the science exposure. This catalog is used to 

622 select sources in order to perform the AL PSF matching on stamp images 

623 around them. The selection steps depend on config options and whether 

624 ``templateSources`` and ``matchingSources`` specified. 

625 templateExposure : `lsst.afw.image.ExposureF`, optional 

626 The template to be subtracted from ``exposure`` in the image subtraction. 

627 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``. 

628 The template exposure should cover the same sky area as the science exposure. 

629 It is either a stich of patches of a coadd skymap image or a calexp 

630 of the same pointing as the science exposure. Can be None only 

631 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None. 

632 templateSources : `lsst.afw.table.SourceCatalog`, optional 

633 Identified sources on the template exposure. 

634 idFactory : `lsst.afw.table.IdFactory` 

635 Generator object to assign ids to detected sources in the difference image. 

636 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional 

637 Background exposure to be added back to the science exposure 

638 if ``config.doAddCalexpBackground==True`` 

639 subtractedExposure : `lsst.afw.image.ExposureF`, optional 

640 If ``config.doSubtract==False`` and ``config.doDetection==True``, 

641 performs the post subtraction source detection only on this exposure. 

642 Otherwise should be None. 

643 

644 Returns 

645 ------- 

646 results : `lsst.pipe.base.Struct` 

647 ``subtractedExposure`` : `lsst.afw.image.ExposureF` 

648 Difference image. 

649 ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None` 

650 The zogy score exposure, if calculated. 

651 ``matchedExposure`` : `lsst.afw.image.ExposureF` 

652 The matched PSF exposure. 

653 ``subtractRes`` : `lsst.pipe.base.Struct` 

654 The returned result structure of the ImagePsfMatchTask subtask. 

655 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

656 The catalog of detected sources. 

657 ``selectSources`` : `lsst.afw.table.SourceCatalog` 

658 The input source catalog with optionally added Qa information. 

659 

660 Notes 

661 ----- 

662 The following major steps are included: 

663 

664 - warp template coadd to match WCS of image 

665 - PSF match image to warped template 

666 - subtract image from PSF-matched, warped template 

667 - detect sources 

668 - measure sources 

669 

670 For details about the image subtraction configuration modes 

671 see `lsst.ip.diffim`. 

672 """ 

673 subtractRes = None 

674 controlSources = None 

675 subtractedExposure = None 

676 scoreExposure = None 

677 diaSources = None 

678 kernelSources = None 

679 # We'll clone exposure if modified but will still need the original 

680 exposureOrig = exposure 

681 

682 if self.config.doAddCalexpBackground: 

683 mi = exposure.getMaskedImage() 

684 mi += calexpBackgroundExposure.getImage() 

685 

686 if not exposure.hasPsf(): 

687 raise pipeBase.TaskError("Exposure has no psf") 

688 sciencePsf = exposure.getPsf() 

689 

690 if self.config.doSubtract: 

691 if self.config.doScaleTemplateVariance: 

692 self.log.info("Rescaling template variance") 

693 templateVarFactor = self.scaleVariance.run( 

694 templateExposure.getMaskedImage()) 

695 self.log.info("Template variance scaling factor: %.2f", templateVarFactor) 

696 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor) 

697 self.metadata.add("psfMatchingAlgorithm", self.config.subtract.name) 

698 

699 if self.config.subtract.name == 'zogy': 

700 subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True) 

701 scoreExposure = subtractRes.scoreExp 

702 subtractedExposure = subtractRes.diffExp 

703 subtractRes.subtractedExposure = subtractedExposure 

704 subtractRes.matchedExposure = None 

705 

706 elif self.config.subtract.name == 'al': 

707 # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution 

708 # Just need a rough estimate; average positions are fine 

709 sciAvgPos = sciencePsf.getAveragePosition() 

710 scienceSigmaOrig = sciencePsf.computeShape(sciAvgPos).getDeterminantRadius() 

711 

712 templatePsf = templateExposure.getPsf() 

713 templateAvgPos = templatePsf.getAveragePosition() 

714 templateSigma = templatePsf.computeShape(templateAvgPos).getDeterminantRadius() 

715 

716 # if requested, convolve the science exposure with its PSF 

717 # (properly, this should be a cross-correlation, but our code does not yet support that) 

718 # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, 

719 # else sigma of original science exposure 

720 # TODO: DM-22762 This functional block should be moved into its own method 

721 preConvPsf = None 

722 if self.config.useScoreImageDetection: 

723 self.log.warning("AL likelihood image: pre-convolution of PSF is not implemented.") 

724 convControl = afwMath.ConvolutionControl() 

725 # cannot convolve in place, so need a new image anyway 

726 srcMI = exposure.maskedImage 

727 exposure = exposure.clone() # New deep copy 

728 srcPsf = sciencePsf 

729 if self.config.useGaussianForPreConvolution: 

730 self.log.info( 

731 "AL likelihood image: Using Gaussian (sigma=%.2f) PSF estimation " 

732 "for science image pre-convolution", scienceSigmaOrig) 

733 # convolve with a simplified PSF model: a double Gaussian 

734 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions() 

735 preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) 

736 else: 

737 # convolve with science exposure's PSF model 

738 self.log.info( 

739 "AL likelihood image: Using the science image PSF for pre-convolution.") 

740 preConvPsf = srcPsf 

741 afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl) 

742 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2) 

743 else: 

744 scienceSigmaPost = scienceSigmaOrig 

745 

746 # If requested, find and select sources from the image 

747 # else, AL subtraction will do its own source detection 

748 # TODO: DM-22762 This functional block should be moved into its own method 

749 if self.config.doSelectSources: 

750 if selectSources is None: 

751 self.log.warning("Src product does not exist; running detection, measurement," 

752 " selection") 

753 # Run own detection and measurement; necessary in nightly processing 

754 selectSources = self.subtract.getSelectSources( 

755 exposure, 

756 sigma=scienceSigmaPost, 

757 doSmooth=not self.config.useScoreImageDetection, 

758 idFactory=idFactory, 

759 ) 

760 

761 if self.config.doAddMetrics: 

762 # Number of basis functions 

763 

764 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active, 

765 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma, 

766 targetFwhmPix=templateSigma*FwhmPerSigma)) 

767 # Modify the schema of all Sources 

768 # DEPRECATED: This is a data dependent (nparam) output product schema 

769 # outside the task constructor. 

770 # NOTE: The pre-determination of nparam at this point 

771 # may be incorrect as the template psf is warped later in 

772 # ImagePsfMatchTask.matchExposures() 

773 kcQa = KernelCandidateQa(nparam) 

774 selectSources = kcQa.addToSchema(selectSources) 

775 if self.config.kernelSourcesFromRef: 

776 # match exposure sources to reference catalog 

777 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources) 

778 matches = astromRet.matches 

779 elif templateSources: 

780 # match exposure sources to template sources 

781 mc = afwTable.MatchControl() 

782 mc.findOnlyClosest = False 

783 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds, 

784 mc) 

785 else: 

786 raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," 

787 "but template sources not available. Cannot match science " 

788 "sources with template sources. Run process* on data from " 

789 "which templates are built.") 

790 

791 kernelSources = self.sourceSelector.run(selectSources, exposure=exposure, 

792 matches=matches).sourceCat 

793 random.shuffle(kernelSources, random.random) 

794 controlSources = kernelSources[::self.config.controlStepSize] 

795 kernelSources = [k for i, k in enumerate(kernelSources) 

796 if i % self.config.controlStepSize] 

797 

798 if self.config.doSelectDcrCatalog: 

799 redSelector = DiaCatalogSourceSelectorTask( 

800 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, 

801 grMax=99.999)) 

802 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat 

803 controlSources.extend(redSources) 

804 

805 blueSelector = DiaCatalogSourceSelectorTask( 

806 DiaCatalogSourceSelectorConfig(grMin=-99.999, 

807 grMax=self.sourceSelector.config.grMin)) 

808 blueSources = blueSelector.selectStars(exposure, selectSources, 

809 matches=matches).starCat 

810 controlSources.extend(blueSources) 

811 

812 if self.config.doSelectVariableCatalog: 

813 varSelector = DiaCatalogSourceSelectorTask( 

814 DiaCatalogSourceSelectorConfig(includeVariable=True)) 

815 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat 

816 controlSources.extend(varSources) 

817 

818 self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)", 

819 len(kernelSources), len(selectSources), len(controlSources)) 

820 

821 allresids = {} 

822 # TODO: DM-22762 This functional block should be moved into its own method 

823 if self.config.doUseRegister: 

824 self.log.info("Registering images") 

825 

826 if templateSources is None: 

827 # Run detection on the template, which is 

828 # temporarily background-subtracted 

829 # sigma of PSF of template image before warping 

830 templateSources = self.subtract.getSelectSources( 

831 templateExposure, 

832 sigma=templateSigma, 

833 doSmooth=True, 

834 idFactory=idFactory 

835 ) 

836 

837 # Third step: we need to fit the relative astrometry. 

838 # 

839 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources) 

840 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs, 

841 exposure.getWcs(), exposure.getBBox()) 

842 templateExposure = warpedExp 

843 

844 # Create debugging outputs on the astrometric 

845 # residuals as a function of position. Persistence 

846 # not yet implemented; expected on (I believe) #2636. 

847 if self.config.doDebugRegister: 

848 # Grab matches to reference catalog 

849 srcToMatch = {x.second.getId(): x.first for x in matches} 

850 

851 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey() 

852 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey() 

853 sids = [m.first.getId() for m in wcsResults.matches] 

854 positions = [m.first.get(refCoordKey) for m in wcsResults.matches] 

855 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky( 

856 m.second.get(inCentroidKey))) for m in wcsResults.matches] 

857 allresids = dict(zip(sids, zip(positions, residuals))) 

858 

859 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset( 

860 wcsResults.wcs.pixelToSky( 

861 m.second.get(inCentroidKey))) for m in wcsResults.matches] 

862 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) 

863 + 2.5*numpy.log10(srcToMatch[x].get("r")) 

864 for x in sids if x in srcToMatch.keys()]) 

865 dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals) 

866 if s in srcToMatch.keys()]) 

867 dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals) 

868 if s in srcToMatch.keys()]) 

869 idx1 = numpy.where(colors < self.sourceSelector.config.grMin) 

870 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) 

871 & (colors <= self.sourceSelector.config.grMax)) 

872 idx3 = numpy.where(colors > self.sourceSelector.config.grMax) 

873 rms1Long = IqrToSigma*( 

874 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25))) 

875 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) 

876 - numpy.percentile(dlat[idx1], 25)) 

877 rms2Long = IqrToSigma*( 

878 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25))) 

879 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) 

880 - numpy.percentile(dlat[idx2], 25)) 

881 rms3Long = IqrToSigma*( 

882 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25))) 

883 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) 

884 - numpy.percentile(dlat[idx3], 25)) 

885 self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f", 

886 numpy.median(dlong[idx1]), rms1Long, 

887 numpy.median(dlat[idx1]), rms1Lat) 

888 self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f", 

889 numpy.median(dlong[idx2]), rms2Long, 

890 numpy.median(dlat[idx2]), rms2Lat) 

891 self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f", 

892 numpy.median(dlong[idx3]), rms3Long, 

893 numpy.median(dlat[idx3]), rms3Lat) 

894 

895 self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1])) 

896 self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2])) 

897 self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3])) 

898 self.metadata.add("RegisterBlueLongOffsetStd", rms1Long) 

899 self.metadata.add("RegisterGreenLongOffsetStd", rms2Long) 

900 self.metadata.add("RegisterRedLongOffsetStd", rms3Long) 

901 

902 self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1])) 

903 self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2])) 

904 self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3])) 

905 self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat) 

906 self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat) 

907 self.metadata.add("RegisterRedLatOffsetStd", rms3Lat) 

908 

909 # warp template exposure to match exposure, 

910 # PSF match template exposure to exposure, 

911 # then return the difference 

912 

913 # Return warped template... Construct sourceKernelCand list after subtract 

914 self.log.info("Subtracting images") 

915 subtractRes = self.subtract.subtractExposures( 

916 templateExposure=templateExposure, 

917 scienceExposure=exposure, 

918 candidateList=kernelSources, 

919 convolveTemplate=self.config.convolveTemplate, 

920 doWarping=not self.config.doUseRegister 

921 ) 

922 if self.config.useScoreImageDetection: 

923 scoreExposure = subtractRes.subtractedExposure 

924 else: 

925 subtractedExposure = subtractRes.subtractedExposure 

926 

927 if self.config.doDetection: 

928 self.log.info("Computing diffim PSF") 

929 

930 # Get Psf from the appropriate input image if it doesn't exist 

931 if subtractedExposure is not None and not subtractedExposure.hasPsf(): 

932 if self.config.convolveTemplate: 

933 subtractedExposure.setPsf(exposure.getPsf()) 

934 else: 

935 subtractedExposure.setPsf(templateExposure.getPsf()) 

936 

937 # If doSubtract is False, then subtractedExposure was fetched from disk (above), 

938 # thus it may have already been decorrelated. Thus, we do not decorrelate if 

939 # doSubtract is False. 

940 

941 # NOTE: At this point doSubtract == True 

942 if self.config.doDecorrelation and self.config.doSubtract: 

943 preConvKernel = None 

944 if self.config.useGaussianForPreConvolution: 

945 preConvKernel = preConvPsf.getLocalKernel() 

946 if self.config.useScoreImageDetection: 

947 scoreExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure, 

948 scoreExposure, 

949 subtractRes.psfMatchingKernel, 

950 spatiallyVarying=self.config.doSpatiallyVarying, 

951 preConvKernel=preConvKernel, 

952 templateMatched=True, 

953 preConvMode=True).correctedExposure 

954 # Note that the subtracted exposure is always decorrelated, 

955 # even if the score image is used for detection 

956 subtractedExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure, 

957 subtractedExposure, 

958 subtractRes.psfMatchingKernel, 

959 spatiallyVarying=self.config.doSpatiallyVarying, 

960 preConvKernel=None, 

961 templateMatched=self.config.convolveTemplate, 

962 preConvMode=False).correctedExposure 

963 # END (if subtractAlgorithm == 'AL') 

964 # END (if self.config.doSubtract) 

965 if self.config.doDetection: 

966 self.log.info("Running diaSource detection") 

967 

968 # subtractedExposure - reserved for task return value 

969 # in zogy, it is always the proper difference image 

970 # in AL, it may be (yet) pre-convolved and/or decorrelated 

971 # 

972 # detectionExposure - controls which exposure to use for detection 

973 # in-place modifications will appear in task return 

974 if self.config.useScoreImageDetection: 

975 # zogy with score image detection enabled 

976 self.log.info("Detection, diffim rescaling and measurements are " 

977 "on AL likelihood or Zogy score image.") 

978 detectionExposure = scoreExposure 

979 else: 

980 # AL or zogy with no score image detection 

981 detectionExposure = subtractedExposure 

982 

983 # Rescale difference image variance plane 

984 if self.config.doScaleDiffimVariance: 

985 self.log.info("Rescaling diffim variance") 

986 diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage()) 

987 self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor) 

988 self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor) 

989 

990 # Erase existing detection mask planes 

991 mask = detectionExposure.getMaskedImage().getMask() 

992 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

993 

994 table = afwTable.SourceTable.make(self.schema, idFactory) 

995 table.setMetadata(self.algMetadata) 

996 results = self.detection.run( 

997 table=table, 

998 exposure=detectionExposure, 

999 doSmooth=not self.config.useScoreImageDetection 

1000 ) 

1001 

1002 if self.config.doMerge: 

1003 fpSet = results.fpSets.positive 

1004 fpSet.merge(results.fpSets.negative, self.config.growFootprint, 

1005 self.config.growFootprint, False) 

1006 diaSources = afwTable.SourceCatalog(table) 

1007 fpSet.makeSources(diaSources) 

1008 self.log.info("Merging detections into %d sources", len(diaSources)) 

1009 else: 

1010 diaSources = results.sources 

1011 # Inject skySources before measurement. 

1012 if self.config.doSkySources: 

1013 skySourceFootprints = self.skySources.run( 

1014 mask=detectionExposure.mask, 

1015 seed=detectionExposure.info.id) 

1016 if skySourceFootprints: 

1017 for foot in skySourceFootprints: 

1018 s = diaSources.addNew() 

1019 s.setFootprint(foot) 

1020 s.set(self.skySourceKey, True) 

1021 

1022 if self.config.doMeasurement: 

1023 newDipoleFitting = self.config.doDipoleFitting 

1024 self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting) 

1025 if not newDipoleFitting: 

1026 # Just fit dipole in diffim 

1027 self.measurement.run(diaSources, detectionExposure) 

1028 else: 

1029 # Use (matched) template and science image (if avail.) to constrain dipole fitting 

1030 if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict(): 

1031 self.measurement.run(diaSources, detectionExposure, exposure, 

1032 subtractRes.matchedExposure) 

1033 else: 

1034 self.measurement.run(diaSources, detectionExposure, exposure) 

1035 if self.config.doApCorr: 

1036 self.applyApCorr.run( 

1037 catalog=diaSources, 

1038 apCorrMap=detectionExposure.getInfo().getApCorrMap() 

1039 ) 

1040 

1041 if self.config.doForcedMeasurement: 

1042 # Run forced psf photometry on the PVI at the diaSource locations. 

1043 # Copy the measured flux and error into the diaSource. 

1044 forcedSources = self.forcedMeasurement.generateMeasCat( 

1045 exposure, diaSources, detectionExposure.getWcs()) 

1046 self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs()) 

1047 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema) 

1048 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], 

1049 "ip_diffim_forced_PsfFlux_instFlux", True) 

1050 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0], 

1051 "ip_diffim_forced_PsfFlux_instFluxErr", True) 

1052 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0], 

1053 "ip_diffim_forced_PsfFlux_area", True) 

1054 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0], 

1055 "ip_diffim_forced_PsfFlux_flag", True) 

1056 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0], 

1057 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True) 

1058 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0], 

1059 "ip_diffim_forced_PsfFlux_flag_edge", True) 

1060 for diaSource, forcedSource in zip(diaSources, forcedSources): 

1061 diaSource.assign(forcedSource, mapper) 

1062 

1063 # Match with the calexp sources if possible 

1064 if self.config.doMatchSources: 

1065 if selectSources is not None: 

1066 # Create key,val pair where key=diaSourceId and val=sourceId 

1067 matchRadAsec = self.config.diaSourceMatchRadius 

1068 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds() 

1069 

1070 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel) 

1071 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for 

1072 srcMatch in srcMatches]) 

1073 self.log.info("Matched %d / %d diaSources to sources", 

1074 len(srcMatchDict), len(diaSources)) 

1075 else: 

1076 self.log.warning("Src product does not exist; cannot match with diaSources") 

1077 srcMatchDict = {} 

1078 

1079 # Create key,val pair where key=diaSourceId and val=refId 

1080 refAstromConfig = AstrometryConfig() 

1081 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec 

1082 refAstrometer = AstrometryTask(self.refObjLoader, config=refAstromConfig) 

1083 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources) 

1084 refMatches = astromRet.matches 

1085 if refMatches is None: 

1086 self.log.warning("No diaSource matches with reference catalog") 

1087 refMatchDict = {} 

1088 else: 

1089 self.log.info("Matched %d / %d diaSources to reference catalog", 

1090 len(refMatches), len(diaSources)) 

1091 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for 

1092 refMatch in refMatches]) 

1093 

1094 # Assign source Ids 

1095 for diaSource in diaSources: 

1096 sid = diaSource.getId() 

1097 if sid in srcMatchDict: 

1098 diaSource.set("srcMatchId", srcMatchDict[sid]) 

1099 if sid in refMatchDict: 

1100 diaSource.set("refMatchId", refMatchDict[sid]) 

1101 

1102 if self.config.doAddMetrics and self.config.doSelectSources: 

1103 self.log.info("Evaluating metrics and control sample") 

1104 

1105 kernelCandList = [] 

1106 for cell in subtractRes.kernelCellSet.getCellList(): 

1107 for cand in cell.begin(False): # include bad candidates 

1108 kernelCandList.append(cand) 

1109 

1110 # Get basis list to build control sample kernels 

1111 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList() 

1112 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters()) 

1113 

1114 controlCandList = ( 

1115 diffimTools.sourceTableToCandidateList(controlSources, 

1116 subtractRes.warpedExposure, exposure, 

1117 self.config.subtract.kernel.active, 

1118 self.config.subtract.kernel.active.detectionConfig, 

1119 self.log, doBuild=True, basisList=basisList)) 

1120 

1121 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel, 

1122 subtractRes.backgroundModel, dof=nparam) 

1123 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel, 

1124 subtractRes.backgroundModel) 

1125 

1126 if self.config.doDetection: 

1127 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources) 

1128 else: 

1129 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids) 

1130 

1131 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) 

1132 return pipeBase.Struct( 

1133 subtractedExposure=subtractedExposure, 

1134 scoreExposure=scoreExposure, 

1135 warpedExposure=subtractRes.warpedExposure, 

1136 matchedExposure=subtractRes.matchedExposure, 

1137 subtractRes=subtractRes, 

1138 diaSources=diaSources, 

1139 selectSources=selectSources 

1140 ) 

1141 

1142 def fitAstrometry(self, templateSources, templateExposure, selectSources): 

1143 """Fit the relative astrometry between templateSources and selectSources 

1144 

1145 Notes 

1146 ----- 

1147 TODO: Remove this method. It originally fit a new WCS to the template before calling register.run 

1148 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed. 

1149 It remains because a subtask overrides it. 

1150 """ 

1151 results = self.register.run(templateSources, templateExposure.getWcs(), 

1152 templateExposure.getBBox(), selectSources) 

1153 return results 

1154 

1155 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources): 

1156 """Make debug plots and displays. 

1157 

1158 Notes 

1159 ----- 

1160 TODO: Test and update for current debug display and slot names. 

1161 """ 

1162 import lsstDebug 

1163 display = lsstDebug.Info(__name__).display 

1164 showSubtracted = lsstDebug.Info(__name__).showSubtracted 

1165 showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals 

1166 showDiaSources = lsstDebug.Info(__name__).showDiaSources 

1167 showDipoles = lsstDebug.Info(__name__).showDipoles 

1168 maskTransparency = lsstDebug.Info(__name__).maskTransparency 

1169 if display: 

1170 disp = afwDisplay.getDisplay(frame=lsstDebug.frame) 

1171 if not maskTransparency: 

1172 maskTransparency = 0 

1173 disp.setMaskTransparency(maskTransparency) 

1174 

1175 if display and showSubtracted: 

1176 disp.mtv(subtractRes.subtractedExposure, title="Subtracted image") 

1177 mi = subtractRes.subtractedExposure.getMaskedImage() 

1178 x0, y0 = mi.getX0(), mi.getY0() 

1179 with disp.Buffering(): 

1180 for s in diaSources: 

1181 x, y = s.getX() - x0, s.getY() - y0 

1182 ctype = "red" if s.get("flags_negative") else "yellow" 

1183 if (s.get("base_PixelFlags_flag_interpolatedCenter") 

1184 or s.get("base_PixelFlags_flag_saturatedCenter") 

1185 or s.get("base_PixelFlags_flag_crCenter")): 

1186 ptype = "x" 

1187 elif (s.get("base_PixelFlags_flag_interpolated") 

1188 or s.get("base_PixelFlags_flag_saturated") 

1189 or s.get("base_PixelFlags_flag_cr")): 

1190 ptype = "+" 

1191 else: 

1192 ptype = "o" 

1193 disp.dot(ptype, x, y, size=4, ctype=ctype) 

1194 lsstDebug.frame += 1 

1195 

1196 if display and showPixelResiduals and selectSources: 

1197 nonKernelSources = [] 

1198 for source in selectSources: 

1199 if source not in kernelSources: 

1200 nonKernelSources.append(source) 

1201 

1202 diUtils.plotPixelResiduals(exposure, 

1203 subtractRes.warpedExposure, 

1204 subtractRes.subtractedExposure, 

1205 subtractRes.kernelCellSet, 

1206 subtractRes.psfMatchingKernel, 

1207 subtractRes.backgroundModel, 

1208 nonKernelSources, 

1209 self.subtract.config.kernel.active.detectionConfig, 

1210 origVariance=False) 

1211 diUtils.plotPixelResiduals(exposure, 

1212 subtractRes.warpedExposure, 

1213 subtractRes.subtractedExposure, 

1214 subtractRes.kernelCellSet, 

1215 subtractRes.psfMatchingKernel, 

1216 subtractRes.backgroundModel, 

1217 nonKernelSources, 

1218 self.subtract.config.kernel.active.detectionConfig, 

1219 origVariance=True) 

1220 if display and showDiaSources: 

1221 flagChecker = SourceFlagChecker(diaSources) 

1222 isFlagged = [flagChecker(x) for x in diaSources] 

1223 isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources] 

1224 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole, 

1225 frame=lsstDebug.frame) 

1226 lsstDebug.frame += 1 

1227 

1228 if display and showDipoles: 

1229 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources, 

1230 frame=lsstDebug.frame) 

1231 lsstDebug.frame += 1 

1232 

1233 def checkTemplateIsSufficient(self, templateExposure): 

1234 """Raise NoWorkFound if template coverage < requiredTemplateFraction 

1235 

1236 Parameters 

1237 ---------- 

1238 templateExposure : `lsst.afw.image.ExposureF` 

1239 The template exposure to check 

1240 

1241 Raises 

1242 ------ 

1243 NoWorkFound 

1244 Raised if fraction of good pixels, defined as not having NO_DATA 

1245 set, is less then the configured requiredTemplateFraction 

1246 """ 

1247 # Count the number of pixels with the NO_DATA mask bit set 

1248 # counting NaN pixels is insufficient because pixels without data are often intepolated over) 

1249 pixNoData = numpy.count_nonzero(templateExposure.mask.array 

1250 & templateExposure.mask.getPlaneBitMask('NO_DATA')) 

1251 pixGood = templateExposure.getBBox().getArea() - pixNoData 

1252 self.log.info("template has %d good pixels (%.1f%%)", pixGood, 

1253 100*pixGood/templateExposure.getBBox().getArea()) 

1254 

1255 if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction: 

1256 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. " 

1257 "To force subtraction, set config requiredTemplateFraction=0." % ( 

1258 100*pixGood/templateExposure.getBBox().getArea(), 

1259 100*self.config.requiredTemplateFraction)) 

1260 raise pipeBase.NoWorkFound(message) 

1261 

1262 

1263class ImageDifferenceFromTemplateConnections(ImageDifferenceTaskConnections, 

1264 defaultTemplates={"coaddName": "goodSeeing"} 

1265 ): 

1266 inputTemplate = pipeBase.connectionTypes.Input( 

1267 doc=("Warped template produced by GetMultiTractCoaddTemplate"), 

1268 dimensions=("instrument", "visit", "detector"), 

1269 storageClass="ExposureF", 

1270 name="{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}", 

1271 ) 

1272 

1273 def __init__(self, *, config=None): 

1274 super().__init__(config=config) 

1275 # ImageDifferenceConnections will have removed one of these. 

1276 # Make sure they're both gone, because no coadds are needed. 

1277 if "coaddExposures" in self.inputs: 

1278 self.inputs.remove("coaddExposures") 

1279 if "dcrCoadds" in self.inputs: 

1280 self.inputs.remove("dcrCoadds") 

1281 

1282 

1283class ImageDifferenceFromTemplateConfig(ImageDifferenceConfig, 

1284 pipelineConnections=ImageDifferenceFromTemplateConnections): 

1285 pass 

1286 

1287 

1288class ImageDifferenceFromTemplateTask(ImageDifferenceTask): 

1289 ConfigClass = ImageDifferenceFromTemplateConfig 

1290 _DefaultName = "imageDifference" 

1291 

1292 @lsst.utils.inheritDoc(pipeBase.PipelineTask) 

1293 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1294 inputs = butlerQC.get(inputRefs) 

1295 self.log.info("Processing %s", butlerQC.quantum.dataId) 

1296 self.checkTemplateIsSufficient(inputs['inputTemplate']) 

1297 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

1298 returnMaxBits=True) 

1299 idFactory = self.makeIdFactory(expId=expId, expBits=expBits) 

1300 

1301 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None) 

1302 exposure = self.prepareCalibratedExposure( 

1303 inputs["exposure"], 

1304 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog 

1305 ) 

1306 

1307 outputs = self.run(exposure=exposure, 

1308 templateExposure=inputs['inputTemplate'], 

1309 idFactory=idFactory) 

1310 

1311 # Consistency with runDataref gen2 handling 

1312 if outputs.diaSources is None: 

1313 del outputs.diaSources 

1314 butlerQC.put(outputs, outputRefs)