Coverage for python/lsst/drp/tasks/dcr_assemble_coadd.py: 15%

413 statements  

« prev     ^ index     » next       coverage.py v7.3.3, created at 2023-12-16 15:07 +0000

1# This file is part of drp_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"] 

23 

24from math import ceil 

25 

26import lsst.afw.image as afwImage 

27import lsst.afw.table as afwTable 

28import lsst.coadd.utils as coaddUtils 

29import lsst.geom as geom 

30import lsst.meas.algorithms as measAlg 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33import lsst.utils as utils 

34import numpy as np 

35from lsst.ip.diffim.dcrModel import DcrModel, applyDcr, calculateDcr 

36from lsst.meas.base import SingleFrameMeasurementTask 

37from lsst.pipe.tasks.coaddBase import makeSkyInfo, subBBoxIter 

38from lsst.pipe.tasks.measurePsf import MeasurePsfTask 

39from lsst.utils.timer import timeMethod 

40from scipy import ndimage 

41 

42from .assemble_coadd import ( 

43 AssembleCoaddConnections, 

44 AssembleCoaddTask, 

45 CompareWarpAssembleCoaddConfig, 

46 CompareWarpAssembleCoaddTask, 

47) 

48 

49 

50class DcrAssembleCoaddConnections( 

51 AssembleCoaddConnections, 

52 dimensions=("tract", "patch", "band", "skymap"), 

53 defaultTemplates={ 

54 "inputWarpName": "deep", 

55 "inputCoaddName": "deep", 

56 "outputCoaddName": "dcr", 

57 "warpType": "direct", 

58 "warpTypeSuffix": "", 

59 "fakesType": "", 

60 }, 

61): 

62 inputWarps = pipeBase.connectionTypes.Input( 

63 doc=( 

64 "Input list of warps to be assembled i.e. stacked." 

65 "Note that this will often be different than the inputCoaddName." 

66 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter" 

67 ), 

68 name="{inputWarpName}Coadd_{warpType}Warp", 

69 storageClass="ExposureF", 

70 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

71 deferLoad=True, 

72 multiple=True, 

73 ) 

74 templateExposure = pipeBase.connectionTypes.Input( 

75 doc="Input coadded exposure, produced by previous call to AssembleCoadd", 

76 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}", 

77 storageClass="ExposureF", 

78 dimensions=("tract", "patch", "skymap", "band"), 

79 ) 

80 dcrCoadds = pipeBase.connectionTypes.Output( 

81 doc="Output coadded exposure, produced by stacking input warps", 

82 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

83 storageClass="ExposureF", 

84 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

85 multiple=True, 

86 ) 

87 dcrNImages = pipeBase.connectionTypes.Output( 

88 doc="Output image of number of input images per pixel", 

89 name="{outputCoaddName}Coadd_nImage", 

90 storageClass="ImageU", 

91 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

92 multiple=True, 

93 ) 

94 

95 def __init__(self, *, config=None): 

96 super().__init__(config=config) 

97 if not config.doWrite: 

98 self.outputs.remove("dcrCoadds") 

99 if not config.doNImage: 

100 self.outputs.remove("dcrNImages") 

101 # Remove outputs inherited from ``AssembleCoaddConnections`` that are 

102 # not used. 

103 self.outputs.remove("coaddExposure") 

104 self.outputs.remove("nImage") 

105 

106 

107class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, pipelineConnections=DcrAssembleCoaddConnections): 

108 dcrNumSubfilters = pexConfig.Field( 

109 dtype=int, 

110 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.", 

111 default=3, 

112 ) 

113 maxNumIter = pexConfig.Field( 

114 dtype=int, 

115 optional=True, 

116 doc="Maximum number of iterations of forward modeling.", 

117 default=None, 

118 ) 

119 minNumIter = pexConfig.Field( 

120 dtype=int, 

121 optional=True, 

122 doc="Minimum number of iterations of forward modeling.", 

123 default=None, 

124 ) 

125 convergenceThreshold = pexConfig.Field( 

126 dtype=float, 

127 doc="Target relative change in convergence between iterations of forward modeling.", 

128 default=0.001, 

129 ) 

130 useConvergence = pexConfig.Field( 

131 dtype=bool, 

132 doc="Use convergence test as a forward modeling end condition?" 

133 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations", 

134 default=True, 

135 ) 

136 baseGain = pexConfig.Field( 

137 dtype=float, 

138 optional=True, 

139 doc="Relative weight to give the new solution vs. the last solution when updating the model." 

140 "A value of 1.0 gives equal weight to both solutions." 

141 "Small values imply slower convergence of the solution, but can " 

142 "help prevent overshooting and failures in the fit." 

143 "If ``baseGain`` is None, a conservative gain " 

144 "will be calculated from the number of subfilters. ", 

145 default=None, 

146 ) 

147 useProgressiveGain = pexConfig.Field( 

148 dtype=bool, 

149 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 

150 "When calculating the next gain, we use up to 5 previous gains and convergence values." 

151 "Can be set to False to force the model to change at the rate of ``baseGain``. ", 

152 default=True, 

153 ) 

154 doAirmassWeight = pexConfig.Field( 

155 dtype=bool, 

156 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.", 

157 default=False, 

158 ) 

159 modelWeightsWidth = pexConfig.Field( 

160 dtype=float, 

161 doc="Width of the region around detected sources to include in the DcrModel.", 

162 default=3, 

163 ) 

164 useModelWeights = pexConfig.Field( 

165 dtype=bool, 

166 doc="Width of the region around detected sources to include in the DcrModel.", 

167 default=True, 

168 ) 

169 splitSubfilters = pexConfig.Field( 

170 dtype=bool, 

171 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter. Instead of at the midpoint", 

172 default=True, 

173 ) 

174 splitThreshold = pexConfig.Field( 

175 dtype=float, 

176 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 

177 "Set to 0 to always split the subfilters.", 

178 default=0.1, 

179 ) 

180 regularizeModelIterations = pexConfig.Field( 

181 dtype=float, 

182 doc="Maximum relative change of the model allowed between iterations. Set to zero to disable.", 

183 default=2.0, 

184 ) 

185 regularizeModelFrequency = pexConfig.Field( 

186 dtype=float, 

187 doc="Maximum relative change of the model allowed between subfilters. Set to zero to disable.", 

188 default=4.0, 

189 ) 

190 convergenceMaskPlanes = pexConfig.ListField( 

191 dtype=str, default=["DETECTED"], doc="Mask planes to use to calculate convergence." 

192 ) 

193 regularizationWidth = pexConfig.Field( 

194 dtype=int, default=2, doc="Minimum radius of a region to include in regularization, in pixels." 

195 ) 

196 imageInterpOrder = pexConfig.Field( 

197 dtype=int, 

198 doc="The order of the spline interpolation used to shift the image plane.", 

199 default=1, 

200 ) 

201 accelerateModel = pexConfig.Field( 

202 dtype=float, 

203 doc="Factor to amplify the differences between model planes by to speed convergence.", 

204 default=3, 

205 ) 

206 doCalculatePsf = pexConfig.Field( 

207 dtype=bool, 

208 doc="Set to detect stars and recalculate the PSF from the final coadd." 

209 "Otherwise the PSF is estimated from a selection of the best input exposures", 

210 default=False, 

211 ) 

212 detectPsfSources = pexConfig.ConfigurableField( 

213 target=measAlg.SourceDetectionTask, 

214 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.", 

215 ) 

216 measurePsfSources = pexConfig.ConfigurableField( 

217 target=SingleFrameMeasurementTask, 

218 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set.", 

219 ) 

220 measurePsf = pexConfig.ConfigurableField( 

221 target=MeasurePsfTask, 

222 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.", 

223 ) 

224 effectiveWavelength = pexConfig.Field( 

225 doc="Effective wavelength of the filter, in nm." 

226 "Required if transmission curves aren't used." 

227 "Support for using transmission curves is to be added in DM-13668.", 

228 dtype=float, 

229 ) 

230 bandwidth = pexConfig.Field( 

231 doc="Bandwidth of the physical filter, in nm." 

232 "Required if transmission curves aren't used." 

233 "Support for using transmission curves is to be added in DM-13668.", 

234 dtype=float, 

235 ) 

236 

237 def setDefaults(self): 

238 CompareWarpAssembleCoaddConfig.setDefaults(self) 

239 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask) 

240 self.doNImage = True 

241 self.assembleStaticSkyModel.warpType = self.warpType 

242 # The deepCoadd and nImage files will be overwritten by this Task, so 

243 # don't write them the first time. 

244 self.assembleStaticSkyModel.doNImage = False 

245 self.assembleStaticSkyModel.doWrite = False 

246 self.detectPsfSources.returnOriginalFootprints = False 

247 self.detectPsfSources.thresholdPolarity = "positive" 

248 # Only use bright sources for PSF measurement 

249 self.detectPsfSources.thresholdValue = 50 

250 self.detectPsfSources.nSigmaToGrow = 2 

251 # A valid star for PSF measurement should at least fill 5x5 pixels 

252 self.detectPsfSources.minPixels = 25 

253 # Use the variance plane to calculate signal to noise 

254 self.detectPsfSources.thresholdType = "pixel_stdev" 

255 # Ensure psf candidate size is as large as piff psf size. 

256 if ( 

257 self.doCalculatePsf 

258 and self.measurePsf.psfDeterminer.name == "piff" 

259 and self.psfDeterminer["piff"].kernelSize > self.makePsfCandidates.kernelSize 

260 ): 

261 self.makePsfCandidates.kernelSize = self.psfDeterminer["piff"].kernelSize 

262 

263 

264class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask): 

265 """Assemble DCR coadded images from a set of warps. 

266 

267 Attributes 

268 ---------- 

269 bufferSize : `int` 

270 The number of pixels to grow each subregion by to allow for DCR. 

271 

272 Notes 

273 ----- 

274 As with AssembleCoaddTask, we want to assemble a coadded image from a set 

275 of Warps (also called coadded temporary exposures), including the effects 

276 of Differential Chromatic Refraction (DCR). 

277 For full details of the mathematics and algorithm, please see 

278 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 

279 

280 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 

281 each subfilter used in the iterative calculation. 

282 It begins by dividing the bandpass-defining filter into N equal bandwidth 

283 "subfilters", and divides the flux in each pixel from an initial coadd 

284 equally into each as a "dcrModel". Because the airmass and parallactic 

285 angle of each individual exposure is known, we can calculate the shift 

286 relative to the center of the band in each subfilter due to DCR. For each 

287 exposure we apply this shift as a linear transformation to the dcrModels 

288 and stack the results to produce a DCR-matched exposure. The matched 

289 exposures are subtracted from the input exposures to produce a set of 

290 residual images, and these residuals are reverse shifted for each 

291 exposures' subfilters and stacked. The shifted and stacked residuals are 

292 added to the dcrModels to produce a new estimate of the flux in each pixel 

293 within each subfilter. The dcrModels are solved for iteratively, which 

294 continues until the solution from a new iteration improves by less than 

295 a set percentage, or a maximum number of iterations is reached. 

296 Two forms of regularization are employed to reduce unphysical results. 

297 First, the new solution is averaged with the solution from the previous 

298 iteration, which mitigates oscillating solutions where the model 

299 overshoots with alternating very high and low values. 

300 Second, a common degeneracy when the data have a limited range of airmass 

301 or parallactic angle values is for one subfilter to be fit with very low or 

302 negative values, while another subfilter is fit with very high values. This 

303 typically appears in the form of holes next to sources in one subfilter, 

304 and corresponding extended wings in another. Because each subfilter has 

305 a narrow bandwidth we assume that physical sources that are above the noise 

306 level will not vary in flux by more than a factor of `frequencyClampFactor` 

307 between subfilters, and pixels that have flux deviations larger than that 

308 factor will have the excess flux distributed evenly among all subfilters. 

309 If `splitSubfilters` is set, then each subfilter will be further sub- 

310 divided during the forward modeling step (only). This approximates using 

311 a higher number of subfilters that may be necessary for high airmass 

312 observations, but does not increase the number of free parameters in the 

313 fit. This is needed when there are high airmass observations which would 

314 otherwise have significant DCR even within a subfilter. Because calculating 

315 the shifted images takes most of the time, splitting the subfilters is 

316 turned off by way of the `splitThreshold` option for low-airmass 

317 observations that do not suffer from DCR within a subfilter. 

318 """ 

319 

320 ConfigClass = DcrAssembleCoaddConfig 

321 _DefaultName = "dcrAssembleCoadd" 

322 

323 def __init__(self, *args, **kwargs): 

324 super().__init__(*args, **kwargs) 

325 if self.config.doCalculatePsf: 

326 self.schema = afwTable.SourceTable.makeMinimalSchema() 

327 self.makeSubtask("detectPsfSources", schema=self.schema) 

328 self.makeSubtask("measurePsfSources", schema=self.schema) 

329 self.makeSubtask("measurePsf", schema=self.schema) 

330 

331 @utils.inheritDoc(pipeBase.PipelineTask) 

332 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

333 # Docstring to be formatted with info from PipelineTask.runQuantum 

334 """ 

335 Notes 

336 ----- 

337 Assemble a coadd from a set of Warps. 

338 """ 

339 inputData = butlerQC.get(inputRefs) 

340 

341 # Construct skyInfo expected by run 

342 # Do not remove skyMap from inputData in case _makeSupplementaryData 

343 # needs it. 

344 skyMap = inputData["skyMap"] 

345 outputDataId = butlerQC.quantum.dataId 

346 

347 inputData["skyInfo"] = makeSkyInfo( 

348 skyMap, tractId=outputDataId["tract"], patchId=outputDataId["patch"] 

349 ) 

350 

351 # Construct list of input Deferred Datasets 

352 warpRefList = inputData["inputWarps"] 

353 

354 inputs = self.prepareInputs(warpRefList) 

355 self.log.info("Found %d %s", len(inputs.tempExpRefList), self.getTempExpDatasetName(self.warpType)) 

356 if len(inputs.tempExpRefList) == 0: 

357 self.log.warning("No coadd temporary exposures found") 

358 return 

359 

360 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

361 retStruct = self.run( 

362 inputData["skyInfo"], 

363 inputs.tempExpRefList, 

364 inputs.imageScalerList, 

365 inputs.weightList, 

366 supplementaryData=supplementaryData, 

367 ) 

368 

369 inputData.setdefault("brightObjectMask", None) 

370 for subfilter in range(self.config.dcrNumSubfilters): 

371 # Use the PSF of the stacked dcrModel, and do not recalculate the 

372 # PSF for each subfilter 

373 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf()) 

374 self.processResults(retStruct.dcrCoadds[subfilter], inputData["brightObjectMask"], outputDataId) 

375 

376 if self.config.doWrite: 

377 butlerQC.put(retStruct, outputRefs) 

378 return retStruct 

379 

380 @utils.inheritDoc(AssembleCoaddTask) 

381 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

382 """Load the previously-generated template coadd. 

383 

384 Returns 

385 ------- 

386 templateCoadd : `lsst.pipe.base.Struct` 

387 Results as a struct with attributes: 

388 

389 ``templateCoadd`` 

390 Coadded exposure (`lsst.afw.image.ExposureF`). 

391 """ 

392 templateCoadd = butlerQC.get(inputRefs.templateExposure) 

393 

394 return pipeBase.Struct(templateCoadd=templateCoadd) 

395 

396 def measureCoaddPsf(self, coaddExposure): 

397 """Detect sources on the coadd exposure and measure the final PSF. 

398 

399 Parameters 

400 ---------- 

401 coaddExposure : `lsst.afw.image.Exposure` 

402 The final coadded exposure. 

403 """ 

404 table = afwTable.SourceTable.make(self.schema) 

405 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False) 

406 coaddSources = detResults.sources 

407 self.measurePsfSources.run(measCat=coaddSources, exposure=coaddExposure) 

408 # Measure the PSF on the stacked subfilter coadds if possible. 

409 # We should already have a decent estimate of the coadd PSF, however, 

410 # so in case of any errors simply log them as a warning and use the 

411 # default PSF. 

412 try: 

413 psfResults = self.measurePsf.run(coaddExposure, coaddSources) 

414 except Exception as e: 

415 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e) 

416 else: 

417 coaddExposure.setPsf(psfResults.psf) 

418 

419 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList): 

420 """Prepare the DCR coadd by iterating through the visitInfo of the 

421 input warps. 

422 

423 Sets the property ``bufferSize``. 

424 

425 Parameters 

426 ---------- 

427 templateCoadd : `lsst.afw.image.ExposureF` 

428 The initial coadd exposure before accounting for DCR. 

429 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

430 The data references to the input warped exposures. 

431 weightList : `list` of `float` 

432 The weight to give each input exposure in the coadd. 

433 Will be modified in place if ``doAirmassWeight`` is set. 

434 

435 Returns 

436 ------- 

437 dcrModels : `lsst.pipe.tasks.DcrModel` 

438 Best fit model of the true sky after correcting chromatic effects. 

439 

440 Raises 

441 ------ 

442 NotImplementedError 

443 If ``lambdaMin`` is missing from the Mapper class of the obs 

444 package being used. 

445 """ 

446 sigma2fwhm = 2.0 * np.sqrt(2.0 * np.log(2.0)) 

447 filterLabel = templateCoadd.getFilter() 

448 dcrShifts = [] 

449 airmassDict = {} 

450 angleDict = {} 

451 psfSizeDict = {} 

452 for visitNum, warpExpRef in enumerate(warpRefList): 

453 visitInfo = warpExpRef.get(component="visitInfo") 

454 psf = warpExpRef.get(component="psf") 

455 visit = warpExpRef.dataId["visit"] 

456 # Just need a rough estimate; average positions are fine 

457 psfAvgPos = psf.getAveragePosition() 

458 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius() * sigma2fwhm 

459 airmass = visitInfo.getBoresightAirmass() 

460 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees() 

461 airmassDict[visit] = airmass 

462 angleDict[visit] = parallacticAngle 

463 psfSizeDict[visit] = psfSize 

464 if self.config.doAirmassWeight: 

465 weightList[visitNum] *= airmass 

466 dcrShifts.append( 

467 np.max( 

468 np.abs( 

469 calculateDcr( 

470 visitInfo, 

471 templateCoadd.getWcs(), 

472 self.config.effectiveWavelength, 

473 self.config.bandwidth, 

474 self.config.dcrNumSubfilters, 

475 ) 

476 ) 

477 ) 

478 ) 

479 self.log.info("Selected airmasses:\n%s", airmassDict) 

480 self.log.info("Selected parallactic angles:\n%s", angleDict) 

481 self.log.info("Selected PSF sizes:\n%s", psfSizeDict) 

482 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1) 

483 try: 

484 psf = self.selectCoaddPsf(templateCoadd, warpRefList) 

485 except Exception as e: 

486 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e) 

487 else: 

488 psf = templateCoadd.getPsf() 

489 dcrModels = DcrModel.fromImage( 

490 templateCoadd.maskedImage, 

491 self.config.dcrNumSubfilters, 

492 effectiveWavelength=self.config.effectiveWavelength, 

493 bandwidth=self.config.bandwidth, 

494 wcs=templateCoadd.getWcs(), 

495 filterLabel=filterLabel, 

496 psf=psf, 

497 ) 

498 return dcrModels 

499 

500 @timeMethod 

501 def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None): 

502 r"""Assemble the coadd. 

503 

504 Requires additional inputs Struct ``supplementaryData`` to contain a 

505 ``templateCoadd`` that serves as the model of the static sky. 

506 

507 Find artifacts and apply them to the warps' masks creating a list of 

508 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

509 plane then pass these alternative masks to the base class's assemble 

510 method. 

511 

512 Divide the ``templateCoadd`` evenly between each subfilter of a 

513 ``DcrModel`` as the starting best estimate of the true wavelength- 

514 dependent sky. Forward model the ``DcrModel`` using the known 

515 chromatic effects in each subfilter and calculate a convergence metric 

516 based on how well the modeled template matches the input warps. If 

517 the convergence has not yet reached the desired threshold, then shift 

518 and stack the residual images to build a new ``DcrModel``. Apply 

519 conditioning to prevent oscillating solutions between iterations or 

520 between subfilters. 

521 

522 Once the ``DcrModel`` reaches convergence or the maximum number of 

523 iterations has been reached, fill the metadata for each subfilter 

524 image and make them proper ``coaddExposure``\ s. 

525 

526 Parameters 

527 ---------- 

528 skyInfo : `lsst.pipe.base.Struct` 

529 Patch geometry information, from getSkyInfo 

530 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

531 The data references to the input warped exposures. 

532 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

533 The image scalars correct for the zero point of the exposures. 

534 weightList : `list` of `float` 

535 The weight to give each input exposure in the coadd 

536 supplementaryData : `lsst.pipe.base.Struct` 

537 Result struct returned by ``_makeSupplementaryData`` with 

538 attributes: 

539 

540 ``templateCoadd`` 

541 Coadded exposure (`lsst.afw.image.Exposure`). 

542 

543 Returns 

544 ------- 

545 result : `lsst.pipe.base.Struct` 

546 Results as a struct with attributes: 

547 

548 ``coaddExposure`` 

549 Coadded exposure (`lsst.afw.image.Exposure`). 

550 ``nImage`` 

551 Exposure count image (`lsst.afw.image.ImageU`). 

552 ``dcrCoadds`` 

553 `list` of coadded exposures for each subfilter. 

554 ``dcrNImages`` 

555 `list` of exposure count images for each subfilter. 

556 """ 

557 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters 

558 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters * 3 

559 templateCoadd = supplementaryData.templateCoadd 

560 baseMask = templateCoadd.mask.clone() 

561 # The variance plane is for each subfilter 

562 # and should be proportionately lower than the full-band image 

563 baseVariance = templateCoadd.variance.clone() 

564 baseVariance /= self.config.dcrNumSubfilters 

565 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList) 

566 # Note that the mask gets cleared in ``findArtifacts``, but we want to 

567 # preserve the mask. 

568 templateCoadd.setMask(baseMask) 

569 badMaskPlanes = self.config.badMaskPlanes[:] 

570 # Note that is important that we do not add "CLIPPED" to 

571 # ``badMaskPlanes``. This is because pixels in observations that are 

572 # significantly affected by DCR are likely to have many pixels that are 

573 # both "DETECTED" and "CLIPPED", but those are necessary to constrain 

574 # the DCR model. 

575 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes) 

576 

577 stats = self.prepareStats(mask=badPixelMask) 

578 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList) 

579 if self.config.doNImage: 

580 dcrNImages, dcrWeights = self.calculateNImage( 

581 dcrModels, skyInfo.bbox, warpRefList, spanSetMaskList, stats.ctrl 

582 ) 

583 nImage = afwImage.ImageU(skyInfo.bbox) 

584 # Note that this nImage will be a factor of dcrNumSubfilters higher 

585 # than the nImage returned by assembleCoadd for most pixels. This 

586 # is because each subfilter may have a different nImage, and 

587 # fractional values are not allowed. 

588 for dcrNImage in dcrNImages: 

589 nImage += dcrNImage 

590 else: 

591 dcrNImages = None 

592 

593 subregionSize = geom.Extent2I(*self.config.subregionSize) 

594 nSubregions = ceil(skyInfo.bbox.getHeight() / subregionSize[1]) * ceil( 

595 skyInfo.bbox.getWidth() / subregionSize[0] 

596 ) 

597 subIter = 0 

598 for subBBox in subBBoxIter(skyInfo.bbox, subregionSize): 

599 modelIter = 0 

600 subIter += 1 

601 self.log.info( 

602 "Computing coadd over patch %s subregion %s of %s: %s", 

603 skyInfo.patchInfo.getIndex(), 

604 subIter, 

605 nSubregions, 

606 subBBox, 

607 ) 

608 dcrBBox = geom.Box2I(subBBox) 

609 dcrBBox.grow(self.bufferSize) 

610 dcrBBox.clip(dcrModels.bbox) 

611 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox) 

612 subExposures = self.loadSubExposures( 

613 dcrBBox, stats.ctrl, warpRefList, imageScalerList, spanSetMaskList 

614 ) 

615 convergenceMetric = self.calculateConvergence( 

616 dcrModels, subExposures, subBBox, warpRefList, weightList, stats.ctrl 

617 ) 

618 self.log.info("Initial convergence : %s", convergenceMetric) 

619 convergenceList = [convergenceMetric] 

620 gainList = [] 

621 convergenceCheck = 1.0 

622 refImage = templateCoadd.image 

623 while convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter: 

624 gain = self.calculateGain(convergenceList, gainList) 

625 self.dcrAssembleSubregion( 

626 dcrModels, 

627 subExposures, 

628 subBBox, 

629 dcrBBox, 

630 warpRefList, 

631 stats.ctrl, 

632 convergenceMetric, 

633 gain, 

634 modelWeights, 

635 refImage, 

636 dcrWeights, 

637 ) 

638 if self.config.useConvergence: 

639 convergenceMetric = self.calculateConvergence( 

640 dcrModels, subExposures, subBBox, warpRefList, weightList, stats.ctrl 

641 ) 

642 if convergenceMetric == 0: 

643 self.log.warning( 

644 "Coadd patch %s subregion %s had convergence metric of 0.0 which is " 

645 "most likely due to there being no valid data in the region.", 

646 skyInfo.patchInfo.getIndex(), 

647 subIter, 

648 ) 

649 break 

650 convergenceCheck = (convergenceList[-1] - convergenceMetric) / convergenceMetric 

651 if (convergenceCheck < 0) & (modelIter > minNumIter): 

652 self.log.warning( 

653 "Coadd patch %s subregion %s diverged before reaching maximum " 

654 "iterations or desired convergence improvement of %s." 

655 " Divergence: %s", 

656 skyInfo.patchInfo.getIndex(), 

657 subIter, 

658 self.config.convergenceThreshold, 

659 convergenceCheck, 

660 ) 

661 break 

662 convergenceList.append(convergenceMetric) 

663 if modelIter > maxNumIter: 

664 if self.config.useConvergence: 

665 self.log.warning( 

666 "Coadd patch %s subregion %s reached maximum iterations " 

667 "before reaching desired convergence improvement of %s." 

668 " Final convergence improvement: %s", 

669 skyInfo.patchInfo.getIndex(), 

670 subIter, 

671 self.config.convergenceThreshold, 

672 convergenceCheck, 

673 ) 

674 break 

675 

676 if self.config.useConvergence: 

677 self.log.info( 

678 "Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)", 

679 modelIter, 

680 convergenceMetric, 

681 100.0 * convergenceCheck, 

682 gain, 

683 ) 

684 modelIter += 1 

685 else: 

686 if self.config.useConvergence: 

687 self.log.info( 

688 "Coadd patch %s subregion %s finished with " 

689 "convergence metric %s after %s iterations", 

690 skyInfo.patchInfo.getIndex(), 

691 subIter, 

692 convergenceMetric, 

693 modelIter, 

694 ) 

695 else: 

696 self.log.info( 

697 "Coadd patch %s subregion %s finished after %s iterations", 

698 skyInfo.patchInfo.getIndex(), 

699 subIter, 

700 modelIter, 

701 ) 

702 if self.config.useConvergence and convergenceMetric > 0: 

703 self.log.info( 

704 "Final convergence improvement was %.4f%% overall", 

705 100 * (convergenceList[0] - convergenceMetric) / convergenceMetric, 

706 ) 

707 

708 dcrCoadds = self.fillCoadd( 

709 dcrModels, 

710 skyInfo, 

711 warpRefList, 

712 weightList, 

713 calibration=self.scaleZeroPoint.getPhotoCalib(), 

714 coaddInputs=templateCoadd.getInfo().getCoaddInputs(), 

715 mask=baseMask, 

716 variance=baseVariance, 

717 ) 

718 coaddExposure = self.stackCoadd(dcrCoadds) 

719 return pipeBase.Struct( 

720 coaddExposure=coaddExposure, nImage=nImage, dcrCoadds=dcrCoadds, dcrNImages=dcrNImages 

721 ) 

722 

723 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl): 

724 """Calculate the number of exposures contributing to each subfilter. 

725 

726 Parameters 

727 ---------- 

728 dcrModels : `lsst.pipe.tasks.DcrModel` 

729 Best fit model of the true sky after correcting chromatic effects. 

730 bbox : `lsst.geom.box.Box2I` 

731 Bounding box of the patch to coadd. 

732 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

733 The data references to the input warped exposures. 

734 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None` 

735 Each element of the `dict` contains the new mask plane name 

736 (e.g. "CLIPPED and/or "NO_DATA") as the key, 

737 and the list of SpanSets to apply to the mask. 

738 statsCtrl : `lsst.afw.math.StatisticsControl` 

739 Statistics control object for coadd 

740 

741 Returns 

742 ------- 

743 dcrNImages : `list` of `lsst.afw.image.ImageU` 

744 List of exposure count images for each subfilter. 

745 dcrWeights : `list` of `lsst.afw.image.ImageF` 

746 Per-pixel weights for each subfilter. 

747 Equal to 1/(number of unmasked images contributing to each pixel). 

748 """ 

749 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

750 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

751 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList): 

752 exposure = warpExpRef.get(parameters={"bbox": bbox}) 

753 visitInfo = exposure.getInfo().getVisitInfo() 

754 wcs = exposure.getInfo().getWcs() 

755 mask = exposure.mask 

756 if altMaskSpans is not None: 

757 self.applyAltMaskPlanes(mask, altMaskSpans) 

758 weightImage = np.zeros_like(exposure.image.array) 

759 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.0 

760 # The weights must be shifted in exactly the same way as the 

761 # residuals, because they will be used as the denominator in the 

762 # weighted average of residuals. 

763 weightsGenerator = self.dcrResiduals( 

764 weightImage, visitInfo, wcs, dcrModels.effectiveWavelength, dcrModels.bandwidth 

765 ) 

766 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights): 

767 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype) 

768 dcrWeight.array += shiftedWeights 

769 # Exclude any pixels that don't have at least one exposure contributing 

770 # in all subfilters 

771 weightsThreshold = 1.0 

772 goodPix = dcrWeights[0].array > weightsThreshold 

773 for weights in dcrWeights[1:]: 

774 goodPix = (weights.array > weightsThreshold) & goodPix 

775 for subfilter in range(self.config.dcrNumSubfilters): 

776 dcrWeights[subfilter].array[goodPix] = 1.0 / dcrWeights[subfilter].array[goodPix] 

777 dcrWeights[subfilter].array[~goodPix] = 0.0 

778 dcrNImages[subfilter].array[~goodPix] = 0 

779 return (dcrNImages, dcrWeights) 

780 

781 def dcrAssembleSubregion( 

782 self, 

783 dcrModels, 

784 subExposures, 

785 bbox, 

786 dcrBBox, 

787 warpRefList, 

788 statsCtrl, 

789 convergenceMetric, 

790 gain, 

791 modelWeights, 

792 refImage, 

793 dcrWeights, 

794 ): 

795 """Assemble the DCR coadd for a sub-region. 

796 

797 Build a DCR-matched template for each input exposure, then shift the 

798 residuals according to the DCR in each subfilter. 

799 Stack the shifted residuals and apply them as a correction to the 

800 solution from the previous iteration. 

801 Restrict the new model solutions from varying by more than a factor of 

802 `modelClampFactor` from the last solution, and additionally restrict 

803 the individual subfilter models from varying by more than a factor of 

804 `frequencyClampFactor` from their average. 

805 Finally, mitigate potentially oscillating solutions by averaging the 

806 new solution with the solution from the previous iteration, weighted by 

807 their convergence metric. 

808 

809 Parameters 

810 ---------- 

811 dcrModels : `lsst.pipe.tasks.DcrModel` 

812 Best fit model of the true sky after correcting chromatic effects. 

813 subExposures : `dict` of `lsst.afw.image.ExposureF` 

814 The pre-loaded exposures for the current subregion. 

815 bbox : `lsst.geom.box.Box2I` 

816 Bounding box of the subregion to coadd. 

817 dcrBBox : `lsst.geom.box.Box2I` 

818 Sub-region of the coadd which includes a buffer to allow for DCR. 

819 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

820 The data references to the input warped exposures. 

821 statsCtrl : `lsst.afw.math.StatisticsControl` 

822 Statistics control object for coadd. 

823 convergenceMetric : `float` 

824 Quality of fit metric for the matched templates of the input 

825 images. 

826 gain : `float`, optional 

827 Relative weight to give the new solution when updating the model. 

828 modelWeights : `numpy.ndarray` or `float` 

829 A 2D array of weight values that tapers smoothly to zero away from 

830 detected sources. Set to a placeholder value of 1.0 if 

831 ``self.config.useModelWeights`` is False. 

832 refImage : `lsst.afw.image.Image` 

833 A reference image used to supply the default pixel values. 

834 dcrWeights : `list` of `lsst.afw.image.Image` 

835 Per-pixel weights for each subfilter. 

836 Equal to 1/(number of unmasked images contributing to each pixel). 

837 """ 

838 residualGeneratorList = [] 

839 

840 for warpExpRef in warpRefList: 

841 visit = warpExpRef.dataId["visit"] 

842 exposure = subExposures[visit] 

843 visitInfo = exposure.getInfo().getVisitInfo() 

844 wcs = exposure.getInfo().getWcs() 

845 templateImage = dcrModels.buildMatchedTemplate( 

846 exposure=exposure, 

847 bbox=exposure.getBBox(), 

848 order=self.config.imageInterpOrder, 

849 splitSubfilters=self.config.splitSubfilters, 

850 splitThreshold=self.config.splitThreshold, 

851 amplifyModel=self.config.accelerateModel, 

852 ) 

853 residual = exposure.image.array - templateImage.array 

854 # Note that the variance plane here is used to store weights, not 

855 # the actual variance 

856 residual *= exposure.variance.array 

857 # The residuals are stored as a list of generators. 

858 # This allows the residual for a given subfilter and exposure to be 

859 # created on the fly, instead of needing to store them all in 

860 # memory. 

861 residualGeneratorList.append( 

862 self.dcrResiduals( 

863 residual, visitInfo, wcs, dcrModels.effectiveWavelength, dcrModels.bandwidth 

864 ) 

865 ) 

866 

867 dcrSubModelOut = self.newModelFromResidual( 

868 dcrModels, 

869 residualGeneratorList, 

870 dcrBBox, 

871 statsCtrl, 

872 gain=gain, 

873 modelWeights=modelWeights, 

874 refImage=refImage, 

875 dcrWeights=dcrWeights, 

876 ) 

877 dcrModels.assign(dcrSubModelOut, bbox) 

878 

879 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth): 

880 """Prepare a residual image for stacking in each subfilter by applying 

881 the reverse DCR shifts. 

882 

883 Parameters 

884 ---------- 

885 residual : `numpy.ndarray` 

886 The residual masked image for one exposure, 

887 after subtracting the matched template. 

888 visitInfo : `lsst.afw.image.VisitInfo` 

889 Metadata for the exposure. 

890 wcs : `lsst.afw.geom.SkyWcs` 

891 Coordinate system definition (wcs) for the exposure. 

892 

893 Yields 

894 ------ 

895 residualImage : `numpy.ndarray` 

896 The residual image for the next subfilter, shifted for DCR. 

897 """ 

898 if self.config.imageInterpOrder > 1: 

899 # Pre-calculate the spline-filtered residual image, so that step 

900 # can be skipped in the shift calculation in `applyDcr`. 

901 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder) 

902 else: 

903 # No need to prefilter if order=1 (it will also raise an error) 

904 filteredResidual = residual 

905 # Note that `splitSubfilters` is always turned off in the reverse 

906 # direction. This option introduces additional blurring if applied to 

907 # the residuals. 

908 dcrShift = calculateDcr( 

909 visitInfo, 

910 wcs, 

911 effectiveWavelength, 

912 bandwidth, 

913 self.config.dcrNumSubfilters, 

914 splitSubfilters=False, 

915 ) 

916 for dcr in dcrShift: 

917 yield applyDcr( 

918 filteredResidual, 

919 dcr, 

920 useInverse=True, 

921 splitSubfilters=False, 

922 doPrefilter=False, 

923 order=self.config.imageInterpOrder, 

924 ) 

925 

926 def newModelFromResidual( 

927 self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights 

928 ): 

929 """Calculate a new DcrModel from a set of image residuals. 

930 

931 Parameters 

932 ---------- 

933 dcrModels : `lsst.pipe.tasks.DcrModel` 

934 Current model of the true sky after correcting chromatic effects. 

935 residualGeneratorList : `generator` of `numpy.ndarray` 

936 The residual image for the next subfilter, shifted for DCR. 

937 dcrBBox : `lsst.geom.box.Box2I` 

938 Sub-region of the coadd which includes a buffer to allow for DCR. 

939 statsCtrl : `lsst.afw.math.StatisticsControl` 

940 Statistics control object for coadd. 

941 gain : `float` 

942 Relative weight to give the new solution when updating the model. 

943 modelWeights : `numpy.ndarray` or `float` 

944 A 2D array of weight values that tapers smoothly to zero away from 

945 detected sources. Set to a placeholder value of 1.0 if 

946 ``self.config.useModelWeights`` is False. 

947 refImage : `lsst.afw.image.Image` 

948 A reference image used to supply the default pixel values. 

949 dcrWeights : `list` of `lsst.afw.image.Image` 

950 Per-pixel weights for each subfilter. 

951 Equal to 1/(number of unmasked images contributing to each pixel). 

952 

953 Returns 

954 ------- 

955 dcrModel : `lsst.pipe.tasks.DcrModel` 

956 New model of the true sky after correcting chromatic effects. 

957 """ 

958 newModelImages = [] 

959 for subfilter, model in enumerate(dcrModels): 

960 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList] 

961 residual = np.sum(residualsList, axis=0) 

962 residual *= dcrWeights[subfilter][dcrBBox].array 

963 # `MaskedImage`s only support in-place addition, so rename for 

964 # readability. 

965 newModel = model[dcrBBox].clone() 

966 newModel.array += residual 

967 # Catch any invalid values 

968 badPixels = ~np.isfinite(newModel.array) 

969 newModel.array[badPixels] = model[dcrBBox].array[badPixels] 

970 if self.config.regularizeModelIterations > 0: 

971 dcrModels.regularizeModelIter( 

972 subfilter, 

973 newModel, 

974 dcrBBox, 

975 self.config.regularizeModelIterations, 

976 self.config.regularizationWidth, 

977 ) 

978 newModelImages.append(newModel) 

979 if self.config.regularizeModelFrequency > 0: 

980 dcrModels.regularizeModelFreq( 

981 newModelImages, 

982 dcrBBox, 

983 statsCtrl, 

984 self.config.regularizeModelFrequency, 

985 self.config.regularizationWidth, 

986 ) 

987 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain) 

988 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights) 

989 return DcrModel( 

990 newModelImages, 

991 dcrModels.filter, 

992 dcrModels.effectiveWavelength, 

993 dcrModels.bandwidth, 

994 dcrModels.psf, 

995 dcrModels.mask, 

996 dcrModels.variance, 

997 ) 

998 

999 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl): 

1000 """Calculate a quality of fit metric for the matched templates. 

1001 

1002 Parameters 

1003 ---------- 

1004 dcrModels : `lsst.pipe.tasks.DcrModel` 

1005 Best fit model of the true sky after correcting chromatic effects. 

1006 subExposures : `dict` of `lsst.afw.image.ExposureF` 

1007 The pre-loaded exposures for the current subregion. 

1008 bbox : `lsst.geom.box.Box2I` 

1009 Sub-region to coadd. 

1010 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1011 The data references to the input warped exposures. 

1012 weightList : `list` of `float` 

1013 The weight to give each input exposure in the coadd. 

1014 statsCtrl : `lsst.afw.math.StatisticsControl` 

1015 Statistics control object for coadd. 

1016 

1017 Returns 

1018 ------- 

1019 convergenceMetric : `float` 

1020 Quality of fit metric for all input exposures, within the 

1021 sub-region. 

1022 """ 

1023 significanceImage = np.abs(dcrModels.getReferenceImage(bbox)) 

1024 nSigma = 3.0 

1025 significanceImage += nSigma * dcrModels.calculateNoiseCutoff( 

1026 dcrModels[1], statsCtrl, bufferSize=self.bufferSize 

1027 ) 

1028 if np.max(significanceImage) == 0: 

1029 significanceImage += 1.0 

1030 weight = 0 

1031 metric = 0.0 

1032 metricList = {} 

1033 for warpExpRef, expWeight in zip(warpRefList, weightList): 

1034 visit = warpExpRef.dataId["visit"] 

1035 exposure = subExposures[visit][bbox] 

1036 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl) 

1037 metric += singleMetric 

1038 metricList[visit] = singleMetric 

1039 weight += 1.0 

1040 self.log.info("Individual metrics:\n%s", metricList) 

1041 return 1.0 if weight == 0.0 else metric / weight 

1042 

1043 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl): 

1044 """Calculate a quality of fit metric for a single matched template. 

1045 

1046 Parameters 

1047 ---------- 

1048 dcrModels : `lsst.pipe.tasks.DcrModel` 

1049 Best fit model of the true sky after correcting chromatic effects. 

1050 exposure : `lsst.afw.image.ExposureF` 

1051 The input warped exposure to evaluate. 

1052 significanceImage : `numpy.ndarray` 

1053 Array of weights for each pixel corresponding to its significance 

1054 for the convergence calculation. 

1055 statsCtrl : `lsst.afw.math.StatisticsControl` 

1056 Statistics control object for coadd. 

1057 

1058 Returns 

1059 ------- 

1060 convergenceMetric : `float` 

1061 Quality of fit metric for one exposure, within the sub-region. 

1062 """ 

1063 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1064 templateImage = dcrModels.buildMatchedTemplate( 

1065 exposure=exposure, 

1066 bbox=exposure.getBBox(), 

1067 order=self.config.imageInterpOrder, 

1068 splitSubfilters=self.config.splitSubfilters, 

1069 splitThreshold=self.config.splitThreshold, 

1070 amplifyModel=self.config.accelerateModel, 

1071 ) 

1072 diffVals = np.abs(exposure.image.array - templateImage.array) * significanceImage 

1073 refVals = np.abs(exposure.image.array + templateImage.array) * significanceImage / 2.0 

1074 

1075 finitePixels = np.isfinite(diffVals) 

1076 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0 

1077 convergeMaskPixels = exposure.mask.array & convergeMask > 0 

1078 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels 

1079 if np.sum(usePixels) == 0: 

1080 metric = 0.0 

1081 else: 

1082 diffUse = diffVals[usePixels] 

1083 refUse = refVals[usePixels] 

1084 metric = np.sum(diffUse / np.median(diffUse)) / np.sum(refUse / np.median(diffUse)) 

1085 return metric 

1086 

1087 def stackCoadd(self, dcrCoadds): 

1088 """Add a list of sub-band coadds together. 

1089 

1090 Parameters 

1091 ---------- 

1092 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1093 A list of coadd exposures, each exposure containing 

1094 the model for one subfilter. 

1095 

1096 Returns 

1097 ------- 

1098 coaddExposure : `lsst.afw.image.ExposureF` 

1099 A single coadd exposure that is the sum of the sub-bands. 

1100 """ 

1101 coaddExposure = dcrCoadds[0].clone() 

1102 for coadd in dcrCoadds[1:]: 

1103 coaddExposure.maskedImage += coadd.maskedImage 

1104 return coaddExposure 

1105 

1106 def fillCoadd( 

1107 self, 

1108 dcrModels, 

1109 skyInfo, 

1110 warpRefList, 

1111 weightList, 

1112 calibration=None, 

1113 coaddInputs=None, 

1114 mask=None, 

1115 variance=None, 

1116 ): 

1117 """Create a list of coadd exposures from a list of masked images. 

1118 

1119 Parameters 

1120 ---------- 

1121 dcrModels : `lsst.pipe.tasks.DcrModel` 

1122 Best fit model of the true sky after correcting chromatic effects. 

1123 skyInfo : `lsst.pipe.base.Struct` 

1124 Patch geometry information, from getSkyInfo. 

1125 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1126 The data references to the input warped exposures. 

1127 weightList : `list` of `float` 

1128 The weight to give each input exposure in the coadd. 

1129 calibration : `lsst.afw.Image.PhotoCalib`, optional 

1130 Scale factor to set the photometric calibration of an exposure. 

1131 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 

1132 A record of the observations that are included in the coadd. 

1133 mask : `lsst.afw.image.Mask`, optional 

1134 Optional mask to override the values in the final coadd. 

1135 variance : `lsst.afw.image.Image`, optional 

1136 Optional variance plane to override the values in the final coadd. 

1137 

1138 Returns 

1139 ------- 

1140 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1141 A list of coadd exposures, each exposure containing 

1142 the model for one subfilter. 

1143 """ 

1144 dcrCoadds = [] 

1145 refModel = dcrModels.getReferenceImage() 

1146 for model in dcrModels: 

1147 if self.config.accelerateModel > 1: 

1148 model.array = (model.array - refModel) * self.config.accelerateModel + refModel 

1149 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

1150 if calibration is not None: 

1151 coaddExposure.setPhotoCalib(calibration) 

1152 if coaddInputs is not None: 

1153 coaddExposure.getInfo().setCoaddInputs(coaddInputs) 

1154 # Set the metadata for the coadd, including PSF and aperture 

1155 # corrections. 

1156 self.assembleMetadata(coaddExposure, warpRefList, weightList) 

1157 # Overwrite the PSF 

1158 coaddExposure.setPsf(dcrModels.psf) 

1159 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox]) 

1160 maskedImage = afwImage.MaskedImageF(dcrModels.bbox) 

1161 maskedImage.image = model 

1162 maskedImage.mask = dcrModels.mask 

1163 maskedImage.variance = dcrModels.variance 

1164 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox]) 

1165 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

1166 if mask is not None: 

1167 coaddExposure.setMask(mask) 

1168 if variance is not None: 

1169 coaddExposure.setVariance(variance) 

1170 dcrCoadds.append(coaddExposure) 

1171 return dcrCoadds 

1172 

1173 def calculateGain(self, convergenceList, gainList): 

1174 """Calculate the gain to use for the current iteration. 

1175 

1176 After calculating a new DcrModel, each value is averaged with the 

1177 value in the corresponding pixel from the previous iteration. This 

1178 reduces oscillating solutions that iterative techniques are plagued by, 

1179 and speeds convergence. By far the biggest changes to the model 

1180 happen in the first couple iterations, so we can also use a more 

1181 aggressive gain later when the model is changing slowly. 

1182 

1183 Parameters 

1184 ---------- 

1185 convergenceList : `list` of `float` 

1186 The quality of fit metric from each previous iteration. 

1187 gainList : `list` of `float` 

1188 The gains used in each previous iteration: appended with the new 

1189 gain value. 

1190 Gains are numbers between ``self.config.baseGain`` and 1. 

1191 

1192 Returns 

1193 ------- 

1194 gain : `float` 

1195 Relative weight to give the new solution when updating the model. 

1196 A value of 1.0 gives equal weight to both solutions. 

1197 

1198 Raises 

1199 ------ 

1200 ValueError 

1201 If ``len(convergenceList) != len(gainList)+1``. 

1202 """ 

1203 nIter = len(convergenceList) 

1204 if nIter != len(gainList) + 1: 

1205 raise ValueError( 

1206 "convergenceList (%d) must be one element longer than gainList (%d)." 

1207 % (len(convergenceList), len(gainList)) 

1208 ) 

1209 

1210 if self.config.baseGain is None: 

1211 # If ``baseGain`` is not set, calculate it from the number of DCR 

1212 # subfilters. The more subfilters being modeled, the lower the gain 

1213 # should be. 

1214 baseGain = 1.0 / (self.config.dcrNumSubfilters - 1) 

1215 else: 

1216 baseGain = self.config.baseGain 

1217 

1218 if self.config.useProgressiveGain and nIter > 2: 

1219 # To calculate the best gain to use, compare the past gains that 

1220 # have been used with the resulting convergences to estimate the 

1221 # best gain to use. Algorithmically, this is a Kalman filter. 

1222 # If forward modeling proceeds perfectly, the convergence metric 

1223 # should asymptotically approach a final value. We can estimate 

1224 # that value from the measured changes in convergence weighted by 

1225 # the gains used in each previous iteration. 

1226 estFinalConv = [ 

1227 ((1 + gainList[i]) * convergenceList[i + 1] - convergenceList[i]) / gainList[i] 

1228 for i in range(nIter - 1) 

1229 ] 

1230 # The convergence metric is strictly positive, so if the estimated 

1231 # final convergence is less than zero, force it to zero. 

1232 estFinalConv = np.array(estFinalConv) 

1233 estFinalConv[estFinalConv < 0] = 0 

1234 # Because the estimate may slowly change over time, only use the 

1235 # most recent measurements. 

1236 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0) :]) 

1237 lastGain = gainList[-1] 

1238 lastConv = convergenceList[-2] 

1239 newConv = convergenceList[-1] 

1240 # The predicted convergence is the value we would get if the new 

1241 # model calculated in the previous iteration was perfect. Recall 

1242 # that the updated model that is actually used is the gain-weighted 

1243 # average of the new and old model, so the convergence would be 

1244 # similarly weighted. 

1245 predictedConv = (estFinalConv * lastGain + lastConv) / (1.0 + lastGain) 

1246 # If the measured and predicted convergence are very close, that 

1247 # indicates that our forward model is accurate and we can use a 

1248 # more aggressive gain. If the measured convergence is 

1249 # significantly worse (or better!) than predicted, that indicates 

1250 # that the model is not converging as expected and we should use a 

1251 # more conservative gain. 

1252 delta = (predictedConv - newConv) / ((lastConv - estFinalConv) / (1 + lastGain)) 

1253 newGain = 1 - abs(delta) 

1254 # Average the gains to prevent oscillating solutions. 

1255 newGain = (newGain + lastGain) / 2.0 

1256 gain = max(baseGain, newGain) 

1257 else: 

1258 gain = baseGain 

1259 gainList.append(gain) 

1260 return gain 

1261 

1262 def calculateModelWeights(self, dcrModels, dcrBBox): 

1263 """Build an array that smoothly tapers to 0 away from detected sources. 

1264 

1265 Parameters 

1266 ---------- 

1267 dcrModels : `lsst.pipe.tasks.DcrModel` 

1268 Best fit model of the true sky after correcting chromatic effects. 

1269 dcrBBox : `lsst.geom.box.Box2I` 

1270 Sub-region of the coadd which includes a buffer to allow for DCR. 

1271 

1272 Returns 

1273 ------- 

1274 weights : `numpy.ndarray` or `float` 

1275 A 2D array of weight values that tapers smoothly to zero away from 

1276 detected sources. Set to a placeholder value of 1.0 if 

1277 ``self.config.useModelWeights`` is False. 

1278 

1279 Raises 

1280 ------ 

1281 ValueError 

1282 If ``useModelWeights`` is set and ``modelWeightsWidth`` is 

1283 negative. 

1284 """ 

1285 if not self.config.useModelWeights: 

1286 return 1.0 

1287 if self.config.modelWeightsWidth < 0: 

1288 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set") 

1289 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1290 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0 

1291 weights = np.zeros_like(dcrModels[0][dcrBBox].array) 

1292 weights[convergeMaskPixels] = 1.0 

1293 weights = ndimage.gaussian_filter(weights, self.config.modelWeightsWidth) 

1294 weights /= np.max(weights) 

1295 return weights 

1296 

1297 def applyModelWeights(self, modelImages, refImage, modelWeights): 

1298 """Smoothly replace model pixel values with those from a 

1299 reference at locations away from detected sources. 

1300 

1301 Parameters 

1302 ---------- 

1303 modelImages : `list` of `lsst.afw.image.Image` 

1304 The new DCR model images from the current iteration. 

1305 The values will be modified in place. 

1306 refImage : `lsst.afw.image.MaskedImage` 

1307 A reference image used to supply the default pixel values. 

1308 modelWeights : `numpy.ndarray` or `float` 

1309 A 2D array of weight values that tapers smoothly to zero away from 

1310 detected sources. Set to a placeholder value of 1.0 if 

1311 ``self.config.useModelWeights`` is False. 

1312 """ 

1313 if self.config.useModelWeights: 

1314 for model in modelImages: 

1315 model.array *= modelWeights 

1316 model.array += refImage.array * (1.0 - modelWeights) / self.config.dcrNumSubfilters 

1317 

1318 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList): 

1319 """Pre-load sub-regions of a list of exposures. 

1320 

1321 Parameters 

1322 ---------- 

1323 bbox : `lsst.geom.box.Box2I` 

1324 Sub-region to coadd. 

1325 statsCtrl : `lsst.afw.math.StatisticsControl` 

1326 Statistics control object for coadd. 

1327 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1328 The data references to the input warped exposures. 

1329 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

1330 The image scalars correct for the zero point of the exposures. 

1331 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None` 

1332 Each element is dict with keys = mask plane name to add the spans 

1333 to. 

1334 

1335 Returns 

1336 ------- 

1337 subExposures : `dict` 

1338 The `dict` keys are the visit IDs, 

1339 and the values are `lsst.afw.image.ExposureF` 

1340 The pre-loaded exposures for the current subregion. 

1341 The variance plane contains weights, and not the variance 

1342 """ 

1343 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList) 

1344 subExposures = {} 

1345 for warpExpRef, imageScaler, altMaskSpans in zipIterables: 

1346 exposure = warpExpRef.get(parameters={"bbox": bbox}) 

1347 visit = warpExpRef.dataId["visit"] 

1348 if altMaskSpans is not None: 

1349 self.applyAltMaskPlanes(exposure.mask, altMaskSpans) 

1350 imageScaler.scaleMaskedImage(exposure.maskedImage) 

1351 # Note that the variance plane here is used to store weights, not 

1352 # the actual variance 

1353 exposure.variance.array[:, :] = 0.0 

1354 # Set the weight of unmasked pixels to 1. 

1355 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.0 

1356 # Set the image value of masked pixels to zero. 

1357 # This eliminates needing the mask plane when stacking images in 

1358 # ``newModelFromResidual`` 

1359 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.0 

1360 subExposures[visit] = exposure 

1361 return subExposures 

1362 

1363 def selectCoaddPsf(self, templateCoadd, warpRefList): 

1364 """Compute the PSF of the coadd from the exposures with the best 

1365 seeing. 

1366 

1367 Parameters 

1368 ---------- 

1369 templateCoadd : `lsst.afw.image.ExposureF` 

1370 The initial coadd exposure before accounting for DCR. 

1371 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1372 The data references to the input warped exposures. 

1373 

1374 Returns 

1375 ------- 

1376 psf : `lsst.meas.algorithms.CoaddPsf` 

1377 The average PSF of the input exposures with the best seeing. 

1378 """ 

1379 sigma2fwhm = 2.0 * np.sqrt(2.0 * np.log(2.0)) 

1380 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry 

1381 # per ccd and per visit. If there are multiple ccds, it will have that 

1382 # many times more elements than ``warpExpRef``. 

1383 ccds = templateCoadd.getInfo().getCoaddInputs().ccds 

1384 templatePsf = templateCoadd.getPsf() 

1385 # Just need a rough estimate; average positions are fine 

1386 templateAvgPos = templatePsf.getAveragePosition() 

1387 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius() * sigma2fwhm 

1388 psfSizes = np.zeros(len(ccds)) 

1389 ccdVisits = np.array(ccds["visit"]) 

1390 for warpExpRef in warpRefList: 

1391 psf = warpExpRef.get(component="psf") 

1392 visit = warpExpRef.dataId["visit"] 

1393 psfAvgPos = psf.getAveragePosition() 

1394 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius() * sigma2fwhm 

1395 psfSizes[ccdVisits == visit] = psfSize 

1396 # Note that the input PSFs include DCR, which should be absent from the 

1397 # DcrCoadd. The selected PSFs are those that have a FWHM less than or 

1398 # equal to the smaller of the mean or median FWHM of the input 

1399 # exposures. 

1400 sizeThreshold = min(np.median(psfSizes), psfRefSize) 

1401 goodPsfs = psfSizes <= sizeThreshold 

1402 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), self.config.coaddPsf.makeControl()) 

1403 return psf