Coverage for python/lsst/drp/tasks/dcr_assemble_coadd.py: 15%

413 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 10:27 +0000

1# This file is part of drp_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"] 

23 

24from math import ceil 

25import numpy as np 

26from scipy import ndimage 

27import lsst.geom as geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.table as afwTable 

30import lsst.coadd.utils as coaddUtils 

31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel 

32import lsst.meas.algorithms as measAlg 

33from lsst.meas.base import SingleFrameMeasurementTask 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.utils as utils 

37from lsst.utils.timer import timeMethod 

38from .assemble_coadd import (AssembleCoaddConnections, 

39 AssembleCoaddTask, 

40 CompareWarpAssembleCoaddConfig, 

41 CompareWarpAssembleCoaddTask, 

42 ) 

43from lsst.pipe.tasks.coaddBase import makeSkyInfo, subBBoxIter 

44from lsst.pipe.tasks.measurePsf import MeasurePsfTask 

45 

46 

47class DcrAssembleCoaddConnections(AssembleCoaddConnections, 

48 dimensions=("tract", "patch", "band", "skymap"), 

49 defaultTemplates={"inputWarpName": "deep", 

50 "inputCoaddName": "deep", 

51 "outputCoaddName": "dcr", 

52 "warpType": "direct", 

53 "warpTypeSuffix": "", 

54 "fakesType": ""}): 

55 inputWarps = pipeBase.connectionTypes.Input( 

56 doc=("Input list of warps to be assembled i.e. stacked." 

57 "Note that this will often be different than the inputCoaddName." 

58 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

59 name="{inputWarpName}Coadd_{warpType}Warp", 

60 storageClass="ExposureF", 

61 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

62 deferLoad=True, 

63 multiple=True 

64 ) 

65 templateExposure = pipeBase.connectionTypes.Input( 

66 doc="Input coadded exposure, produced by previous call to AssembleCoadd", 

67 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}", 

68 storageClass="ExposureF", 

69 dimensions=("tract", "patch", "skymap", "band"), 

70 ) 

71 dcrCoadds = pipeBase.connectionTypes.Output( 

72 doc="Output coadded exposure, produced by stacking input warps", 

73 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

74 storageClass="ExposureF", 

75 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

76 multiple=True, 

77 ) 

78 dcrNImages = pipeBase.connectionTypes.Output( 

79 doc="Output image of number of input images per pixel", 

80 name="{outputCoaddName}Coadd_nImage", 

81 storageClass="ImageU", 

82 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

83 multiple=True, 

84 ) 

85 

86 def __init__(self, *, config=None): 

87 super().__init__(config=config) 

88 if not config.doWrite: 

89 self.outputs.remove("dcrCoadds") 

90 if not config.doNImage: 

91 self.outputs.remove("dcrNImages") 

92 # Remove outputs inherited from ``AssembleCoaddConnections`` that are 

93 # not used. 

94 self.outputs.remove("coaddExposure") 

95 self.outputs.remove("nImage") 

96 

97 

98class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, 

99 pipelineConnections=DcrAssembleCoaddConnections): 

100 dcrNumSubfilters = pexConfig.Field( 

101 dtype=int, 

102 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.", 

103 default=3, 

104 ) 

105 maxNumIter = pexConfig.Field( 

106 dtype=int, 

107 optional=True, 

108 doc="Maximum number of iterations of forward modeling.", 

109 default=None, 

110 ) 

111 minNumIter = pexConfig.Field( 

112 dtype=int, 

113 optional=True, 

114 doc="Minimum number of iterations of forward modeling.", 

115 default=None, 

116 ) 

117 convergenceThreshold = pexConfig.Field( 

118 dtype=float, 

119 doc="Target relative change in convergence between iterations of forward modeling.", 

120 default=0.001, 

121 ) 

122 useConvergence = pexConfig.Field( 

123 dtype=bool, 

124 doc="Use convergence test as a forward modeling end condition?" 

125 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations", 

126 default=True, 

127 ) 

128 baseGain = pexConfig.Field( 

129 dtype=float, 

130 optional=True, 

131 doc="Relative weight to give the new solution vs. the last solution when updating the model." 

132 "A value of 1.0 gives equal weight to both solutions." 

133 "Small values imply slower convergence of the solution, but can " 

134 "help prevent overshooting and failures in the fit." 

135 "If ``baseGain`` is None, a conservative gain " 

136 "will be calculated from the number of subfilters. ", 

137 default=None, 

138 ) 

139 useProgressiveGain = pexConfig.Field( 

140 dtype=bool, 

141 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 

142 "When calculating the next gain, we use up to 5 previous gains and convergence values." 

143 "Can be set to False to force the model to change at the rate of ``baseGain``. ", 

144 default=True, 

145 ) 

146 doAirmassWeight = pexConfig.Field( 

147 dtype=bool, 

148 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.", 

149 default=False, 

150 ) 

151 modelWeightsWidth = pexConfig.Field( 

152 dtype=float, 

153 doc="Width of the region around detected sources to include in the DcrModel.", 

154 default=3, 

155 ) 

156 useModelWeights = pexConfig.Field( 

157 dtype=bool, 

158 doc="Width of the region around detected sources to include in the DcrModel.", 

159 default=True, 

160 ) 

161 splitSubfilters = pexConfig.Field( 

162 dtype=bool, 

163 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter." 

164 "Instead of at the midpoint", 

165 default=True, 

166 ) 

167 splitThreshold = pexConfig.Field( 

168 dtype=float, 

169 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 

170 "Set to 0 to always split the subfilters.", 

171 default=0.1, 

172 ) 

173 regularizeModelIterations = pexConfig.Field( 

174 dtype=float, 

175 doc="Maximum relative change of the model allowed between iterations." 

176 "Set to zero to disable.", 

177 default=2., 

178 ) 

179 regularizeModelFrequency = pexConfig.Field( 

180 dtype=float, 

181 doc="Maximum relative change of the model allowed between subfilters." 

182 "Set to zero to disable.", 

183 default=4., 

184 ) 

185 convergenceMaskPlanes = pexConfig.ListField( 

186 dtype=str, 

187 default=["DETECTED"], 

188 doc="Mask planes to use to calculate convergence." 

189 ) 

190 regularizationWidth = pexConfig.Field( 

191 dtype=int, 

192 default=2, 

193 doc="Minimum radius of a region to include in regularization, in pixels." 

194 ) 

195 imageInterpOrder = pexConfig.Field( 

196 dtype=int, 

197 doc="The order of the spline interpolation used to shift the image plane.", 

198 default=1, 

199 ) 

200 accelerateModel = pexConfig.Field( 

201 dtype=float, 

202 doc="Factor to amplify the differences between model planes by to speed convergence.", 

203 default=3, 

204 ) 

205 doCalculatePsf = pexConfig.Field( 

206 dtype=bool, 

207 doc="Set to detect stars and recalculate the PSF from the final coadd." 

208 "Otherwise the PSF is estimated from a selection of the best input exposures", 

209 default=False, 

210 ) 

211 detectPsfSources = pexConfig.ConfigurableField( 

212 target=measAlg.SourceDetectionTask, 

213 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.", 

214 ) 

215 measurePsfSources = pexConfig.ConfigurableField( 

216 target=SingleFrameMeasurementTask, 

217 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 

218 ) 

219 measurePsf = pexConfig.ConfigurableField( 

220 target=MeasurePsfTask, 

221 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.", 

222 ) 

223 effectiveWavelength = pexConfig.Field( 

224 doc="Effective wavelength of the filter, in nm." 

225 "Required if transmission curves aren't used." 

226 "Support for using transmission curves is to be added in DM-13668.", 

227 dtype=float, 

228 ) 

229 bandwidth = pexConfig.Field( 

230 doc="Bandwidth of the physical filter, in nm." 

231 "Required if transmission curves aren't used." 

232 "Support for using transmission curves is to be added in DM-13668.", 

233 dtype=float, 

234 ) 

235 

236 def setDefaults(self): 

237 CompareWarpAssembleCoaddConfig.setDefaults(self) 

238 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask) 

239 self.doNImage = True 

240 self.assembleStaticSkyModel.warpType = self.warpType 

241 # The deepCoadd and nImage files will be overwritten by this Task, so 

242 # don't write them the first time. 

243 self.assembleStaticSkyModel.doNImage = False 

244 self.assembleStaticSkyModel.doWrite = False 

245 self.detectPsfSources.returnOriginalFootprints = False 

246 self.detectPsfSources.thresholdPolarity = "positive" 

247 # Only use bright sources for PSF measurement 

248 self.detectPsfSources.thresholdValue = 50 

249 self.detectPsfSources.nSigmaToGrow = 2 

250 # A valid star for PSF measurement should at least fill 5x5 pixels 

251 self.detectPsfSources.minPixels = 25 

252 # Use the variance plane to calculate signal to noise 

253 self.detectPsfSources.thresholdType = "pixel_stdev" 

254 # Ensure psf candidate size is as large as piff psf size. 

255 if (self.doCalculatePsf and self.measurePsf.psfDeterminer.name == "piff" 

256 and self.psfDeterminer["piff"].kernelSize > self.makePsfCandidates.kernelSize): 

257 self.makePsfCandidates.kernelSize = self.psfDeterminer["piff"].kernelSize 

258 

259 

260class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask): 

261 """Assemble DCR coadded images from a set of warps. 

262 

263 Attributes 

264 ---------- 

265 bufferSize : `int` 

266 The number of pixels to grow each subregion by to allow for DCR. 

267 

268 Notes 

269 ----- 

270 As with AssembleCoaddTask, we want to assemble a coadded image from a set 

271 of Warps (also called coadded temporary exposures), including the effects 

272 of Differential Chromatic Refraction (DCR). 

273 For full details of the mathematics and algorithm, please see 

274 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 

275 

276 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 

277 each subfilter used in the iterative calculation. 

278 It begins by dividing the bandpass-defining filter into N equal bandwidth 

279 "subfilters", and divides the flux in each pixel from an initial coadd 

280 equally into each as a "dcrModel". Because the airmass and parallactic 

281 angle of each individual exposure is known, we can calculate the shift 

282 relative to the center of the band in each subfilter due to DCR. For each 

283 exposure we apply this shift as a linear transformation to the dcrModels 

284 and stack the results to produce a DCR-matched exposure. The matched 

285 exposures are subtracted from the input exposures to produce a set of 

286 residual images, and these residuals are reverse shifted for each 

287 exposures' subfilters and stacked. The shifted and stacked residuals are 

288 added to the dcrModels to produce a new estimate of the flux in each pixel 

289 within each subfilter. The dcrModels are solved for iteratively, which 

290 continues until the solution from a new iteration improves by less than 

291 a set percentage, or a maximum number of iterations is reached. 

292 Two forms of regularization are employed to reduce unphysical results. 

293 First, the new solution is averaged with the solution from the previous 

294 iteration, which mitigates oscillating solutions where the model 

295 overshoots with alternating very high and low values. 

296 Second, a common degeneracy when the data have a limited range of airmass 

297 or parallactic angle values is for one subfilter to be fit with very low or 

298 negative values, while another subfilter is fit with very high values. This 

299 typically appears in the form of holes next to sources in one subfilter, 

300 and corresponding extended wings in another. Because each subfilter has 

301 a narrow bandwidth we assume that physical sources that are above the noise 

302 level will not vary in flux by more than a factor of `frequencyClampFactor` 

303 between subfilters, and pixels that have flux deviations larger than that 

304 factor will have the excess flux distributed evenly among all subfilters. 

305 If `splitSubfilters` is set, then each subfilter will be further sub- 

306 divided during the forward modeling step (only). This approximates using 

307 a higher number of subfilters that may be necessary for high airmass 

308 observations, but does not increase the number of free parameters in the 

309 fit. This is needed when there are high airmass observations which would 

310 otherwise have significant DCR even within a subfilter. Because calculating 

311 the shifted images takes most of the time, splitting the subfilters is 

312 turned off by way of the `splitThreshold` option for low-airmass 

313 observations that do not suffer from DCR within a subfilter. 

314 """ 

315 

316 ConfigClass = DcrAssembleCoaddConfig 

317 _DefaultName = "dcrAssembleCoadd" 

318 

319 def __init__(self, *args, **kwargs): 

320 super().__init__(*args, **kwargs) 

321 if self.config.doCalculatePsf: 

322 self.schema = afwTable.SourceTable.makeMinimalSchema() 

323 self.makeSubtask("detectPsfSources", schema=self.schema) 

324 self.makeSubtask("measurePsfSources", schema=self.schema) 

325 self.makeSubtask("measurePsf", schema=self.schema) 

326 

327 @utils.inheritDoc(pipeBase.PipelineTask) 

328 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

329 # Docstring to be formatted with info from PipelineTask.runQuantum 

330 """ 

331 Notes 

332 ----- 

333 Assemble a coadd from a set of Warps. 

334 """ 

335 inputData = butlerQC.get(inputRefs) 

336 

337 # Construct skyInfo expected by run 

338 # Do not remove skyMap from inputData in case _makeSupplementaryData 

339 # needs it. 

340 skyMap = inputData["skyMap"] 

341 outputDataId = butlerQC.quantum.dataId 

342 

343 inputData['skyInfo'] = makeSkyInfo(skyMap, 

344 tractId=outputDataId['tract'], 

345 patchId=outputDataId['patch']) 

346 

347 # Construct list of input Deferred Datasets 

348 warpRefList = inputData['inputWarps'] 

349 

350 inputs = self.prepareInputs(warpRefList) 

351 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

352 self.getTempExpDatasetName(self.warpType)) 

353 if len(inputs.tempExpRefList) == 0: 

354 self.log.warning("No coadd temporary exposures found") 

355 return 

356 

357 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

358 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

359 inputs.weightList, supplementaryData=supplementaryData) 

360 

361 inputData.setdefault('brightObjectMask', None) 

362 for subfilter in range(self.config.dcrNumSubfilters): 

363 # Use the PSF of the stacked dcrModel, and do not recalculate the 

364 # PSF for each subfilter 

365 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf()) 

366 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId) 

367 

368 if self.config.doWrite: 

369 butlerQC.put(retStruct, outputRefs) 

370 return retStruct 

371 

372 @utils.inheritDoc(AssembleCoaddTask) 

373 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

374 """Load the previously-generated template coadd. 

375 

376 Returns 

377 ------- 

378 templateCoadd : `lsst.pipe.base.Struct` 

379 Results as a struct with attributes: 

380 

381 ``templateCoadd`` 

382 Coadded exposure (`lsst.afw.image.ExposureF`). 

383 """ 

384 templateCoadd = butlerQC.get(inputRefs.templateExposure) 

385 

386 return pipeBase.Struct(templateCoadd=templateCoadd) 

387 

388 def measureCoaddPsf(self, coaddExposure): 

389 """Detect sources on the coadd exposure and measure the final PSF. 

390 

391 Parameters 

392 ---------- 

393 coaddExposure : `lsst.afw.image.Exposure` 

394 The final coadded exposure. 

395 """ 

396 table = afwTable.SourceTable.make(self.schema) 

397 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False) 

398 coaddSources = detResults.sources 

399 self.measurePsfSources.run( 

400 measCat=coaddSources, 

401 exposure=coaddExposure 

402 ) 

403 # Measure the PSF on the stacked subfilter coadds if possible. 

404 # We should already have a decent estimate of the coadd PSF, however, 

405 # so in case of any errors simply log them as a warning and use the 

406 # default PSF. 

407 try: 

408 psfResults = self.measurePsf.run(coaddExposure, coaddSources) 

409 except Exception as e: 

410 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e) 

411 else: 

412 coaddExposure.setPsf(psfResults.psf) 

413 

414 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList): 

415 """Prepare the DCR coadd by iterating through the visitInfo of the 

416 input warps. 

417 

418 Sets the property ``bufferSize``. 

419 

420 Parameters 

421 ---------- 

422 templateCoadd : `lsst.afw.image.ExposureF` 

423 The initial coadd exposure before accounting for DCR. 

424 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

425 The data references to the input warped exposures. 

426 weightList : `list` of `float` 

427 The weight to give each input exposure in the coadd. 

428 Will be modified in place if ``doAirmassWeight`` is set. 

429 

430 Returns 

431 ------- 

432 dcrModels : `lsst.pipe.tasks.DcrModel` 

433 Best fit model of the true sky after correcting chromatic effects. 

434 

435 Raises 

436 ------ 

437 NotImplementedError 

438 If ``lambdaMin`` is missing from the Mapper class of the obs 

439 package being used. 

440 """ 

441 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

442 filterLabel = templateCoadd.getFilter() 

443 dcrShifts = [] 

444 airmassDict = {} 

445 angleDict = {} 

446 psfSizeDict = {} 

447 for visitNum, warpExpRef in enumerate(warpRefList): 

448 visitInfo = warpExpRef.get(component="visitInfo") 

449 psf = warpExpRef.get(component="psf") 

450 visit = warpExpRef.dataId["visit"] 

451 # Just need a rough estimate; average positions are fine 

452 psfAvgPos = psf.getAveragePosition() 

453 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm 

454 airmass = visitInfo.getBoresightAirmass() 

455 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees() 

456 airmassDict[visit] = airmass 

457 angleDict[visit] = parallacticAngle 

458 psfSizeDict[visit] = psfSize 

459 if self.config.doAirmassWeight: 

460 weightList[visitNum] *= airmass 

461 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(), 

462 self.config.effectiveWavelength, 

463 self.config.bandwidth, 

464 self.config.dcrNumSubfilters)))) 

465 self.log.info("Selected airmasses:\n%s", airmassDict) 

466 self.log.info("Selected parallactic angles:\n%s", angleDict) 

467 self.log.info("Selected PSF sizes:\n%s", psfSizeDict) 

468 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1) 

469 try: 

470 psf = self.selectCoaddPsf(templateCoadd, warpRefList) 

471 except Exception as e: 

472 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e) 

473 else: 

474 psf = templateCoadd.getPsf() 

475 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage, 

476 self.config.dcrNumSubfilters, 

477 effectiveWavelength=self.config.effectiveWavelength, 

478 bandwidth=self.config.bandwidth, 

479 wcs=templateCoadd.getWcs(), 

480 filterLabel=filterLabel, 

481 psf=psf) 

482 return dcrModels 

483 

484 @timeMethod 

485 def run(self, skyInfo, warpRefList, imageScalerList, weightList, 

486 supplementaryData=None): 

487 r"""Assemble the coadd. 

488 

489 Requires additional inputs Struct ``supplementaryData`` to contain a 

490 ``templateCoadd`` that serves as the model of the static sky. 

491 

492 Find artifacts and apply them to the warps' masks creating a list of 

493 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

494 plane then pass these alternative masks to the base class's assemble 

495 method. 

496 

497 Divide the ``templateCoadd`` evenly between each subfilter of a 

498 ``DcrModel`` as the starting best estimate of the true wavelength- 

499 dependent sky. Forward model the ``DcrModel`` using the known 

500 chromatic effects in each subfilter and calculate a convergence metric 

501 based on how well the modeled template matches the input warps. If 

502 the convergence has not yet reached the desired threshold, then shift 

503 and stack the residual images to build a new ``DcrModel``. Apply 

504 conditioning to prevent oscillating solutions between iterations or 

505 between subfilters. 

506 

507 Once the ``DcrModel`` reaches convergence or the maximum number of 

508 iterations has been reached, fill the metadata for each subfilter 

509 image and make them proper ``coaddExposure``\ s. 

510 

511 Parameters 

512 ---------- 

513 skyInfo : `lsst.pipe.base.Struct` 

514 Patch geometry information, from getSkyInfo 

515 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

516 The data references to the input warped exposures. 

517 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

518 The image scalars correct for the zero point of the exposures. 

519 weightList : `list` of `float` 

520 The weight to give each input exposure in the coadd 

521 supplementaryData : `lsst.pipe.base.Struct` 

522 Result struct returned by ``_makeSupplementaryData`` with 

523 attributes: 

524 

525 ``templateCoadd`` 

526 Coadded exposure (`lsst.afw.image.Exposure`). 

527 

528 Returns 

529 ------- 

530 result : `lsst.pipe.base.Struct` 

531 Results as a struct with attributes: 

532 

533 ``coaddExposure`` 

534 Coadded exposure (`lsst.afw.image.Exposure`). 

535 ``nImage`` 

536 Exposure count image (`lsst.afw.image.ImageU`). 

537 ``dcrCoadds`` 

538 `list` of coadded exposures for each subfilter. 

539 ``dcrNImages`` 

540 `list` of exposure count images for each subfilter. 

541 """ 

542 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters 

543 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3 

544 templateCoadd = supplementaryData.templateCoadd 

545 baseMask = templateCoadd.mask.clone() 

546 # The variance plane is for each subfilter 

547 # and should be proportionately lower than the full-band image 

548 baseVariance = templateCoadd.variance.clone() 

549 baseVariance /= self.config.dcrNumSubfilters 

550 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList) 

551 # Note that the mask gets cleared in ``findArtifacts``, but we want to 

552 # preserve the mask. 

553 templateCoadd.setMask(baseMask) 

554 badMaskPlanes = self.config.badMaskPlanes[:] 

555 # Note that is important that we do not add "CLIPPED" to 

556 # ``badMaskPlanes``. This is because pixels in observations that are 

557 # significantly affected by DCR are likely to have many pixels that are 

558 # both "DETECTED" and "CLIPPED", but those are necessary to constrain 

559 # the DCR model. 

560 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes) 

561 

562 stats = self.prepareStats(mask=badPixelMask) 

563 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList) 

564 if self.config.doNImage: 

565 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList, 

566 spanSetMaskList, stats.ctrl) 

567 nImage = afwImage.ImageU(skyInfo.bbox) 

568 # Note that this nImage will be a factor of dcrNumSubfilters higher 

569 # than the nImage returned by assembleCoadd for most pixels. This 

570 # is because each subfilter may have a different nImage, and 

571 # fractional values are not allowed. 

572 for dcrNImage in dcrNImages: 

573 nImage += dcrNImage 

574 else: 

575 dcrNImages = None 

576 

577 subregionSize = geom.Extent2I(*self.config.subregionSize) 

578 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) 

579 * ceil(skyInfo.bbox.getWidth()/subregionSize[0])) 

580 subIter = 0 

581 for subBBox in subBBoxIter(skyInfo.bbox, subregionSize): 

582 modelIter = 0 

583 subIter += 1 

584 self.log.info("Computing coadd over patch %s subregion %s of %s: %s", 

585 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox) 

586 dcrBBox = geom.Box2I(subBBox) 

587 dcrBBox.grow(self.bufferSize) 

588 dcrBBox.clip(dcrModels.bbox) 

589 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox) 

590 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList, 

591 imageScalerList, spanSetMaskList) 

592 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

593 warpRefList, weightList, stats.ctrl) 

594 self.log.info("Initial convergence : %s", convergenceMetric) 

595 convergenceList = [convergenceMetric] 

596 gainList = [] 

597 convergenceCheck = 1. 

598 refImage = templateCoadd.image 

599 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter): 

600 gain = self.calculateGain(convergenceList, gainList) 

601 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList, 

602 stats.ctrl, convergenceMetric, gain, 

603 modelWeights, refImage, dcrWeights) 

604 if self.config.useConvergence: 

605 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

606 warpRefList, weightList, stats.ctrl) 

607 if convergenceMetric == 0: 

608 self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is " 

609 "most likely due to there being no valid data in the region.", 

610 skyInfo.patchInfo.getIndex(), subIter) 

611 break 

612 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric 

613 if (convergenceCheck < 0) & (modelIter > minNumIter): 

614 self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum " 

615 "iterations or desired convergence improvement of %s." 

616 " Divergence: %s", 

617 skyInfo.patchInfo.getIndex(), subIter, 

618 self.config.convergenceThreshold, convergenceCheck) 

619 break 

620 convergenceList.append(convergenceMetric) 

621 if modelIter > maxNumIter: 

622 if self.config.useConvergence: 

623 self.log.warning("Coadd patch %s subregion %s reached maximum iterations " 

624 "before reaching desired convergence improvement of %s." 

625 " Final convergence improvement: %s", 

626 skyInfo.patchInfo.getIndex(), subIter, 

627 self.config.convergenceThreshold, convergenceCheck) 

628 break 

629 

630 if self.config.useConvergence: 

631 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)", 

632 modelIter, convergenceMetric, 100.*convergenceCheck, gain) 

633 modelIter += 1 

634 else: 

635 if self.config.useConvergence: 

636 self.log.info("Coadd patch %s subregion %s finished with " 

637 "convergence metric %s after %s iterations", 

638 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter) 

639 else: 

640 self.log.info("Coadd patch %s subregion %s finished after %s iterations", 

641 skyInfo.patchInfo.getIndex(), subIter, modelIter) 

642 if self.config.useConvergence and convergenceMetric > 0: 

643 self.log.info("Final convergence improvement was %.4f%% overall", 

644 100*(convergenceList[0] - convergenceMetric)/convergenceMetric) 

645 

646 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList, 

647 calibration=self.scaleZeroPoint.getPhotoCalib(), 

648 coaddInputs=templateCoadd.getInfo().getCoaddInputs(), 

649 mask=baseMask, 

650 variance=baseVariance) 

651 coaddExposure = self.stackCoadd(dcrCoadds) 

652 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

653 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages) 

654 

655 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl): 

656 """Calculate the number of exposures contributing to each subfilter. 

657 

658 Parameters 

659 ---------- 

660 dcrModels : `lsst.pipe.tasks.DcrModel` 

661 Best fit model of the true sky after correcting chromatic effects. 

662 bbox : `lsst.geom.box.Box2I` 

663 Bounding box of the patch to coadd. 

664 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

665 The data references to the input warped exposures. 

666 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None` 

667 Each element of the `dict` contains the new mask plane name 

668 (e.g. "CLIPPED and/or "NO_DATA") as the key, 

669 and the list of SpanSets to apply to the mask. 

670 statsCtrl : `lsst.afw.math.StatisticsControl` 

671 Statistics control object for coadd 

672 

673 Returns 

674 ------- 

675 dcrNImages : `list` of `lsst.afw.image.ImageU` 

676 List of exposure count images for each subfilter. 

677 dcrWeights : `list` of `lsst.afw.image.ImageF` 

678 Per-pixel weights for each subfilter. 

679 Equal to 1/(number of unmasked images contributing to each pixel). 

680 """ 

681 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

682 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

683 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList): 

684 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

685 visitInfo = exposure.getInfo().getVisitInfo() 

686 wcs = exposure.getInfo().getWcs() 

687 mask = exposure.mask 

688 if altMaskSpans is not None: 

689 self.applyAltMaskPlanes(mask, altMaskSpans) 

690 weightImage = np.zeros_like(exposure.image.array) 

691 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1. 

692 # The weights must be shifted in exactly the same way as the 

693 # residuals, because they will be used as the denominator in the 

694 # weighted average of residuals. 

695 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, 

696 dcrModels.effectiveWavelength, dcrModels.bandwidth) 

697 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights): 

698 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype) 

699 dcrWeight.array += shiftedWeights 

700 # Exclude any pixels that don't have at least one exposure contributing 

701 # in all subfilters 

702 weightsThreshold = 1. 

703 goodPix = dcrWeights[0].array > weightsThreshold 

704 for weights in dcrWeights[1:]: 

705 goodPix = (weights.array > weightsThreshold) & goodPix 

706 for subfilter in range(self.config.dcrNumSubfilters): 

707 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix] 

708 dcrWeights[subfilter].array[~goodPix] = 0. 

709 dcrNImages[subfilter].array[~goodPix] = 0 

710 return (dcrNImages, dcrWeights) 

711 

712 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, 

713 statsCtrl, convergenceMetric, 

714 gain, modelWeights, refImage, dcrWeights): 

715 """Assemble the DCR coadd for a sub-region. 

716 

717 Build a DCR-matched template for each input exposure, then shift the 

718 residuals according to the DCR in each subfilter. 

719 Stack the shifted residuals and apply them as a correction to the 

720 solution from the previous iteration. 

721 Restrict the new model solutions from varying by more than a factor of 

722 `modelClampFactor` from the last solution, and additionally restrict 

723 the individual subfilter models from varying by more than a factor of 

724 `frequencyClampFactor` from their average. 

725 Finally, mitigate potentially oscillating solutions by averaging the 

726 new solution with the solution from the previous iteration, weighted by 

727 their convergence metric. 

728 

729 Parameters 

730 ---------- 

731 dcrModels : `lsst.pipe.tasks.DcrModel` 

732 Best fit model of the true sky after correcting chromatic effects. 

733 subExposures : `dict` of `lsst.afw.image.ExposureF` 

734 The pre-loaded exposures for the current subregion. 

735 bbox : `lsst.geom.box.Box2I` 

736 Bounding box of the subregion to coadd. 

737 dcrBBox : `lsst.geom.box.Box2I` 

738 Sub-region of the coadd which includes a buffer to allow for DCR. 

739 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

740 The data references to the input warped exposures. 

741 statsCtrl : `lsst.afw.math.StatisticsControl` 

742 Statistics control object for coadd. 

743 convergenceMetric : `float` 

744 Quality of fit metric for the matched templates of the input 

745 images. 

746 gain : `float`, optional 

747 Relative weight to give the new solution when updating the model. 

748 modelWeights : `numpy.ndarray` or `float` 

749 A 2D array of weight values that tapers smoothly to zero away from 

750 detected sources. Set to a placeholder value of 1.0 if 

751 ``self.config.useModelWeights`` is False. 

752 refImage : `lsst.afw.image.Image` 

753 A reference image used to supply the default pixel values. 

754 dcrWeights : `list` of `lsst.afw.image.Image` 

755 Per-pixel weights for each subfilter. 

756 Equal to 1/(number of unmasked images contributing to each pixel). 

757 """ 

758 residualGeneratorList = [] 

759 

760 for warpExpRef in warpRefList: 

761 visit = warpExpRef.dataId["visit"] 

762 exposure = subExposures[visit] 

763 visitInfo = exposure.getInfo().getVisitInfo() 

764 wcs = exposure.getInfo().getWcs() 

765 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

766 bbox=exposure.getBBox(), 

767 order=self.config.imageInterpOrder, 

768 splitSubfilters=self.config.splitSubfilters, 

769 splitThreshold=self.config.splitThreshold, 

770 amplifyModel=self.config.accelerateModel) 

771 residual = exposure.image.array - templateImage.array 

772 # Note that the variance plane here is used to store weights, not 

773 # the actual variance 

774 residual *= exposure.variance.array 

775 # The residuals are stored as a list of generators. 

776 # This allows the residual for a given subfilter and exposure to be 

777 # created on the fly, instead of needing to store them all in 

778 # memory. 

779 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, 

780 dcrModels.effectiveWavelength, 

781 dcrModels.bandwidth)) 

782 

783 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

784 gain=gain, 

785 modelWeights=modelWeights, 

786 refImage=refImage, 

787 dcrWeights=dcrWeights) 

788 dcrModels.assign(dcrSubModelOut, bbox) 

789 

790 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth): 

791 """Prepare a residual image for stacking in each subfilter by applying 

792 the reverse DCR shifts. 

793 

794 Parameters 

795 ---------- 

796 residual : `numpy.ndarray` 

797 The residual masked image for one exposure, 

798 after subtracting the matched template. 

799 visitInfo : `lsst.afw.image.VisitInfo` 

800 Metadata for the exposure. 

801 wcs : `lsst.afw.geom.SkyWcs` 

802 Coordinate system definition (wcs) for the exposure. 

803 

804 Yields 

805 ------ 

806 residualImage : `numpy.ndarray` 

807 The residual image for the next subfilter, shifted for DCR. 

808 """ 

809 if self.config.imageInterpOrder > 1: 

810 # Pre-calculate the spline-filtered residual image, so that step 

811 # can be skipped in the shift calculation in `applyDcr`. 

812 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder) 

813 else: 

814 # No need to prefilter if order=1 (it will also raise an error) 

815 filteredResidual = residual 

816 # Note that `splitSubfilters` is always turned off in the reverse 

817 # direction. This option introduces additional blurring if applied to 

818 # the residuals. 

819 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters, 

820 splitSubfilters=False) 

821 for dcr in dcrShift: 

822 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False, 

823 doPrefilter=False, order=self.config.imageInterpOrder) 

824 

825 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

826 gain, modelWeights, refImage, dcrWeights): 

827 """Calculate a new DcrModel from a set of image residuals. 

828 

829 Parameters 

830 ---------- 

831 dcrModels : `lsst.pipe.tasks.DcrModel` 

832 Current model of the true sky after correcting chromatic effects. 

833 residualGeneratorList : `generator` of `numpy.ndarray` 

834 The residual image for the next subfilter, shifted for DCR. 

835 dcrBBox : `lsst.geom.box.Box2I` 

836 Sub-region of the coadd which includes a buffer to allow for DCR. 

837 statsCtrl : `lsst.afw.math.StatisticsControl` 

838 Statistics control object for coadd. 

839 gain : `float` 

840 Relative weight to give the new solution when updating the model. 

841 modelWeights : `numpy.ndarray` or `float` 

842 A 2D array of weight values that tapers smoothly to zero away from 

843 detected sources. Set to a placeholder value of 1.0 if 

844 ``self.config.useModelWeights`` is False. 

845 refImage : `lsst.afw.image.Image` 

846 A reference image used to supply the default pixel values. 

847 dcrWeights : `list` of `lsst.afw.image.Image` 

848 Per-pixel weights for each subfilter. 

849 Equal to 1/(number of unmasked images contributing to each pixel). 

850 

851 Returns 

852 ------- 

853 dcrModel : `lsst.pipe.tasks.DcrModel` 

854 New model of the true sky after correcting chromatic effects. 

855 """ 

856 newModelImages = [] 

857 for subfilter, model in enumerate(dcrModels): 

858 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList] 

859 residual = np.sum(residualsList, axis=0) 

860 residual *= dcrWeights[subfilter][dcrBBox].array 

861 # `MaskedImage`s only support in-place addition, so rename for 

862 # readability. 

863 newModel = model[dcrBBox].clone() 

864 newModel.array += residual 

865 # Catch any invalid values 

866 badPixels = ~np.isfinite(newModel.array) 

867 newModel.array[badPixels] = model[dcrBBox].array[badPixels] 

868 if self.config.regularizeModelIterations > 0: 

869 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox, 

870 self.config.regularizeModelIterations, 

871 self.config.regularizationWidth) 

872 newModelImages.append(newModel) 

873 if self.config.regularizeModelFrequency > 0: 

874 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl, 

875 self.config.regularizeModelFrequency, 

876 self.config.regularizationWidth) 

877 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain) 

878 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights) 

879 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength, 

880 dcrModels.bandwidth, dcrModels.psf, 

881 dcrModels.mask, dcrModels.variance) 

882 

883 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl): 

884 """Calculate a quality of fit metric for the matched templates. 

885 

886 Parameters 

887 ---------- 

888 dcrModels : `lsst.pipe.tasks.DcrModel` 

889 Best fit model of the true sky after correcting chromatic effects. 

890 subExposures : `dict` of `lsst.afw.image.ExposureF` 

891 The pre-loaded exposures for the current subregion. 

892 bbox : `lsst.geom.box.Box2I` 

893 Sub-region to coadd. 

894 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

895 The data references to the input warped exposures. 

896 weightList : `list` of `float` 

897 The weight to give each input exposure in the coadd. 

898 statsCtrl : `lsst.afw.math.StatisticsControl` 

899 Statistics control object for coadd. 

900 

901 Returns 

902 ------- 

903 convergenceMetric : `float` 

904 Quality of fit metric for all input exposures, within the 

905 sub-region. 

906 """ 

907 significanceImage = np.abs(dcrModels.getReferenceImage(bbox)) 

908 nSigma = 3. 

909 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl, 

910 bufferSize=self.bufferSize) 

911 if np.max(significanceImage) == 0: 

912 significanceImage += 1. 

913 weight = 0 

914 metric = 0. 

915 metricList = {} 

916 for warpExpRef, expWeight in zip(warpRefList, weightList): 

917 visit = warpExpRef.dataId["visit"] 

918 exposure = subExposures[visit][bbox] 

919 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl) 

920 metric += singleMetric 

921 metricList[visit] = singleMetric 

922 weight += 1. 

923 self.log.info("Individual metrics:\n%s", metricList) 

924 return 1.0 if weight == 0.0 else metric/weight 

925 

926 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl): 

927 """Calculate a quality of fit metric for a single matched template. 

928 

929 Parameters 

930 ---------- 

931 dcrModels : `lsst.pipe.tasks.DcrModel` 

932 Best fit model of the true sky after correcting chromatic effects. 

933 exposure : `lsst.afw.image.ExposureF` 

934 The input warped exposure to evaluate. 

935 significanceImage : `numpy.ndarray` 

936 Array of weights for each pixel corresponding to its significance 

937 for the convergence calculation. 

938 statsCtrl : `lsst.afw.math.StatisticsControl` 

939 Statistics control object for coadd. 

940 

941 Returns 

942 ------- 

943 convergenceMetric : `float` 

944 Quality of fit metric for one exposure, within the sub-region. 

945 """ 

946 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

947 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

948 bbox=exposure.getBBox(), 

949 order=self.config.imageInterpOrder, 

950 splitSubfilters=self.config.splitSubfilters, 

951 splitThreshold=self.config.splitThreshold, 

952 amplifyModel=self.config.accelerateModel) 

953 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage 

954 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2. 

955 

956 finitePixels = np.isfinite(diffVals) 

957 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0 

958 convergeMaskPixels = exposure.mask.array & convergeMask > 0 

959 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels 

960 if np.sum(usePixels) == 0: 

961 metric = 0. 

962 else: 

963 diffUse = diffVals[usePixels] 

964 refUse = refVals[usePixels] 

965 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse)) 

966 return metric 

967 

968 def stackCoadd(self, dcrCoadds): 

969 """Add a list of sub-band coadds together. 

970 

971 Parameters 

972 ---------- 

973 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

974 A list of coadd exposures, each exposure containing 

975 the model for one subfilter. 

976 

977 Returns 

978 ------- 

979 coaddExposure : `lsst.afw.image.ExposureF` 

980 A single coadd exposure that is the sum of the sub-bands. 

981 """ 

982 coaddExposure = dcrCoadds[0].clone() 

983 for coadd in dcrCoadds[1:]: 

984 coaddExposure.maskedImage += coadd.maskedImage 

985 return coaddExposure 

986 

987 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, 

988 mask=None, variance=None): 

989 """Create a list of coadd exposures from a list of masked images. 

990 

991 Parameters 

992 ---------- 

993 dcrModels : `lsst.pipe.tasks.DcrModel` 

994 Best fit model of the true sky after correcting chromatic effects. 

995 skyInfo : `lsst.pipe.base.Struct` 

996 Patch geometry information, from getSkyInfo. 

997 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

998 The data references to the input warped exposures. 

999 weightList : `list` of `float` 

1000 The weight to give each input exposure in the coadd. 

1001 calibration : `lsst.afw.Image.PhotoCalib`, optional 

1002 Scale factor to set the photometric calibration of an exposure. 

1003 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 

1004 A record of the observations that are included in the coadd. 

1005 mask : `lsst.afw.image.Mask`, optional 

1006 Optional mask to override the values in the final coadd. 

1007 variance : `lsst.afw.image.Image`, optional 

1008 Optional variance plane to override the values in the final coadd. 

1009 

1010 Returns 

1011 ------- 

1012 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1013 A list of coadd exposures, each exposure containing 

1014 the model for one subfilter. 

1015 """ 

1016 dcrCoadds = [] 

1017 refModel = dcrModels.getReferenceImage() 

1018 for model in dcrModels: 

1019 if self.config.accelerateModel > 1: 

1020 model.array = (model.array - refModel)*self.config.accelerateModel + refModel 

1021 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

1022 if calibration is not None: 

1023 coaddExposure.setPhotoCalib(calibration) 

1024 if coaddInputs is not None: 

1025 coaddExposure.getInfo().setCoaddInputs(coaddInputs) 

1026 # Set the metadata for the coadd, including PSF and aperture 

1027 # corrections. 

1028 self.assembleMetadata(coaddExposure, warpRefList, weightList) 

1029 # Overwrite the PSF 

1030 coaddExposure.setPsf(dcrModels.psf) 

1031 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox]) 

1032 maskedImage = afwImage.MaskedImageF(dcrModels.bbox) 

1033 maskedImage.image = model 

1034 maskedImage.mask = dcrModels.mask 

1035 maskedImage.variance = dcrModels.variance 

1036 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox]) 

1037 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

1038 if mask is not None: 

1039 coaddExposure.setMask(mask) 

1040 if variance is not None: 

1041 coaddExposure.setVariance(variance) 

1042 dcrCoadds.append(coaddExposure) 

1043 return dcrCoadds 

1044 

1045 def calculateGain(self, convergenceList, gainList): 

1046 """Calculate the gain to use for the current iteration. 

1047 

1048 After calculating a new DcrModel, each value is averaged with the 

1049 value in the corresponding pixel from the previous iteration. This 

1050 reduces oscillating solutions that iterative techniques are plagued by, 

1051 and speeds convergence. By far the biggest changes to the model 

1052 happen in the first couple iterations, so we can also use a more 

1053 aggressive gain later when the model is changing slowly. 

1054 

1055 Parameters 

1056 ---------- 

1057 convergenceList : `list` of `float` 

1058 The quality of fit metric from each previous iteration. 

1059 gainList : `list` of `float` 

1060 The gains used in each previous iteration: appended with the new 

1061 gain value. 

1062 Gains are numbers between ``self.config.baseGain`` and 1. 

1063 

1064 Returns 

1065 ------- 

1066 gain : `float` 

1067 Relative weight to give the new solution when updating the model. 

1068 A value of 1.0 gives equal weight to both solutions. 

1069 

1070 Raises 

1071 ------ 

1072 ValueError 

1073 If ``len(convergenceList) != len(gainList)+1``. 

1074 """ 

1075 nIter = len(convergenceList) 

1076 if nIter != len(gainList) + 1: 

1077 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)." 

1078 % (len(convergenceList), len(gainList))) 

1079 

1080 if self.config.baseGain is None: 

1081 # If ``baseGain`` is not set, calculate it from the number of DCR 

1082 # subfilters. The more subfilters being modeled, the lower the gain 

1083 # should be. 

1084 baseGain = 1./(self.config.dcrNumSubfilters - 1) 

1085 else: 

1086 baseGain = self.config.baseGain 

1087 

1088 if self.config.useProgressiveGain and nIter > 2: 

1089 # To calculate the best gain to use, compare the past gains that 

1090 # have been used with the resulting convergences to estimate the 

1091 # best gain to use. Algorithmically, this is a Kalman filter. 

1092 # If forward modeling proceeds perfectly, the convergence metric 

1093 # should asymptotically approach a final value. We can estimate 

1094 # that value from the measured changes in convergence weighted by 

1095 # the gains used in each previous iteration. 

1096 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i] 

1097 for i in range(nIter - 1)] 

1098 # The convergence metric is strictly positive, so if the estimated 

1099 # final convergence is less than zero, force it to zero. 

1100 estFinalConv = np.array(estFinalConv) 

1101 estFinalConv[estFinalConv < 0] = 0 

1102 # Because the estimate may slowly change over time, only use the 

1103 # most recent measurements. 

1104 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):]) 

1105 lastGain = gainList[-1] 

1106 lastConv = convergenceList[-2] 

1107 newConv = convergenceList[-1] 

1108 # The predicted convergence is the value we would get if the new 

1109 # model calculated in the previous iteration was perfect. Recall 

1110 # that the updated model that is actually used is the gain-weighted 

1111 # average of the new and old model, so the convergence would be 

1112 # similarly weighted. 

1113 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain) 

1114 # If the measured and predicted convergence are very close, that 

1115 # indicates that our forward model is accurate and we can use a 

1116 # more aggressive gain. If the measured convergence is 

1117 # significantly worse (or better!) than predicted, that indicates 

1118 # that the model is not converging as expected and we should use a 

1119 # more conservative gain. 

1120 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain)) 

1121 newGain = 1 - abs(delta) 

1122 # Average the gains to prevent oscillating solutions. 

1123 newGain = (newGain + lastGain)/2. 

1124 gain = max(baseGain, newGain) 

1125 else: 

1126 gain = baseGain 

1127 gainList.append(gain) 

1128 return gain 

1129 

1130 def calculateModelWeights(self, dcrModels, dcrBBox): 

1131 """Build an array that smoothly tapers to 0 away from detected sources. 

1132 

1133 Parameters 

1134 ---------- 

1135 dcrModels : `lsst.pipe.tasks.DcrModel` 

1136 Best fit model of the true sky after correcting chromatic effects. 

1137 dcrBBox : `lsst.geom.box.Box2I` 

1138 Sub-region of the coadd which includes a buffer to allow for DCR. 

1139 

1140 Returns 

1141 ------- 

1142 weights : `numpy.ndarray` or `float` 

1143 A 2D array of weight values that tapers smoothly to zero away from 

1144 detected sources. Set to a placeholder value of 1.0 if 

1145 ``self.config.useModelWeights`` is False. 

1146 

1147 Raises 

1148 ------ 

1149 ValueError 

1150 If ``useModelWeights`` is set and ``modelWeightsWidth`` is 

1151 negative. 

1152 """ 

1153 if not self.config.useModelWeights: 

1154 return 1.0 

1155 if self.config.modelWeightsWidth < 0: 

1156 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set") 

1157 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1158 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0 

1159 weights = np.zeros_like(dcrModels[0][dcrBBox].array) 

1160 weights[convergeMaskPixels] = 1. 

1161 weights = ndimage.gaussian_filter(weights, self.config.modelWeightsWidth) 

1162 weights /= np.max(weights) 

1163 return weights 

1164 

1165 def applyModelWeights(self, modelImages, refImage, modelWeights): 

1166 """Smoothly replace model pixel values with those from a 

1167 reference at locations away from detected sources. 

1168 

1169 Parameters 

1170 ---------- 

1171 modelImages : `list` of `lsst.afw.image.Image` 

1172 The new DCR model images from the current iteration. 

1173 The values will be modified in place. 

1174 refImage : `lsst.afw.image.MaskedImage` 

1175 A reference image used to supply the default pixel values. 

1176 modelWeights : `numpy.ndarray` or `float` 

1177 A 2D array of weight values that tapers smoothly to zero away from 

1178 detected sources. Set to a placeholder value of 1.0 if 

1179 ``self.config.useModelWeights`` is False. 

1180 """ 

1181 if self.config.useModelWeights: 

1182 for model in modelImages: 

1183 model.array *= modelWeights 

1184 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters 

1185 

1186 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList): 

1187 """Pre-load sub-regions of a list of exposures. 

1188 

1189 Parameters 

1190 ---------- 

1191 bbox : `lsst.geom.box.Box2I` 

1192 Sub-region to coadd. 

1193 statsCtrl : `lsst.afw.math.StatisticsControl` 

1194 Statistics control object for coadd. 

1195 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1196 The data references to the input warped exposures. 

1197 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

1198 The image scalars correct for the zero point of the exposures. 

1199 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None` 

1200 Each element is dict with keys = mask plane name to add the spans 

1201 to. 

1202 

1203 Returns 

1204 ------- 

1205 subExposures : `dict` 

1206 The `dict` keys are the visit IDs, 

1207 and the values are `lsst.afw.image.ExposureF` 

1208 The pre-loaded exposures for the current subregion. 

1209 The variance plane contains weights, and not the variance 

1210 """ 

1211 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList) 

1212 subExposures = {} 

1213 for warpExpRef, imageScaler, altMaskSpans in zipIterables: 

1214 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

1215 visit = warpExpRef.dataId["visit"] 

1216 if altMaskSpans is not None: 

1217 self.applyAltMaskPlanes(exposure.mask, altMaskSpans) 

1218 imageScaler.scaleMaskedImage(exposure.maskedImage) 

1219 # Note that the variance plane here is used to store weights, not 

1220 # the actual variance 

1221 exposure.variance.array[:, :] = 0. 

1222 # Set the weight of unmasked pixels to 1. 

1223 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1. 

1224 # Set the image value of masked pixels to zero. 

1225 # This eliminates needing the mask plane when stacking images in 

1226 # ``newModelFromResidual`` 

1227 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0. 

1228 subExposures[visit] = exposure 

1229 return subExposures 

1230 

1231 def selectCoaddPsf(self, templateCoadd, warpRefList): 

1232 """Compute the PSF of the coadd from the exposures with the best 

1233 seeing. 

1234 

1235 Parameters 

1236 ---------- 

1237 templateCoadd : `lsst.afw.image.ExposureF` 

1238 The initial coadd exposure before accounting for DCR. 

1239 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1240 The data references to the input warped exposures. 

1241 

1242 Returns 

1243 ------- 

1244 psf : `lsst.meas.algorithms.CoaddPsf` 

1245 The average PSF of the input exposures with the best seeing. 

1246 """ 

1247 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

1248 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry 

1249 # per ccd and per visit. If there are multiple ccds, it will have that 

1250 # many times more elements than ``warpExpRef``. 

1251 ccds = templateCoadd.getInfo().getCoaddInputs().ccds 

1252 templatePsf = templateCoadd.getPsf() 

1253 # Just need a rough estimate; average positions are fine 

1254 templateAvgPos = templatePsf.getAveragePosition() 

1255 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()*sigma2fwhm 

1256 psfSizes = np.zeros(len(ccds)) 

1257 ccdVisits = np.array(ccds["visit"]) 

1258 for warpExpRef in warpRefList: 

1259 psf = warpExpRef.get(component="psf") 

1260 visit = warpExpRef.dataId["visit"] 

1261 psfAvgPos = psf.getAveragePosition() 

1262 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm 

1263 psfSizes[ccdVisits == visit] = psfSize 

1264 # Note that the input PSFs include DCR, which should be absent from the 

1265 # DcrCoadd. The selected PSFs are those that have a FWHM less than or 

1266 # equal to the smaller of the mean or median FWHM of the input 

1267 # exposures. 

1268 sizeThreshold = min(np.median(psfSizes), psfRefSize) 

1269 goodPsfs = psfSizes <= sizeThreshold 

1270 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), 

1271 self.config.coaddPsf.makeControl()) 

1272 return psf