Coverage for python/lsst/drp/tasks/dcr_assemble_coadd.py: 15%

414 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-21 19:42 +0000

1# This file is part of drp_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"] 

23 

24from math import ceil 

25import numpy as np 

26from scipy import ndimage 

27import lsst.geom as geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.table as afwTable 

30import lsst.coadd.utils as coaddUtils 

31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel 

32import lsst.meas.algorithms as measAlg 

33from lsst.meas.base import SingleFrameMeasurementTask 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.utils as utils 

37from lsst.utils.timer import timeMethod 

38from .assemble_coadd import (AssembleCoaddConnections, 

39 AssembleCoaddTask, 

40 CompareWarpAssembleCoaddConfig, 

41 CompareWarpAssembleCoaddTask, 

42 ) 

43from lsst.pipe.tasks.coaddBase import makeSkyInfo, subBBoxIter 

44from lsst.pipe.tasks.measurePsf import MeasurePsfTask 

45 

46 

47class DcrAssembleCoaddConnections(AssembleCoaddConnections, 

48 dimensions=("tract", "patch", "band", "skymap"), 

49 defaultTemplates={"inputWarpName": "deep", 

50 "inputCoaddName": "deep", 

51 "outputCoaddName": "dcr", 

52 "warpType": "direct", 

53 "warpTypeSuffix": "", 

54 "fakesType": ""}): 

55 inputWarps = pipeBase.connectionTypes.Input( 

56 doc=("Input list of warps to be assembled i.e. stacked." 

57 "Note that this will often be different than the inputCoaddName." 

58 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

59 name="{inputWarpName}Coadd_{warpType}Warp", 

60 storageClass="ExposureF", 

61 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

62 deferLoad=True, 

63 multiple=True 

64 ) 

65 templateExposure = pipeBase.connectionTypes.Input( 

66 doc="Input coadded exposure, produced by previous call to AssembleCoadd", 

67 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}", 

68 storageClass="ExposureF", 

69 dimensions=("tract", "patch", "skymap", "band"), 

70 ) 

71 dcrCoadds = pipeBase.connectionTypes.Output( 

72 doc="Output coadded exposure, produced by stacking input warps", 

73 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

74 storageClass="ExposureF", 

75 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

76 multiple=True, 

77 ) 

78 dcrNImages = pipeBase.connectionTypes.Output( 

79 doc="Output image of number of input images per pixel", 

80 name="{outputCoaddName}Coadd_nImage", 

81 storageClass="ImageU", 

82 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

83 multiple=True, 

84 ) 

85 

86 def __init__(self, *, config=None): 

87 super().__init__(config=config) 

88 if not config.doWrite: 

89 self.outputs.remove("dcrCoadds") 

90 if not config.doNImage: 

91 self.outputs.remove("dcrNImages") 

92 # Remove outputs inherited from ``AssembleCoaddConnections`` that are 

93 # not used. 

94 self.outputs.remove("coaddExposure") 

95 self.outputs.remove("nImage") 

96 

97 

98class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, 

99 pipelineConnections=DcrAssembleCoaddConnections): 

100 dcrNumSubfilters = pexConfig.Field( 

101 dtype=int, 

102 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.", 

103 default=3, 

104 ) 

105 maxNumIter = pexConfig.Field( 

106 dtype=int, 

107 optional=True, 

108 doc="Maximum number of iterations of forward modeling.", 

109 default=None, 

110 ) 

111 minNumIter = pexConfig.Field( 

112 dtype=int, 

113 optional=True, 

114 doc="Minimum number of iterations of forward modeling.", 

115 default=None, 

116 ) 

117 convergenceThreshold = pexConfig.Field( 

118 dtype=float, 

119 doc="Target relative change in convergence between iterations of forward modeling.", 

120 default=0.001, 

121 ) 

122 useConvergence = pexConfig.Field( 

123 dtype=bool, 

124 doc="Use convergence test as a forward modeling end condition?" 

125 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations", 

126 default=True, 

127 ) 

128 baseGain = pexConfig.Field( 

129 dtype=float, 

130 optional=True, 

131 doc="Relative weight to give the new solution vs. the last solution when updating the model." 

132 "A value of 1.0 gives equal weight to both solutions." 

133 "Small values imply slower convergence of the solution, but can " 

134 "help prevent overshooting and failures in the fit." 

135 "If ``baseGain`` is None, a conservative gain " 

136 "will be calculated from the number of subfilters. ", 

137 default=None, 

138 ) 

139 useProgressiveGain = pexConfig.Field( 

140 dtype=bool, 

141 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 

142 "When calculating the next gain, we use up to 5 previous gains and convergence values." 

143 "Can be set to False to force the model to change at the rate of ``baseGain``. ", 

144 default=True, 

145 ) 

146 doAirmassWeight = pexConfig.Field( 

147 dtype=bool, 

148 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.", 

149 default=False, 

150 ) 

151 modelWeightsWidth = pexConfig.Field( 

152 dtype=float, 

153 doc="Width of the region around detected sources to include in the DcrModel.", 

154 default=3, 

155 ) 

156 useModelWeights = pexConfig.Field( 

157 dtype=bool, 

158 doc="Width of the region around detected sources to include in the DcrModel.", 

159 default=True, 

160 ) 

161 splitSubfilters = pexConfig.Field( 

162 dtype=bool, 

163 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter." 

164 "Instead of at the midpoint", 

165 default=True, 

166 ) 

167 splitThreshold = pexConfig.Field( 

168 dtype=float, 

169 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 

170 "Set to 0 to always split the subfilters.", 

171 default=0.1, 

172 ) 

173 regularizeModelIterations = pexConfig.Field( 

174 dtype=float, 

175 doc="Maximum relative change of the model allowed between iterations." 

176 "Set to zero to disable.", 

177 default=2., 

178 ) 

179 regularizeModelFrequency = pexConfig.Field( 

180 dtype=float, 

181 doc="Maximum relative change of the model allowed between subfilters." 

182 "Set to zero to disable.", 

183 default=4., 

184 ) 

185 convergenceMaskPlanes = pexConfig.ListField( 

186 dtype=str, 

187 default=["DETECTED"], 

188 doc="Mask planes to use to calculate convergence." 

189 ) 

190 regularizationWidth = pexConfig.Field( 

191 dtype=int, 

192 default=2, 

193 doc="Minimum radius of a region to include in regularization, in pixels." 

194 ) 

195 imageInterpOrder = pexConfig.Field( 

196 dtype=int, 

197 doc="The order of the spline interpolation used to shift the image plane.", 

198 default=1, 

199 ) 

200 accelerateModel = pexConfig.Field( 

201 dtype=float, 

202 doc="Factor to amplify the differences between model planes by to speed convergence.", 

203 default=3, 

204 ) 

205 doCalculatePsf = pexConfig.Field( 

206 dtype=bool, 

207 doc="Set to detect stars and recalculate the PSF from the final coadd." 

208 "Otherwise the PSF is estimated from a selection of the best input exposures", 

209 default=False, 

210 ) 

211 detectPsfSources = pexConfig.ConfigurableField( 

212 target=measAlg.SourceDetectionTask, 

213 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.", 

214 ) 

215 measurePsfSources = pexConfig.ConfigurableField( 

216 target=SingleFrameMeasurementTask, 

217 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 

218 ) 

219 measurePsf = pexConfig.ConfigurableField( 

220 target=MeasurePsfTask, 

221 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.", 

222 ) 

223 effectiveWavelength = pexConfig.Field( 

224 doc="Effective wavelength of the filter, in nm." 

225 "Required if transmission curves aren't used." 

226 "Support for using transmission curves is to be added in DM-13668.", 

227 dtype=float, 

228 ) 

229 bandwidth = pexConfig.Field( 

230 doc="Bandwidth of the physical filter, in nm." 

231 "Required if transmission curves aren't used." 

232 "Support for using transmission curves is to be added in DM-13668.", 

233 dtype=float, 

234 ) 

235 

236 def setDefaults(self): 

237 CompareWarpAssembleCoaddConfig.setDefaults(self) 

238 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask) 

239 self.doNImage = True 

240 self.assembleStaticSkyModel.warpType = self.warpType 

241 # The deepCoadd and nImage files will be overwritten by this Task, so 

242 # don't write them the first time. 

243 self.assembleStaticSkyModel.doNImage = False 

244 self.assembleStaticSkyModel.doWrite = False 

245 self.detectPsfSources.returnOriginalFootprints = False 

246 self.detectPsfSources.thresholdPolarity = "positive" 

247 # Only use bright sources for PSF measurement 

248 self.detectPsfSources.thresholdValue = 50 

249 self.detectPsfSources.nSigmaToGrow = 2 

250 # A valid star for PSF measurement should at least fill 5x5 pixels 

251 self.detectPsfSources.minPixels = 25 

252 # Use the variance plane to calculate signal to noise 

253 self.detectPsfSources.thresholdType = "pixel_stdev" 

254 # The signal to noise limit is good enough, while the flux limit is set 

255 # in dimensionless units and may not be appropriate for all data sets. 

256 self.measurePsf.starSelector["objectSize"].doFluxLimit = False 

257 # Ensure psf candidate size is as large as piff psf size. 

258 if (self.doCalculatePsf and self.measurePsf.psfDeterminer.name == "piff" 

259 and self.psfDeterminer["piff"].kernelSize > self.makePsfCandidates.kernelSize): 

260 self.makePsfCandidates.kernelSize = self.psfDeterminer["piff"].kernelSize 

261 

262 

263class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask): 

264 """Assemble DCR coadded images from a set of warps. 

265 

266 Attributes 

267 ---------- 

268 bufferSize : `int` 

269 The number of pixels to grow each subregion by to allow for DCR. 

270 

271 Notes 

272 ----- 

273 As with AssembleCoaddTask, we want to assemble a coadded image from a set 

274 of Warps (also called coadded temporary exposures), including the effects 

275 of Differential Chromatic Refraction (DCR). 

276 For full details of the mathematics and algorithm, please see 

277 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 

278 

279 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 

280 each subfilter used in the iterative calculation. 

281 It begins by dividing the bandpass-defining filter into N equal bandwidth 

282 "subfilters", and divides the flux in each pixel from an initial coadd 

283 equally into each as a "dcrModel". Because the airmass and parallactic 

284 angle of each individual exposure is known, we can calculate the shift 

285 relative to the center of the band in each subfilter due to DCR. For each 

286 exposure we apply this shift as a linear transformation to the dcrModels 

287 and stack the results to produce a DCR-matched exposure. The matched 

288 exposures are subtracted from the input exposures to produce a set of 

289 residual images, and these residuals are reverse shifted for each 

290 exposures' subfilters and stacked. The shifted and stacked residuals are 

291 added to the dcrModels to produce a new estimate of the flux in each pixel 

292 within each subfilter. The dcrModels are solved for iteratively, which 

293 continues until the solution from a new iteration improves by less than 

294 a set percentage, or a maximum number of iterations is reached. 

295 Two forms of regularization are employed to reduce unphysical results. 

296 First, the new solution is averaged with the solution from the previous 

297 iteration, which mitigates oscillating solutions where the model 

298 overshoots with alternating very high and low values. 

299 Second, a common degeneracy when the data have a limited range of airmass 

300 or parallactic angle values is for one subfilter to be fit with very low or 

301 negative values, while another subfilter is fit with very high values. This 

302 typically appears in the form of holes next to sources in one subfilter, 

303 and corresponding extended wings in another. Because each subfilter has 

304 a narrow bandwidth we assume that physical sources that are above the noise 

305 level will not vary in flux by more than a factor of `frequencyClampFactor` 

306 between subfilters, and pixels that have flux deviations larger than that 

307 factor will have the excess flux distributed evenly among all subfilters. 

308 If `splitSubfilters` is set, then each subfilter will be further sub- 

309 divided during the forward modeling step (only). This approximates using 

310 a higher number of subfilters that may be necessary for high airmass 

311 observations, but does not increase the number of free parameters in the 

312 fit. This is needed when there are high airmass observations which would 

313 otherwise have significant DCR even within a subfilter. Because calculating 

314 the shifted images takes most of the time, splitting the subfilters is 

315 turned off by way of the `splitThreshold` option for low-airmass 

316 observations that do not suffer from DCR within a subfilter. 

317 """ 

318 

319 ConfigClass = DcrAssembleCoaddConfig 

320 _DefaultName = "dcrAssembleCoadd" 

321 

322 def __init__(self, *args, **kwargs): 

323 super().__init__(*args, **kwargs) 

324 if self.config.doCalculatePsf: 

325 self.schema = afwTable.SourceTable.makeMinimalSchema() 

326 self.makeSubtask("detectPsfSources", schema=self.schema) 

327 self.makeSubtask("measurePsfSources", schema=self.schema) 

328 self.makeSubtask("measurePsf", schema=self.schema) 

329 

330 @utils.inheritDoc(pipeBase.PipelineTask) 

331 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

332 # Docstring to be formatted with info from PipelineTask.runQuantum 

333 """ 

334 Notes 

335 ----- 

336 Assemble a coadd from a set of Warps. 

337 """ 

338 inputData = butlerQC.get(inputRefs) 

339 

340 # Construct skyInfo expected by run 

341 # Do not remove skyMap from inputData in case _makeSupplementaryData 

342 # needs it. 

343 skyMap = inputData["skyMap"] 

344 outputDataId = butlerQC.quantum.dataId 

345 

346 inputData['skyInfo'] = makeSkyInfo(skyMap, 

347 tractId=outputDataId['tract'], 

348 patchId=outputDataId['patch']) 

349 

350 # Construct list of input Deferred Datasets 

351 warpRefList = inputData['inputWarps'] 

352 

353 inputs = self.prepareInputs(warpRefList) 

354 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

355 self.getTempExpDatasetName(self.warpType)) 

356 if len(inputs.tempExpRefList) == 0: 

357 self.log.warning("No coadd temporary exposures found") 

358 return 

359 

360 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

361 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

362 inputs.weightList, supplementaryData=supplementaryData) 

363 

364 inputData.setdefault('brightObjectMask', None) 

365 for subfilter in range(self.config.dcrNumSubfilters): 

366 # Use the PSF of the stacked dcrModel, and do not recalculate the 

367 # PSF for each subfilter 

368 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf()) 

369 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId) 

370 

371 if self.config.doWrite: 

372 butlerQC.put(retStruct, outputRefs) 

373 return retStruct 

374 

375 @utils.inheritDoc(AssembleCoaddTask) 

376 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

377 """Load the previously-generated template coadd. 

378 

379 Returns 

380 ------- 

381 templateCoadd : `lsst.pipe.base.Struct` 

382 Results as a struct with attributes: 

383 

384 ``templateCoadd`` 

385 Coadded exposure (`lsst.afw.image.ExposureF`). 

386 """ 

387 templateCoadd = butlerQC.get(inputRefs.templateExposure) 

388 

389 return pipeBase.Struct(templateCoadd=templateCoadd) 

390 

391 def measureCoaddPsf(self, coaddExposure): 

392 """Detect sources on the coadd exposure and measure the final PSF. 

393 

394 Parameters 

395 ---------- 

396 coaddExposure : `lsst.afw.image.Exposure` 

397 The final coadded exposure. 

398 """ 

399 table = afwTable.SourceTable.make(self.schema) 

400 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False) 

401 coaddSources = detResults.sources 

402 self.measurePsfSources.run( 

403 measCat=coaddSources, 

404 exposure=coaddExposure 

405 ) 

406 # Measure the PSF on the stacked subfilter coadds if possible. 

407 # We should already have a decent estimate of the coadd PSF, however, 

408 # so in case of any errors simply log them as a warning and use the 

409 # default PSF. 

410 try: 

411 psfResults = self.measurePsf.run(coaddExposure, coaddSources) 

412 except Exception as e: 

413 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e) 

414 else: 

415 coaddExposure.setPsf(psfResults.psf) 

416 

417 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList): 

418 """Prepare the DCR coadd by iterating through the visitInfo of the 

419 input warps. 

420 

421 Sets the property ``bufferSize``. 

422 

423 Parameters 

424 ---------- 

425 templateCoadd : `lsst.afw.image.ExposureF` 

426 The initial coadd exposure before accounting for DCR. 

427 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

428 The data references to the input warped exposures. 

429 weightList : `list` of `float` 

430 The weight to give each input exposure in the coadd. 

431 Will be modified in place if ``doAirmassWeight`` is set. 

432 

433 Returns 

434 ------- 

435 dcrModels : `lsst.pipe.tasks.DcrModel` 

436 Best fit model of the true sky after correcting chromatic effects. 

437 

438 Raises 

439 ------ 

440 NotImplementedError 

441 If ``lambdaMin`` is missing from the Mapper class of the obs 

442 package being used. 

443 """ 

444 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

445 filterLabel = templateCoadd.getFilter() 

446 dcrShifts = [] 

447 airmassDict = {} 

448 angleDict = {} 

449 psfSizeDict = {} 

450 for visitNum, warpExpRef in enumerate(warpRefList): 

451 visitInfo = warpExpRef.get(component="visitInfo") 

452 psf = warpExpRef.get(component="psf") 

453 visit = warpExpRef.dataId["visit"] 

454 # Just need a rough estimate; average positions are fine 

455 psfAvgPos = psf.getAveragePosition() 

456 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm 

457 airmass = visitInfo.getBoresightAirmass() 

458 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees() 

459 airmassDict[visit] = airmass 

460 angleDict[visit] = parallacticAngle 

461 psfSizeDict[visit] = psfSize 

462 if self.config.doAirmassWeight: 

463 weightList[visitNum] *= airmass 

464 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(), 

465 self.config.effectiveWavelength, 

466 self.config.bandwidth, 

467 self.config.dcrNumSubfilters)))) 

468 self.log.info("Selected airmasses:\n%s", airmassDict) 

469 self.log.info("Selected parallactic angles:\n%s", angleDict) 

470 self.log.info("Selected PSF sizes:\n%s", psfSizeDict) 

471 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1) 

472 try: 

473 psf = self.selectCoaddPsf(templateCoadd, warpRefList) 

474 except Exception as e: 

475 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e) 

476 else: 

477 psf = templateCoadd.getPsf() 

478 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage, 

479 self.config.dcrNumSubfilters, 

480 effectiveWavelength=self.config.effectiveWavelength, 

481 bandwidth=self.config.bandwidth, 

482 wcs=templateCoadd.getWcs(), 

483 filterLabel=filterLabel, 

484 psf=psf) 

485 return dcrModels 

486 

487 @timeMethod 

488 def run(self, skyInfo, warpRefList, imageScalerList, weightList, 

489 supplementaryData=None): 

490 r"""Assemble the coadd. 

491 

492 Requires additional inputs Struct ``supplementaryData`` to contain a 

493 ``templateCoadd`` that serves as the model of the static sky. 

494 

495 Find artifacts and apply them to the warps' masks creating a list of 

496 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

497 plane then pass these alternative masks to the base class's assemble 

498 method. 

499 

500 Divide the ``templateCoadd`` evenly between each subfilter of a 

501 ``DcrModel`` as the starting best estimate of the true wavelength- 

502 dependent sky. Forward model the ``DcrModel`` using the known 

503 chromatic effects in each subfilter and calculate a convergence metric 

504 based on how well the modeled template matches the input warps. If 

505 the convergence has not yet reached the desired threshold, then shift 

506 and stack the residual images to build a new ``DcrModel``. Apply 

507 conditioning to prevent oscillating solutions between iterations or 

508 between subfilters. 

509 

510 Once the ``DcrModel`` reaches convergence or the maximum number of 

511 iterations has been reached, fill the metadata for each subfilter 

512 image and make them proper ``coaddExposure``\ s. 

513 

514 Parameters 

515 ---------- 

516 skyInfo : `lsst.pipe.base.Struct` 

517 Patch geometry information, from getSkyInfo 

518 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

519 The data references to the input warped exposures. 

520 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

521 The image scalars correct for the zero point of the exposures. 

522 weightList : `list` of `float` 

523 The weight to give each input exposure in the coadd 

524 supplementaryData : `lsst.pipe.base.Struct` 

525 Result struct returned by ``_makeSupplementaryData`` with 

526 attributes: 

527 

528 ``templateCoadd`` 

529 Coadded exposure (`lsst.afw.image.Exposure`). 

530 

531 Returns 

532 ------- 

533 result : `lsst.pipe.base.Struct` 

534 Results as a struct with attributes: 

535 

536 ``coaddExposure`` 

537 Coadded exposure (`lsst.afw.image.Exposure`). 

538 ``nImage`` 

539 Exposure count image (`lsst.afw.image.ImageU`). 

540 ``dcrCoadds`` 

541 `list` of coadded exposures for each subfilter. 

542 ``dcrNImages`` 

543 `list` of exposure count images for each subfilter. 

544 """ 

545 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters 

546 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3 

547 templateCoadd = supplementaryData.templateCoadd 

548 baseMask = templateCoadd.mask.clone() 

549 # The variance plane is for each subfilter 

550 # and should be proportionately lower than the full-band image 

551 baseVariance = templateCoadd.variance.clone() 

552 baseVariance /= self.config.dcrNumSubfilters 

553 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList) 

554 # Note that the mask gets cleared in ``findArtifacts``, but we want to 

555 # preserve the mask. 

556 templateCoadd.setMask(baseMask) 

557 badMaskPlanes = self.config.badMaskPlanes[:] 

558 # Note that is important that we do not add "CLIPPED" to 

559 # ``badMaskPlanes``. This is because pixels in observations that are 

560 # significantly affected by DCR are likely to have many pixels that are 

561 # both "DETECTED" and "CLIPPED", but those are necessary to constrain 

562 # the DCR model. 

563 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes) 

564 

565 stats = self.prepareStats(mask=badPixelMask) 

566 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList) 

567 if self.config.doNImage: 

568 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList, 

569 spanSetMaskList, stats.ctrl) 

570 nImage = afwImage.ImageU(skyInfo.bbox) 

571 # Note that this nImage will be a factor of dcrNumSubfilters higher 

572 # than the nImage returned by assembleCoadd for most pixels. This 

573 # is because each subfilter may have a different nImage, and 

574 # fractional values are not allowed. 

575 for dcrNImage in dcrNImages: 

576 nImage += dcrNImage 

577 else: 

578 dcrNImages = None 

579 

580 subregionSize = geom.Extent2I(*self.config.subregionSize) 

581 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) 

582 * ceil(skyInfo.bbox.getWidth()/subregionSize[0])) 

583 subIter = 0 

584 for subBBox in subBBoxIter(skyInfo.bbox, subregionSize): 

585 modelIter = 0 

586 subIter += 1 

587 self.log.info("Computing coadd over patch %s subregion %s of %s: %s", 

588 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox) 

589 dcrBBox = geom.Box2I(subBBox) 

590 dcrBBox.grow(self.bufferSize) 

591 dcrBBox.clip(dcrModels.bbox) 

592 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox) 

593 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList, 

594 imageScalerList, spanSetMaskList) 

595 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

596 warpRefList, weightList, stats.ctrl) 

597 self.log.info("Initial convergence : %s", convergenceMetric) 

598 convergenceList = [convergenceMetric] 

599 gainList = [] 

600 convergenceCheck = 1. 

601 refImage = templateCoadd.image 

602 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter): 

603 gain = self.calculateGain(convergenceList, gainList) 

604 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList, 

605 stats.ctrl, convergenceMetric, gain, 

606 modelWeights, refImage, dcrWeights) 

607 if self.config.useConvergence: 

608 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

609 warpRefList, weightList, stats.ctrl) 

610 if convergenceMetric == 0: 

611 self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is " 

612 "most likely due to there being no valid data in the region.", 

613 skyInfo.patchInfo.getIndex(), subIter) 

614 break 

615 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric 

616 if (convergenceCheck < 0) & (modelIter > minNumIter): 

617 self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum " 

618 "iterations or desired convergence improvement of %s." 

619 " Divergence: %s", 

620 skyInfo.patchInfo.getIndex(), subIter, 

621 self.config.convergenceThreshold, convergenceCheck) 

622 break 

623 convergenceList.append(convergenceMetric) 

624 if modelIter > maxNumIter: 

625 if self.config.useConvergence: 

626 self.log.warning("Coadd patch %s subregion %s reached maximum iterations " 

627 "before reaching desired convergence improvement of %s." 

628 " Final convergence improvement: %s", 

629 skyInfo.patchInfo.getIndex(), subIter, 

630 self.config.convergenceThreshold, convergenceCheck) 

631 break 

632 

633 if self.config.useConvergence: 

634 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)", 

635 modelIter, convergenceMetric, 100.*convergenceCheck, gain) 

636 modelIter += 1 

637 else: 

638 if self.config.useConvergence: 

639 self.log.info("Coadd patch %s subregion %s finished with " 

640 "convergence metric %s after %s iterations", 

641 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter) 

642 else: 

643 self.log.info("Coadd patch %s subregion %s finished after %s iterations", 

644 skyInfo.patchInfo.getIndex(), subIter, modelIter) 

645 if self.config.useConvergence and convergenceMetric > 0: 

646 self.log.info("Final convergence improvement was %.4f%% overall", 

647 100*(convergenceList[0] - convergenceMetric)/convergenceMetric) 

648 

649 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList, 

650 calibration=self.scaleZeroPoint.getPhotoCalib(), 

651 coaddInputs=templateCoadd.getInfo().getCoaddInputs(), 

652 mask=baseMask, 

653 variance=baseVariance) 

654 coaddExposure = self.stackCoadd(dcrCoadds) 

655 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

656 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages) 

657 

658 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl): 

659 """Calculate the number of exposures contributing to each subfilter. 

660 

661 Parameters 

662 ---------- 

663 dcrModels : `lsst.pipe.tasks.DcrModel` 

664 Best fit model of the true sky after correcting chromatic effects. 

665 bbox : `lsst.geom.box.Box2I` 

666 Bounding box of the patch to coadd. 

667 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

668 The data references to the input warped exposures. 

669 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None` 

670 Each element of the `dict` contains the new mask plane name 

671 (e.g. "CLIPPED and/or "NO_DATA") as the key, 

672 and the list of SpanSets to apply to the mask. 

673 statsCtrl : `lsst.afw.math.StatisticsControl` 

674 Statistics control object for coadd 

675 

676 Returns 

677 ------- 

678 dcrNImages : `list` of `lsst.afw.image.ImageU` 

679 List of exposure count images for each subfilter. 

680 dcrWeights : `list` of `lsst.afw.image.ImageF` 

681 Per-pixel weights for each subfilter. 

682 Equal to 1/(number of unmasked images contributing to each pixel). 

683 """ 

684 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

685 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

686 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList): 

687 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

688 visitInfo = exposure.getInfo().getVisitInfo() 

689 wcs = exposure.getInfo().getWcs() 

690 mask = exposure.mask 

691 if altMaskSpans is not None: 

692 self.applyAltMaskPlanes(mask, altMaskSpans) 

693 weightImage = np.zeros_like(exposure.image.array) 

694 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1. 

695 # The weights must be shifted in exactly the same way as the 

696 # residuals, because they will be used as the denominator in the 

697 # weighted average of residuals. 

698 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, 

699 dcrModels.effectiveWavelength, dcrModels.bandwidth) 

700 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights): 

701 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype) 

702 dcrWeight.array += shiftedWeights 

703 # Exclude any pixels that don't have at least one exposure contributing 

704 # in all subfilters 

705 weightsThreshold = 1. 

706 goodPix = dcrWeights[0].array > weightsThreshold 

707 for weights in dcrWeights[1:]: 

708 goodPix = (weights.array > weightsThreshold) & goodPix 

709 for subfilter in range(self.config.dcrNumSubfilters): 

710 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix] 

711 dcrWeights[subfilter].array[~goodPix] = 0. 

712 dcrNImages[subfilter].array[~goodPix] = 0 

713 return (dcrNImages, dcrWeights) 

714 

715 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, 

716 statsCtrl, convergenceMetric, 

717 gain, modelWeights, refImage, dcrWeights): 

718 """Assemble the DCR coadd for a sub-region. 

719 

720 Build a DCR-matched template for each input exposure, then shift the 

721 residuals according to the DCR in each subfilter. 

722 Stack the shifted residuals and apply them as a correction to the 

723 solution from the previous iteration. 

724 Restrict the new model solutions from varying by more than a factor of 

725 `modelClampFactor` from the last solution, and additionally restrict 

726 the individual subfilter models from varying by more than a factor of 

727 `frequencyClampFactor` from their average. 

728 Finally, mitigate potentially oscillating solutions by averaging the 

729 new solution with the solution from the previous iteration, weighted by 

730 their convergence metric. 

731 

732 Parameters 

733 ---------- 

734 dcrModels : `lsst.pipe.tasks.DcrModel` 

735 Best fit model of the true sky after correcting chromatic effects. 

736 subExposures : `dict` of `lsst.afw.image.ExposureF` 

737 The pre-loaded exposures for the current subregion. 

738 bbox : `lsst.geom.box.Box2I` 

739 Bounding box of the subregion to coadd. 

740 dcrBBox : `lsst.geom.box.Box2I` 

741 Sub-region of the coadd which includes a buffer to allow for DCR. 

742 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

743 The data references to the input warped exposures. 

744 statsCtrl : `lsst.afw.math.StatisticsControl` 

745 Statistics control object for coadd. 

746 convergenceMetric : `float` 

747 Quality of fit metric for the matched templates of the input 

748 images. 

749 gain : `float`, optional 

750 Relative weight to give the new solution when updating the model. 

751 modelWeights : `numpy.ndarray` or `float` 

752 A 2D array of weight values that tapers smoothly to zero away from 

753 detected sources. Set to a placeholder value of 1.0 if 

754 ``self.config.useModelWeights`` is False. 

755 refImage : `lsst.afw.image.Image` 

756 A reference image used to supply the default pixel values. 

757 dcrWeights : `list` of `lsst.afw.image.Image` 

758 Per-pixel weights for each subfilter. 

759 Equal to 1/(number of unmasked images contributing to each pixel). 

760 """ 

761 residualGeneratorList = [] 

762 

763 for warpExpRef in warpRefList: 

764 visit = warpExpRef.dataId["visit"] 

765 exposure = subExposures[visit] 

766 visitInfo = exposure.getInfo().getVisitInfo() 

767 wcs = exposure.getInfo().getWcs() 

768 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

769 bbox=exposure.getBBox(), 

770 order=self.config.imageInterpOrder, 

771 splitSubfilters=self.config.splitSubfilters, 

772 splitThreshold=self.config.splitThreshold, 

773 amplifyModel=self.config.accelerateModel) 

774 residual = exposure.image.array - templateImage.array 

775 # Note that the variance plane here is used to store weights, not 

776 # the actual variance 

777 residual *= exposure.variance.array 

778 # The residuals are stored as a list of generators. 

779 # This allows the residual for a given subfilter and exposure to be 

780 # created on the fly, instead of needing to store them all in 

781 # memory. 

782 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, 

783 dcrModels.effectiveWavelength, 

784 dcrModels.bandwidth)) 

785 

786 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

787 gain=gain, 

788 modelWeights=modelWeights, 

789 refImage=refImage, 

790 dcrWeights=dcrWeights) 

791 dcrModels.assign(dcrSubModelOut, bbox) 

792 

793 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth): 

794 """Prepare a residual image for stacking in each subfilter by applying 

795 the reverse DCR shifts. 

796 

797 Parameters 

798 ---------- 

799 residual : `numpy.ndarray` 

800 The residual masked image for one exposure, 

801 after subtracting the matched template. 

802 visitInfo : `lsst.afw.image.VisitInfo` 

803 Metadata for the exposure. 

804 wcs : `lsst.afw.geom.SkyWcs` 

805 Coordinate system definition (wcs) for the exposure. 

806 

807 Yields 

808 ------ 

809 residualImage : `numpy.ndarray` 

810 The residual image for the next subfilter, shifted for DCR. 

811 """ 

812 if self.config.imageInterpOrder > 1: 

813 # Pre-calculate the spline-filtered residual image, so that step 

814 # can be skipped in the shift calculation in `applyDcr`. 

815 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder) 

816 else: 

817 # No need to prefilter if order=1 (it will also raise an error) 

818 filteredResidual = residual 

819 # Note that `splitSubfilters` is always turned off in the reverse 

820 # direction. This option introduces additional blurring if applied to 

821 # the residuals. 

822 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters, 

823 splitSubfilters=False) 

824 for dcr in dcrShift: 

825 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False, 

826 doPrefilter=False, order=self.config.imageInterpOrder) 

827 

828 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

829 gain, modelWeights, refImage, dcrWeights): 

830 """Calculate a new DcrModel from a set of image residuals. 

831 

832 Parameters 

833 ---------- 

834 dcrModels : `lsst.pipe.tasks.DcrModel` 

835 Current model of the true sky after correcting chromatic effects. 

836 residualGeneratorList : `generator` of `numpy.ndarray` 

837 The residual image for the next subfilter, shifted for DCR. 

838 dcrBBox : `lsst.geom.box.Box2I` 

839 Sub-region of the coadd which includes a buffer to allow for DCR. 

840 statsCtrl : `lsst.afw.math.StatisticsControl` 

841 Statistics control object for coadd. 

842 gain : `float` 

843 Relative weight to give the new solution when updating the model. 

844 modelWeights : `numpy.ndarray` or `float` 

845 A 2D array of weight values that tapers smoothly to zero away from 

846 detected sources. Set to a placeholder value of 1.0 if 

847 ``self.config.useModelWeights`` is False. 

848 refImage : `lsst.afw.image.Image` 

849 A reference image used to supply the default pixel values. 

850 dcrWeights : `list` of `lsst.afw.image.Image` 

851 Per-pixel weights for each subfilter. 

852 Equal to 1/(number of unmasked images contributing to each pixel). 

853 

854 Returns 

855 ------- 

856 dcrModel : `lsst.pipe.tasks.DcrModel` 

857 New model of the true sky after correcting chromatic effects. 

858 """ 

859 newModelImages = [] 

860 for subfilter, model in enumerate(dcrModels): 

861 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList] 

862 residual = np.sum(residualsList, axis=0) 

863 residual *= dcrWeights[subfilter][dcrBBox].array 

864 # `MaskedImage`s only support in-place addition, so rename for 

865 # readability. 

866 newModel = model[dcrBBox].clone() 

867 newModel.array += residual 

868 # Catch any invalid values 

869 badPixels = ~np.isfinite(newModel.array) 

870 newModel.array[badPixels] = model[dcrBBox].array[badPixels] 

871 if self.config.regularizeModelIterations > 0: 

872 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox, 

873 self.config.regularizeModelIterations, 

874 self.config.regularizationWidth) 

875 newModelImages.append(newModel) 

876 if self.config.regularizeModelFrequency > 0: 

877 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl, 

878 self.config.regularizeModelFrequency, 

879 self.config.regularizationWidth) 

880 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain) 

881 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights) 

882 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength, 

883 dcrModels.bandwidth, dcrModels.psf, 

884 dcrModels.mask, dcrModels.variance) 

885 

886 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl): 

887 """Calculate a quality of fit metric for the matched templates. 

888 

889 Parameters 

890 ---------- 

891 dcrModels : `lsst.pipe.tasks.DcrModel` 

892 Best fit model of the true sky after correcting chromatic effects. 

893 subExposures : `dict` of `lsst.afw.image.ExposureF` 

894 The pre-loaded exposures for the current subregion. 

895 bbox : `lsst.geom.box.Box2I` 

896 Sub-region to coadd. 

897 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

898 The data references to the input warped exposures. 

899 weightList : `list` of `float` 

900 The weight to give each input exposure in the coadd. 

901 statsCtrl : `lsst.afw.math.StatisticsControl` 

902 Statistics control object for coadd. 

903 

904 Returns 

905 ------- 

906 convergenceMetric : `float` 

907 Quality of fit metric for all input exposures, within the 

908 sub-region. 

909 """ 

910 significanceImage = np.abs(dcrModels.getReferenceImage(bbox)) 

911 nSigma = 3. 

912 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl, 

913 bufferSize=self.bufferSize) 

914 if np.max(significanceImage) == 0: 

915 significanceImage += 1. 

916 weight = 0 

917 metric = 0. 

918 metricList = {} 

919 for warpExpRef, expWeight in zip(warpRefList, weightList): 

920 visit = warpExpRef.dataId["visit"] 

921 exposure = subExposures[visit][bbox] 

922 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl) 

923 metric += singleMetric 

924 metricList[visit] = singleMetric 

925 weight += 1. 

926 self.log.info("Individual metrics:\n%s", metricList) 

927 return 1.0 if weight == 0.0 else metric/weight 

928 

929 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl): 

930 """Calculate a quality of fit metric for a single matched template. 

931 

932 Parameters 

933 ---------- 

934 dcrModels : `lsst.pipe.tasks.DcrModel` 

935 Best fit model of the true sky after correcting chromatic effects. 

936 exposure : `lsst.afw.image.ExposureF` 

937 The input warped exposure to evaluate. 

938 significanceImage : `numpy.ndarray` 

939 Array of weights for each pixel corresponding to its significance 

940 for the convergence calculation. 

941 statsCtrl : `lsst.afw.math.StatisticsControl` 

942 Statistics control object for coadd. 

943 

944 Returns 

945 ------- 

946 convergenceMetric : `float` 

947 Quality of fit metric for one exposure, within the sub-region. 

948 """ 

949 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

950 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

951 bbox=exposure.getBBox(), 

952 order=self.config.imageInterpOrder, 

953 splitSubfilters=self.config.splitSubfilters, 

954 splitThreshold=self.config.splitThreshold, 

955 amplifyModel=self.config.accelerateModel) 

956 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage 

957 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2. 

958 

959 finitePixels = np.isfinite(diffVals) 

960 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0 

961 convergeMaskPixels = exposure.mask.array & convergeMask > 0 

962 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels 

963 if np.sum(usePixels) == 0: 

964 metric = 0. 

965 else: 

966 diffUse = diffVals[usePixels] 

967 refUse = refVals[usePixels] 

968 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse)) 

969 return metric 

970 

971 def stackCoadd(self, dcrCoadds): 

972 """Add a list of sub-band coadds together. 

973 

974 Parameters 

975 ---------- 

976 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

977 A list of coadd exposures, each exposure containing 

978 the model for one subfilter. 

979 

980 Returns 

981 ------- 

982 coaddExposure : `lsst.afw.image.ExposureF` 

983 A single coadd exposure that is the sum of the sub-bands. 

984 """ 

985 coaddExposure = dcrCoadds[0].clone() 

986 for coadd in dcrCoadds[1:]: 

987 coaddExposure.maskedImage += coadd.maskedImage 

988 return coaddExposure 

989 

990 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, 

991 mask=None, variance=None): 

992 """Create a list of coadd exposures from a list of masked images. 

993 

994 Parameters 

995 ---------- 

996 dcrModels : `lsst.pipe.tasks.DcrModel` 

997 Best fit model of the true sky after correcting chromatic effects. 

998 skyInfo : `lsst.pipe.base.Struct` 

999 Patch geometry information, from getSkyInfo. 

1000 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1001 The data references to the input warped exposures. 

1002 weightList : `list` of `float` 

1003 The weight to give each input exposure in the coadd. 

1004 calibration : `lsst.afw.Image.PhotoCalib`, optional 

1005 Scale factor to set the photometric calibration of an exposure. 

1006 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 

1007 A record of the observations that are included in the coadd. 

1008 mask : `lsst.afw.image.Mask`, optional 

1009 Optional mask to override the values in the final coadd. 

1010 variance : `lsst.afw.image.Image`, optional 

1011 Optional variance plane to override the values in the final coadd. 

1012 

1013 Returns 

1014 ------- 

1015 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1016 A list of coadd exposures, each exposure containing 

1017 the model for one subfilter. 

1018 """ 

1019 dcrCoadds = [] 

1020 refModel = dcrModels.getReferenceImage() 

1021 for model in dcrModels: 

1022 if self.config.accelerateModel > 1: 

1023 model.array = (model.array - refModel)*self.config.accelerateModel + refModel 

1024 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

1025 if calibration is not None: 

1026 coaddExposure.setPhotoCalib(calibration) 

1027 if coaddInputs is not None: 

1028 coaddExposure.getInfo().setCoaddInputs(coaddInputs) 

1029 # Set the metadata for the coadd, including PSF and aperture 

1030 # corrections. 

1031 self.assembleMetadata(coaddExposure, warpRefList, weightList) 

1032 # Overwrite the PSF 

1033 coaddExposure.setPsf(dcrModels.psf) 

1034 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox]) 

1035 maskedImage = afwImage.MaskedImageF(dcrModels.bbox) 

1036 maskedImage.image = model 

1037 maskedImage.mask = dcrModels.mask 

1038 maskedImage.variance = dcrModels.variance 

1039 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox]) 

1040 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

1041 if mask is not None: 

1042 coaddExposure.setMask(mask) 

1043 if variance is not None: 

1044 coaddExposure.setVariance(variance) 

1045 dcrCoadds.append(coaddExposure) 

1046 return dcrCoadds 

1047 

1048 def calculateGain(self, convergenceList, gainList): 

1049 """Calculate the gain to use for the current iteration. 

1050 

1051 After calculating a new DcrModel, each value is averaged with the 

1052 value in the corresponding pixel from the previous iteration. This 

1053 reduces oscillating solutions that iterative techniques are plagued by, 

1054 and speeds convergence. By far the biggest changes to the model 

1055 happen in the first couple iterations, so we can also use a more 

1056 aggressive gain later when the model is changing slowly. 

1057 

1058 Parameters 

1059 ---------- 

1060 convergenceList : `list` of `float` 

1061 The quality of fit metric from each previous iteration. 

1062 gainList : `list` of `float` 

1063 The gains used in each previous iteration: appended with the new 

1064 gain value. 

1065 Gains are numbers between ``self.config.baseGain`` and 1. 

1066 

1067 Returns 

1068 ------- 

1069 gain : `float` 

1070 Relative weight to give the new solution when updating the model. 

1071 A value of 1.0 gives equal weight to both solutions. 

1072 

1073 Raises 

1074 ------ 

1075 ValueError 

1076 If ``len(convergenceList) != len(gainList)+1``. 

1077 """ 

1078 nIter = len(convergenceList) 

1079 if nIter != len(gainList) + 1: 

1080 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)." 

1081 % (len(convergenceList), len(gainList))) 

1082 

1083 if self.config.baseGain is None: 

1084 # If ``baseGain`` is not set, calculate it from the number of DCR 

1085 # subfilters. The more subfilters being modeled, the lower the gain 

1086 # should be. 

1087 baseGain = 1./(self.config.dcrNumSubfilters - 1) 

1088 else: 

1089 baseGain = self.config.baseGain 

1090 

1091 if self.config.useProgressiveGain and nIter > 2: 

1092 # To calculate the best gain to use, compare the past gains that 

1093 # have been used with the resulting convergences to estimate the 

1094 # best gain to use. Algorithmically, this is a Kalman filter. 

1095 # If forward modeling proceeds perfectly, the convergence metric 

1096 # should asymptotically approach a final value. We can estimate 

1097 # that value from the measured changes in convergence weighted by 

1098 # the gains used in each previous iteration. 

1099 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i] 

1100 for i in range(nIter - 1)] 

1101 # The convergence metric is strictly positive, so if the estimated 

1102 # final convergence is less than zero, force it to zero. 

1103 estFinalConv = np.array(estFinalConv) 

1104 estFinalConv[estFinalConv < 0] = 0 

1105 # Because the estimate may slowly change over time, only use the 

1106 # most recent measurements. 

1107 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):]) 

1108 lastGain = gainList[-1] 

1109 lastConv = convergenceList[-2] 

1110 newConv = convergenceList[-1] 

1111 # The predicted convergence is the value we would get if the new 

1112 # model calculated in the previous iteration was perfect. Recall 

1113 # that the updated model that is actually used is the gain-weighted 

1114 # average of the new and old model, so the convergence would be 

1115 # similarly weighted. 

1116 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain) 

1117 # If the measured and predicted convergence are very close, that 

1118 # indicates that our forward model is accurate and we can use a 

1119 # more aggressive gain. If the measured convergence is 

1120 # significantly worse (or better!) than predicted, that indicates 

1121 # that the model is not converging as expected and we should use a 

1122 # more conservative gain. 

1123 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain)) 

1124 newGain = 1 - abs(delta) 

1125 # Average the gains to prevent oscillating solutions. 

1126 newGain = (newGain + lastGain)/2. 

1127 gain = max(baseGain, newGain) 

1128 else: 

1129 gain = baseGain 

1130 gainList.append(gain) 

1131 return gain 

1132 

1133 def calculateModelWeights(self, dcrModels, dcrBBox): 

1134 """Build an array that smoothly tapers to 0 away from detected sources. 

1135 

1136 Parameters 

1137 ---------- 

1138 dcrModels : `lsst.pipe.tasks.DcrModel` 

1139 Best fit model of the true sky after correcting chromatic effects. 

1140 dcrBBox : `lsst.geom.box.Box2I` 

1141 Sub-region of the coadd which includes a buffer to allow for DCR. 

1142 

1143 Returns 

1144 ------- 

1145 weights : `numpy.ndarray` or `float` 

1146 A 2D array of weight values that tapers smoothly to zero away from 

1147 detected sources. Set to a placeholder value of 1.0 if 

1148 ``self.config.useModelWeights`` is False. 

1149 

1150 Raises 

1151 ------ 

1152 ValueError 

1153 If ``useModelWeights`` is set and ``modelWeightsWidth`` is 

1154 negative. 

1155 """ 

1156 if not self.config.useModelWeights: 

1157 return 1.0 

1158 if self.config.modelWeightsWidth < 0: 

1159 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set") 

1160 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1161 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0 

1162 weights = np.zeros_like(dcrModels[0][dcrBBox].array) 

1163 weights[convergeMaskPixels] = 1. 

1164 weights = ndimage.gaussian_filter(weights, self.config.modelWeightsWidth) 

1165 weights /= np.max(weights) 

1166 return weights 

1167 

1168 def applyModelWeights(self, modelImages, refImage, modelWeights): 

1169 """Smoothly replace model pixel values with those from a 

1170 reference at locations away from detected sources. 

1171 

1172 Parameters 

1173 ---------- 

1174 modelImages : `list` of `lsst.afw.image.Image` 

1175 The new DCR model images from the current iteration. 

1176 The values will be modified in place. 

1177 refImage : `lsst.afw.image.MaskedImage` 

1178 A reference image used to supply the default pixel values. 

1179 modelWeights : `numpy.ndarray` or `float` 

1180 A 2D array of weight values that tapers smoothly to zero away from 

1181 detected sources. Set to a placeholder value of 1.0 if 

1182 ``self.config.useModelWeights`` is False. 

1183 """ 

1184 if self.config.useModelWeights: 

1185 for model in modelImages: 

1186 model.array *= modelWeights 

1187 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters 

1188 

1189 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList): 

1190 """Pre-load sub-regions of a list of exposures. 

1191 

1192 Parameters 

1193 ---------- 

1194 bbox : `lsst.geom.box.Box2I` 

1195 Sub-region to coadd. 

1196 statsCtrl : `lsst.afw.math.StatisticsControl` 

1197 Statistics control object for coadd. 

1198 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1199 The data references to the input warped exposures. 

1200 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

1201 The image scalars correct for the zero point of the exposures. 

1202 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None` 

1203 Each element is dict with keys = mask plane name to add the spans 

1204 to. 

1205 

1206 Returns 

1207 ------- 

1208 subExposures : `dict` 

1209 The `dict` keys are the visit IDs, 

1210 and the values are `lsst.afw.image.ExposureF` 

1211 The pre-loaded exposures for the current subregion. 

1212 The variance plane contains weights, and not the variance 

1213 """ 

1214 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList) 

1215 subExposures = {} 

1216 for warpExpRef, imageScaler, altMaskSpans in zipIterables: 

1217 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

1218 visit = warpExpRef.dataId["visit"] 

1219 if altMaskSpans is not None: 

1220 self.applyAltMaskPlanes(exposure.mask, altMaskSpans) 

1221 imageScaler.scaleMaskedImage(exposure.maskedImage) 

1222 # Note that the variance plane here is used to store weights, not 

1223 # the actual variance 

1224 exposure.variance.array[:, :] = 0. 

1225 # Set the weight of unmasked pixels to 1. 

1226 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1. 

1227 # Set the image value of masked pixels to zero. 

1228 # This eliminates needing the mask plane when stacking images in 

1229 # ``newModelFromResidual`` 

1230 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0. 

1231 subExposures[visit] = exposure 

1232 return subExposures 

1233 

1234 def selectCoaddPsf(self, templateCoadd, warpRefList): 

1235 """Compute the PSF of the coadd from the exposures with the best 

1236 seeing. 

1237 

1238 Parameters 

1239 ---------- 

1240 templateCoadd : `lsst.afw.image.ExposureF` 

1241 The initial coadd exposure before accounting for DCR. 

1242 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1243 The data references to the input warped exposures. 

1244 

1245 Returns 

1246 ------- 

1247 psf : `lsst.meas.algorithms.CoaddPsf` 

1248 The average PSF of the input exposures with the best seeing. 

1249 """ 

1250 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

1251 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry 

1252 # per ccd and per visit. If there are multiple ccds, it will have that 

1253 # many times more elements than ``warpExpRef``. 

1254 ccds = templateCoadd.getInfo().getCoaddInputs().ccds 

1255 templatePsf = templateCoadd.getPsf() 

1256 # Just need a rough estimate; average positions are fine 

1257 templateAvgPos = templatePsf.getAveragePosition() 

1258 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()*sigma2fwhm 

1259 psfSizes = np.zeros(len(ccds)) 

1260 ccdVisits = np.array(ccds["visit"]) 

1261 for warpExpRef in warpRefList: 

1262 psf = warpExpRef.get(component="psf") 

1263 visit = warpExpRef.dataId["visit"] 

1264 psfAvgPos = psf.getAveragePosition() 

1265 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm 

1266 psfSizes[ccdVisits == visit] = psfSize 

1267 # Note that the input PSFs include DCR, which should be absent from the 

1268 # DcrCoadd. The selected PSFs are those that have a FWHM less than or 

1269 # equal to the smaller of the mean or median FWHM of the input 

1270 # exposures. 

1271 sizeThreshold = min(np.median(psfSizes), psfRefSize) 

1272 goodPsfs = psfSizes <= sizeThreshold 

1273 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), 

1274 self.config.coaddPsf.makeControl()) 

1275 return psf