Coverage for python/lsst/pipe/tasks/dcrAssembleCoadd.py: 14%

414 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-02 10:25 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"] 

23 

24from math import ceil 

25import numpy as np 

26from scipy import ndimage 

27import lsst.geom as geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.table as afwTable 

30import lsst.coadd.utils as coaddUtils 

31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel 

32import lsst.meas.algorithms as measAlg 

33from lsst.meas.base import SingleFrameMeasurementTask 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.utils as utils 

37from lsst.utils.timer import timeMethod 

38from .assembleCoadd import (AssembleCoaddConnections, 

39 AssembleCoaddTask, 

40 CompareWarpAssembleCoaddConfig, 

41 CompareWarpAssembleCoaddTask) 

42from .coaddBase import makeSkyInfo 

43from .measurePsf import MeasurePsfTask 

44 

45 

46class DcrAssembleCoaddConnections(AssembleCoaddConnections, 

47 dimensions=("tract", "patch", "band", "skymap"), 

48 defaultTemplates={"inputWarpName": "deep", 

49 "inputCoaddName": "deep", 

50 "outputCoaddName": "dcr", 

51 "warpType": "direct", 

52 "warpTypeSuffix": "", 

53 "fakesType": ""}): 

54 inputWarps = pipeBase.connectionTypes.Input( 

55 doc=("Input list of warps to be assembled i.e. stacked." 

56 "Note that this will often be different than the inputCoaddName." 

57 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

58 name="{inputWarpName}Coadd_{warpType}Warp", 

59 storageClass="ExposureF", 

60 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

61 deferLoad=True, 

62 multiple=True 

63 ) 

64 templateExposure = pipeBase.connectionTypes.Input( 

65 doc="Input coadded exposure, produced by previous call to AssembleCoadd", 

66 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}", 

67 storageClass="ExposureF", 

68 dimensions=("tract", "patch", "skymap", "band"), 

69 ) 

70 dcrCoadds = pipeBase.connectionTypes.Output( 

71 doc="Output coadded exposure, produced by stacking input warps", 

72 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

73 storageClass="ExposureF", 

74 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

75 multiple=True, 

76 ) 

77 dcrNImages = pipeBase.connectionTypes.Output( 

78 doc="Output image of number of input images per pixel", 

79 name="{outputCoaddName}Coadd_nImage", 

80 storageClass="ImageU", 

81 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

82 multiple=True, 

83 ) 

84 

85 def __init__(self, *, config=None): 

86 super().__init__(config=config) 

87 if not config.doWrite: 

88 self.outputs.remove("dcrCoadds") 

89 if not config.doNImage: 

90 self.outputs.remove("dcrNImages") 

91 # Remove outputs inherited from ``AssembleCoaddConnections`` that are not used 

92 self.outputs.remove("coaddExposure") 

93 self.outputs.remove("nImage") 

94 

95 

96class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, 

97 pipelineConnections=DcrAssembleCoaddConnections): 

98 dcrNumSubfilters = pexConfig.Field( 

99 dtype=int, 

100 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.", 

101 default=3, 

102 ) 

103 maxNumIter = pexConfig.Field( 

104 dtype=int, 

105 optional=True, 

106 doc="Maximum number of iterations of forward modeling.", 

107 default=None, 

108 ) 

109 minNumIter = pexConfig.Field( 

110 dtype=int, 

111 optional=True, 

112 doc="Minimum number of iterations of forward modeling.", 

113 default=None, 

114 ) 

115 convergenceThreshold = pexConfig.Field( 

116 dtype=float, 

117 doc="Target relative change in convergence between iterations of forward modeling.", 

118 default=0.001, 

119 ) 

120 useConvergence = pexConfig.Field( 

121 dtype=bool, 

122 doc="Use convergence test as a forward modeling end condition?" 

123 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations", 

124 default=True, 

125 ) 

126 baseGain = pexConfig.Field( 

127 dtype=float, 

128 optional=True, 

129 doc="Relative weight to give the new solution vs. the last solution when updating the model." 

130 "A value of 1.0 gives equal weight to both solutions." 

131 "Small values imply slower convergence of the solution, but can " 

132 "help prevent overshooting and failures in the fit." 

133 "If ``baseGain`` is None, a conservative gain " 

134 "will be calculated from the number of subfilters. ", 

135 default=None, 

136 ) 

137 useProgressiveGain = pexConfig.Field( 

138 dtype=bool, 

139 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 

140 "When calculating the next gain, we use up to 5 previous gains and convergence values." 

141 "Can be set to False to force the model to change at the rate of ``baseGain``. ", 

142 default=True, 

143 ) 

144 doAirmassWeight = pexConfig.Field( 

145 dtype=bool, 

146 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.", 

147 default=False, 

148 ) 

149 modelWeightsWidth = pexConfig.Field( 

150 dtype=float, 

151 doc="Width of the region around detected sources to include in the DcrModel.", 

152 default=3, 

153 ) 

154 useModelWeights = pexConfig.Field( 

155 dtype=bool, 

156 doc="Width of the region around detected sources to include in the DcrModel.", 

157 default=True, 

158 ) 

159 splitSubfilters = pexConfig.Field( 

160 dtype=bool, 

161 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter." 

162 "Instead of at the midpoint", 

163 default=True, 

164 ) 

165 splitThreshold = pexConfig.Field( 

166 dtype=float, 

167 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 

168 "Set to 0 to always split the subfilters.", 

169 default=0.1, 

170 ) 

171 regularizeModelIterations = pexConfig.Field( 

172 dtype=float, 

173 doc="Maximum relative change of the model allowed between iterations." 

174 "Set to zero to disable.", 

175 default=2., 

176 ) 

177 regularizeModelFrequency = pexConfig.Field( 

178 dtype=float, 

179 doc="Maximum relative change of the model allowed between subfilters." 

180 "Set to zero to disable.", 

181 default=4., 

182 ) 

183 convergenceMaskPlanes = pexConfig.ListField( 

184 dtype=str, 

185 default=["DETECTED"], 

186 doc="Mask planes to use to calculate convergence." 

187 ) 

188 regularizationWidth = pexConfig.Field( 

189 dtype=int, 

190 default=2, 

191 doc="Minimum radius of a region to include in regularization, in pixels." 

192 ) 

193 imageInterpOrder = pexConfig.Field( 

194 dtype=int, 

195 doc="The order of the spline interpolation used to shift the image plane.", 

196 default=1, 

197 ) 

198 accelerateModel = pexConfig.Field( 

199 dtype=float, 

200 doc="Factor to amplify the differences between model planes by to speed convergence.", 

201 default=3, 

202 ) 

203 doCalculatePsf = pexConfig.Field( 

204 dtype=bool, 

205 doc="Set to detect stars and recalculate the PSF from the final coadd." 

206 "Otherwise the PSF is estimated from a selection of the best input exposures", 

207 default=False, 

208 ) 

209 detectPsfSources = pexConfig.ConfigurableField( 

210 target=measAlg.SourceDetectionTask, 

211 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.", 

212 ) 

213 measurePsfSources = pexConfig.ConfigurableField( 

214 target=SingleFrameMeasurementTask, 

215 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 

216 ) 

217 measurePsf = pexConfig.ConfigurableField( 

218 target=MeasurePsfTask, 

219 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.", 

220 ) 

221 effectiveWavelength = pexConfig.Field( 

222 doc="Effective wavelength of the filter, in nm." 

223 "Required if transmission curves aren't used." 

224 "Support for using transmission curves is to be added in DM-13668.", 

225 dtype=float, 

226 ) 

227 bandwidth = pexConfig.Field( 

228 doc="Bandwidth of the physical filter, in nm." 

229 "Required if transmission curves aren't used." 

230 "Support for using transmission curves is to be added in DM-13668.", 

231 dtype=float, 

232 ) 

233 

234 def setDefaults(self): 

235 CompareWarpAssembleCoaddConfig.setDefaults(self) 

236 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask) 

237 self.doNImage = True 

238 self.assembleStaticSkyModel.warpType = self.warpType 

239 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time 

240 self.assembleStaticSkyModel.doNImage = False 

241 self.assembleStaticSkyModel.doWrite = False 

242 self.detectPsfSources.returnOriginalFootprints = False 

243 self.detectPsfSources.thresholdPolarity = "positive" 

244 # Only use bright sources for PSF measurement 

245 self.detectPsfSources.thresholdValue = 50 

246 self.detectPsfSources.nSigmaToGrow = 2 

247 # A valid star for PSF measurement should at least fill 5x5 pixels 

248 self.detectPsfSources.minPixels = 25 

249 # Use the variance plane to calculate signal to noise 

250 self.detectPsfSources.thresholdType = "pixel_stdev" 

251 # The signal to noise limit is good enough, while the flux limit is set 

252 # in dimensionless units and may not be appropriate for all data sets. 

253 self.measurePsf.starSelector["objectSize"].doFluxLimit = False 

254 # Ensure psf candidate size is as large as piff psf size. 

255 if (self.doCalculatePsf and self.measurePsf.psfDeterminer.name == "piff" 

256 and self.psfDeterminer["piff"].kernelSize > self.makePsfCandidates.kernelSize): 

257 self.makePsfCandidates.kernelSize = self.psfDeterminer["piff"].kernelSize 

258 

259 

260class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask): 

261 """Assemble DCR coadded images from a set of warps. 

262 

263 Attributes 

264 ---------- 

265 bufferSize : `int` 

266 The number of pixels to grow each subregion by to allow for DCR. 

267 

268 Notes 

269 ----- 

270 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 

271 Warps (also called coadded temporary exposures), including the effects of 

272 Differential Chromatic Refraction (DCR). 

273 For full details of the mathematics and algorithm, please see 

274 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 

275 

276 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 

277 each subfilter used in the iterative calculation. 

278 It begins by dividing the bandpass-defining filter into N equal bandwidth 

279 "subfilters", and divides the flux in each pixel from an initial coadd 

280 equally into each as a "dcrModel". Because the airmass and parallactic 

281 angle of each individual exposure is known, we can calculate the shift 

282 relative to the center of the band in each subfilter due to DCR. For each 

283 exposure we apply this shift as a linear transformation to the dcrModels 

284 and stack the results to produce a DCR-matched exposure. The matched 

285 exposures are subtracted from the input exposures to produce a set of 

286 residual images, and these residuals are reverse shifted for each 

287 exposures' subfilters and stacked. The shifted and stacked residuals are 

288 added to the dcrModels to produce a new estimate of the flux in each pixel 

289 within each subfilter. The dcrModels are solved for iteratively, which 

290 continues until the solution from a new iteration improves by less than 

291 a set percentage, or a maximum number of iterations is reached. 

292 Two forms of regularization are employed to reduce unphysical results. 

293 First, the new solution is averaged with the solution from the previous 

294 iteration, which mitigates oscillating solutions where the model 

295 overshoots with alternating very high and low values. 

296 Second, a common degeneracy when the data have a limited range of airmass or 

297 parallactic angle values is for one subfilter to be fit with very low or 

298 negative values, while another subfilter is fit with very high values. This 

299 typically appears in the form of holes next to sources in one subfilter, 

300 and corresponding extended wings in another. Because each subfilter has 

301 a narrow bandwidth we assume that physical sources that are above the noise 

302 level will not vary in flux by more than a factor of `frequencyClampFactor` 

303 between subfilters, and pixels that have flux deviations larger than that 

304 factor will have the excess flux distributed evenly among all subfilters. 

305 If `splitSubfilters` is set, then each subfilter will be further sub- 

306 divided during the forward modeling step (only). This approximates using 

307 a higher number of subfilters that may be necessary for high airmass 

308 observations, but does not increase the number of free parameters in the 

309 fit. This is needed when there are high airmass observations which would 

310 otherwise have significant DCR even within a subfilter. Because calculating 

311 the shifted images takes most of the time, splitting the subfilters is 

312 turned off by way of the `splitThreshold` option for low-airmass 

313 observations that do not suffer from DCR within a subfilter. 

314 """ 

315 

316 ConfigClass = DcrAssembleCoaddConfig 

317 _DefaultName = "dcrAssembleCoadd" 

318 

319 def __init__(self, *args, **kwargs): 

320 super().__init__(*args, **kwargs) 

321 if self.config.doCalculatePsf: 

322 self.schema = afwTable.SourceTable.makeMinimalSchema() 

323 self.makeSubtask("detectPsfSources", schema=self.schema) 

324 self.makeSubtask("measurePsfSources", schema=self.schema) 

325 self.makeSubtask("measurePsf", schema=self.schema) 

326 

327 @utils.inheritDoc(pipeBase.PipelineTask) 

328 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

329 # Docstring to be formatted with info from PipelineTask.runQuantum 

330 """ 

331 Notes 

332 ----- 

333 Assemble a coadd from a set of Warps. 

334 """ 

335 inputData = butlerQC.get(inputRefs) 

336 

337 # Construct skyInfo expected by run 

338 # Do not remove skyMap from inputData in case _makeSupplementaryData needs it 

339 skyMap = inputData["skyMap"] 

340 outputDataId = butlerQC.quantum.dataId 

341 

342 inputData['skyInfo'] = makeSkyInfo(skyMap, 

343 tractId=outputDataId['tract'], 

344 patchId=outputDataId['patch']) 

345 

346 # Construct list of input Deferred Datasets 

347 warpRefList = inputData['inputWarps'] 

348 

349 inputs = self.prepareInputs(warpRefList) 

350 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

351 self.getTempExpDatasetName(self.warpType)) 

352 if len(inputs.tempExpRefList) == 0: 

353 self.log.warning("No coadd temporary exposures found") 

354 return 

355 

356 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

357 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

358 inputs.weightList, supplementaryData=supplementaryData) 

359 

360 inputData.setdefault('brightObjectMask', None) 

361 for subfilter in range(self.config.dcrNumSubfilters): 

362 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

363 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf()) 

364 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId) 

365 

366 if self.config.doWrite: 

367 butlerQC.put(retStruct, outputRefs) 

368 return retStruct 

369 

370 @utils.inheritDoc(AssembleCoaddTask) 

371 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

372 """Load the previously-generated template coadd. 

373 

374 Returns 

375 ------- 

376 templateCoadd : `lsst.pipe.base.Struct` 

377 Results as a struct with attributes: 

378 

379 ``templateCoadd`` 

380 Coadded exposure (`lsst.afw.image.ExposureF`). 

381 """ 

382 templateCoadd = butlerQC.get(inputRefs.templateExposure) 

383 

384 return pipeBase.Struct(templateCoadd=templateCoadd) 

385 

386 def measureCoaddPsf(self, coaddExposure): 

387 """Detect sources on the coadd exposure and measure the final PSF. 

388 

389 Parameters 

390 ---------- 

391 coaddExposure : `lsst.afw.image.Exposure` 

392 The final coadded exposure. 

393 """ 

394 table = afwTable.SourceTable.make(self.schema) 

395 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False) 

396 coaddSources = detResults.sources 

397 self.measurePsfSources.run( 

398 measCat=coaddSources, 

399 exposure=coaddExposure 

400 ) 

401 # Measure the PSF on the stacked subfilter coadds if possible. 

402 # We should already have a decent estimate of the coadd PSF, however, 

403 # so in case of any errors simply log them as a warning and use the 

404 # default PSF. 

405 try: 

406 psfResults = self.measurePsf.run(coaddExposure, coaddSources) 

407 except Exception as e: 

408 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e) 

409 else: 

410 coaddExposure.setPsf(psfResults.psf) 

411 

412 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList): 

413 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 

414 

415 Sets the property ``bufferSize``. 

416 

417 Parameters 

418 ---------- 

419 templateCoadd : `lsst.afw.image.ExposureF` 

420 The initial coadd exposure before accounting for DCR. 

421 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

422 The data references to the input warped exposures. 

423 weightList : `list` of `float` 

424 The weight to give each input exposure in the coadd. 

425 Will be modified in place if ``doAirmassWeight`` is set. 

426 

427 Returns 

428 ------- 

429 dcrModels : `lsst.pipe.tasks.DcrModel` 

430 Best fit model of the true sky after correcting chromatic effects. 

431 

432 Raises 

433 ------ 

434 NotImplementedError 

435 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 

436 """ 

437 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

438 filterLabel = templateCoadd.getFilter() 

439 dcrShifts = [] 

440 airmassDict = {} 

441 angleDict = {} 

442 psfSizeDict = {} 

443 for visitNum, warpExpRef in enumerate(warpRefList): 

444 visitInfo = warpExpRef.get(component="visitInfo") 

445 psf = warpExpRef.get(component="psf") 

446 visit = warpExpRef.dataId["visit"] 

447 # Just need a rough estimate; average positions are fine 

448 psfAvgPos = psf.getAveragePosition() 

449 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm 

450 airmass = visitInfo.getBoresightAirmass() 

451 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees() 

452 airmassDict[visit] = airmass 

453 angleDict[visit] = parallacticAngle 

454 psfSizeDict[visit] = psfSize 

455 if self.config.doAirmassWeight: 

456 weightList[visitNum] *= airmass 

457 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(), 

458 self.config.effectiveWavelength, 

459 self.config.bandwidth, 

460 self.config.dcrNumSubfilters)))) 

461 self.log.info("Selected airmasses:\n%s", airmassDict) 

462 self.log.info("Selected parallactic angles:\n%s", angleDict) 

463 self.log.info("Selected PSF sizes:\n%s", psfSizeDict) 

464 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1) 

465 try: 

466 psf = self.selectCoaddPsf(templateCoadd, warpRefList) 

467 except Exception as e: 

468 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e) 

469 else: 

470 psf = templateCoadd.getPsf() 

471 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage, 

472 self.config.dcrNumSubfilters, 

473 effectiveWavelength=self.config.effectiveWavelength, 

474 bandwidth=self.config.bandwidth, 

475 wcs=templateCoadd.getWcs(), 

476 filterLabel=filterLabel, 

477 psf=psf) 

478 return dcrModels 

479 

480 @timeMethod 

481 def run(self, skyInfo, warpRefList, imageScalerList, weightList, 

482 supplementaryData=None): 

483 r"""Assemble the coadd. 

484 

485 Requires additional inputs Struct ``supplementaryData`` to contain a 

486 ``templateCoadd`` that serves as the model of the static sky. 

487 

488 Find artifacts and apply them to the warps' masks creating a list of 

489 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 

490 Then pass these alternative masks to the base class's assemble method. 

491 

492 Divide the ``templateCoadd`` evenly between each subfilter of a 

493 ``DcrModel`` as the starting best estimate of the true wavelength- 

494 dependent sky. Forward model the ``DcrModel`` using the known 

495 chromatic effects in each subfilter and calculate a convergence metric 

496 based on how well the modeled template matches the input warps. If 

497 the convergence has not yet reached the desired threshold, then shift 

498 and stack the residual images to build a new ``DcrModel``. Apply 

499 conditioning to prevent oscillating solutions between iterations or 

500 between subfilters. 

501 

502 Once the ``DcrModel`` reaches convergence or the maximum number of 

503 iterations has been reached, fill the metadata for each subfilter 

504 image and make them proper ``coaddExposure``\ s. 

505 

506 Parameters 

507 ---------- 

508 skyInfo : `lsst.pipe.base.Struct` 

509 Patch geometry information, from getSkyInfo 

510 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

511 The data references to the input warped exposures. 

512 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

513 The image scalars correct for the zero point of the exposures. 

514 weightList : `list` of `float` 

515 The weight to give each input exposure in the coadd 

516 supplementaryData : `lsst.pipe.base.Struct` 

517 Result struct returned by ``_makeSupplementaryData`` with attributes: 

518 

519 ``templateCoadd`` 

520 Coadded exposure (`lsst.afw.image.Exposure`). 

521 

522 Returns 

523 ------- 

524 result : `lsst.pipe.base.Struct` 

525 Results as a struct with attributes: 

526 

527 ``coaddExposure`` 

528 Coadded exposure (`lsst.afw.image.Exposure`). 

529 ``nImage`` 

530 Exposure count image (`lsst.afw.image.ImageU`). 

531 ``dcrCoadds`` 

532 `list` of coadded exposures for each subfilter. 

533 ``dcrNImages`` 

534 `list` of exposure count images for each subfilter. 

535 """ 

536 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters 

537 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3 

538 templateCoadd = supplementaryData.templateCoadd 

539 baseMask = templateCoadd.mask.clone() 

540 # The variance plane is for each subfilter 

541 # and should be proportionately lower than the full-band image 

542 baseVariance = templateCoadd.variance.clone() 

543 baseVariance /= self.config.dcrNumSubfilters 

544 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList) 

545 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask. 

546 templateCoadd.setMask(baseMask) 

547 badMaskPlanes = self.config.badMaskPlanes[:] 

548 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes`` 

549 # This is because pixels in observations that are significantly affect by DCR 

550 # are likely to have many pixels that are both "DETECTED" and "CLIPPED", 

551 # but those are necessary to constrain the DCR model. 

552 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes) 

553 

554 stats = self.prepareStats(mask=badPixelMask) 

555 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList) 

556 if self.config.doNImage: 

557 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList, 

558 spanSetMaskList, stats.ctrl) 

559 nImage = afwImage.ImageU(skyInfo.bbox) 

560 # Note that this nImage will be a factor of dcrNumSubfilters higher than 

561 # the nImage returned by assembleCoadd for most pixels. This is because each 

562 # subfilter may have a different nImage, and fractional values are not allowed. 

563 for dcrNImage in dcrNImages: 

564 nImage += dcrNImage 

565 else: 

566 dcrNImages = None 

567 

568 subregionSize = geom.Extent2I(*self.config.subregionSize) 

569 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) 

570 * ceil(skyInfo.bbox.getWidth()/subregionSize[0])) 

571 subIter = 0 

572 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

573 modelIter = 0 

574 subIter += 1 

575 self.log.info("Computing coadd over patch %s subregion %s of %s: %s", 

576 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox) 

577 dcrBBox = geom.Box2I(subBBox) 

578 dcrBBox.grow(self.bufferSize) 

579 dcrBBox.clip(dcrModels.bbox) 

580 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox) 

581 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList, 

582 imageScalerList, spanSetMaskList) 

583 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

584 warpRefList, weightList, stats.ctrl) 

585 self.log.info("Initial convergence : %s", convergenceMetric) 

586 convergenceList = [convergenceMetric] 

587 gainList = [] 

588 convergenceCheck = 1. 

589 refImage = templateCoadd.image 

590 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter): 

591 gain = self.calculateGain(convergenceList, gainList) 

592 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList, 

593 stats.ctrl, convergenceMetric, gain, 

594 modelWeights, refImage, dcrWeights) 

595 if self.config.useConvergence: 

596 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

597 warpRefList, weightList, stats.ctrl) 

598 if convergenceMetric == 0: 

599 self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is " 

600 "most likely due to there being no valid data in the region.", 

601 skyInfo.patchInfo.getIndex(), subIter) 

602 break 

603 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric 

604 if (convergenceCheck < 0) & (modelIter > minNumIter): 

605 self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum " 

606 "iterations or desired convergence improvement of %s." 

607 " Divergence: %s", 

608 skyInfo.patchInfo.getIndex(), subIter, 

609 self.config.convergenceThreshold, convergenceCheck) 

610 break 

611 convergenceList.append(convergenceMetric) 

612 if modelIter > maxNumIter: 

613 if self.config.useConvergence: 

614 self.log.warning("Coadd patch %s subregion %s reached maximum iterations " 

615 "before reaching desired convergence improvement of %s." 

616 " Final convergence improvement: %s", 

617 skyInfo.patchInfo.getIndex(), subIter, 

618 self.config.convergenceThreshold, convergenceCheck) 

619 break 

620 

621 if self.config.useConvergence: 

622 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)", 

623 modelIter, convergenceMetric, 100.*convergenceCheck, gain) 

624 modelIter += 1 

625 else: 

626 if self.config.useConvergence: 

627 self.log.info("Coadd patch %s subregion %s finished with " 

628 "convergence metric %s after %s iterations", 

629 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter) 

630 else: 

631 self.log.info("Coadd patch %s subregion %s finished after %s iterations", 

632 skyInfo.patchInfo.getIndex(), subIter, modelIter) 

633 if self.config.useConvergence and convergenceMetric > 0: 

634 self.log.info("Final convergence improvement was %.4f%% overall", 

635 100*(convergenceList[0] - convergenceMetric)/convergenceMetric) 

636 

637 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList, 

638 calibration=self.scaleZeroPoint.getPhotoCalib(), 

639 coaddInputs=templateCoadd.getInfo().getCoaddInputs(), 

640 mask=baseMask, 

641 variance=baseVariance) 

642 coaddExposure = self.stackCoadd(dcrCoadds) 

643 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

644 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages) 

645 

646 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl): 

647 """Calculate the number of exposures contributing to each subfilter. 

648 

649 Parameters 

650 ---------- 

651 dcrModels : `lsst.pipe.tasks.DcrModel` 

652 Best fit model of the true sky after correcting chromatic effects. 

653 bbox : `lsst.geom.box.Box2I` 

654 Bounding box of the patch to coadd. 

655 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

656 The data references to the input warped exposures. 

657 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None` 

658 Each element of the `dict` contains the new mask plane name 

659 (e.g. "CLIPPED and/or "NO_DATA") as the key, 

660 and the list of SpanSets to apply to the mask. 

661 statsCtrl : `lsst.afw.math.StatisticsControl` 

662 Statistics control object for coadd 

663 

664 Returns 

665 ------- 

666 dcrNImages : `list` of `lsst.afw.image.ImageU` 

667 List of exposure count images for each subfilter. 

668 dcrWeights : `list` of `lsst.afw.image.ImageF` 

669 Per-pixel weights for each subfilter. 

670 Equal to 1/(number of unmasked images contributing to each pixel). 

671 """ 

672 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

673 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

674 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList): 

675 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

676 visitInfo = exposure.getInfo().getVisitInfo() 

677 wcs = exposure.getInfo().getWcs() 

678 mask = exposure.mask 

679 if altMaskSpans is not None: 

680 self.applyAltMaskPlanes(mask, altMaskSpans) 

681 weightImage = np.zeros_like(exposure.image.array) 

682 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1. 

683 # The weights must be shifted in exactly the same way as the residuals, 

684 # because they will be used as the denominator in the weighted average of residuals. 

685 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, 

686 dcrModels.effectiveWavelength, dcrModels.bandwidth) 

687 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights): 

688 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype) 

689 dcrWeight.array += shiftedWeights 

690 # Exclude any pixels that don't have at least one exposure contributing in all subfilters 

691 weightsThreshold = 1. 

692 goodPix = dcrWeights[0].array > weightsThreshold 

693 for weights in dcrWeights[1:]: 

694 goodPix = (weights.array > weightsThreshold) & goodPix 

695 for subfilter in range(self.config.dcrNumSubfilters): 

696 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix] 

697 dcrWeights[subfilter].array[~goodPix] = 0. 

698 dcrNImages[subfilter].array[~goodPix] = 0 

699 return (dcrNImages, dcrWeights) 

700 

701 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, 

702 statsCtrl, convergenceMetric, 

703 gain, modelWeights, refImage, dcrWeights): 

704 """Assemble the DCR coadd for a sub-region. 

705 

706 Build a DCR-matched template for each input exposure, then shift the 

707 residuals according to the DCR in each subfilter. 

708 Stack the shifted residuals and apply them as a correction to the 

709 solution from the previous iteration. 

710 Restrict the new model solutions from varying by more than a factor of 

711 `modelClampFactor` from the last solution, and additionally restrict the 

712 individual subfilter models from varying by more than a factor of 

713 `frequencyClampFactor` from their average. 

714 Finally, mitigate potentially oscillating solutions by averaging the new 

715 solution with the solution from the previous iteration, weighted by 

716 their convergence metric. 

717 

718 Parameters 

719 ---------- 

720 dcrModels : `lsst.pipe.tasks.DcrModel` 

721 Best fit model of the true sky after correcting chromatic effects. 

722 subExposures : `dict` of `lsst.afw.image.ExposureF` 

723 The pre-loaded exposures for the current subregion. 

724 bbox : `lsst.geom.box.Box2I` 

725 Bounding box of the subregion to coadd. 

726 dcrBBox : `lsst.geom.box.Box2I` 

727 Sub-region of the coadd which includes a buffer to allow for DCR. 

728 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

729 The data references to the input warped exposures. 

730 statsCtrl : `lsst.afw.math.StatisticsControl` 

731 Statistics control object for coadd. 

732 convergenceMetric : `float` 

733 Quality of fit metric for the matched templates of the input images. 

734 gain : `float`, optional 

735 Relative weight to give the new solution when updating the model. 

736 modelWeights : `numpy.ndarray` or `float` 

737 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

738 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

739 refImage : `lsst.afw.image.Image` 

740 A reference image used to supply the default pixel values. 

741 dcrWeights : `list` of `lsst.afw.image.Image` 

742 Per-pixel weights for each subfilter. 

743 Equal to 1/(number of unmasked images contributing to each pixel). 

744 """ 

745 residualGeneratorList = [] 

746 

747 for warpExpRef in warpRefList: 

748 visit = warpExpRef.dataId["visit"] 

749 exposure = subExposures[visit] 

750 visitInfo = exposure.getInfo().getVisitInfo() 

751 wcs = exposure.getInfo().getWcs() 

752 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

753 bbox=exposure.getBBox(), 

754 order=self.config.imageInterpOrder, 

755 splitSubfilters=self.config.splitSubfilters, 

756 splitThreshold=self.config.splitThreshold, 

757 amplifyModel=self.config.accelerateModel) 

758 residual = exposure.image.array - templateImage.array 

759 # Note that the variance plane here is used to store weights, not the actual variance 

760 residual *= exposure.variance.array 

761 # The residuals are stored as a list of generators. 

762 # This allows the residual for a given subfilter and exposure to be created 

763 # on the fly, instead of needing to store them all in memory. 

764 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, 

765 dcrModels.effectiveWavelength, 

766 dcrModels.bandwidth)) 

767 

768 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

769 gain=gain, 

770 modelWeights=modelWeights, 

771 refImage=refImage, 

772 dcrWeights=dcrWeights) 

773 dcrModels.assign(dcrSubModelOut, bbox) 

774 

775 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth): 

776 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 

777 

778 Parameters 

779 ---------- 

780 residual : `numpy.ndarray` 

781 The residual masked image for one exposure, 

782 after subtracting the matched template. 

783 visitInfo : `lsst.afw.image.VisitInfo` 

784 Metadata for the exposure. 

785 wcs : `lsst.afw.geom.SkyWcs` 

786 Coordinate system definition (wcs) for the exposure. 

787 

788 Yields 

789 ------ 

790 residualImage : `numpy.ndarray` 

791 The residual image for the next subfilter, shifted for DCR. 

792 """ 

793 if self.config.imageInterpOrder > 1: 

794 # Pre-calculate the spline-filtered residual image, so that step can be 

795 # skipped in the shift calculation in `applyDcr`. 

796 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder) 

797 else: 

798 # No need to prefilter if order=1 (it will also raise an error) 

799 filteredResidual = residual 

800 # Note that `splitSubfilters` is always turned off in the reverse direction. 

801 # This option introduces additional blurring if applied to the residuals. 

802 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters, 

803 splitSubfilters=False) 

804 for dcr in dcrShift: 

805 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False, 

806 doPrefilter=False, order=self.config.imageInterpOrder) 

807 

808 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

809 gain, modelWeights, refImage, dcrWeights): 

810 """Calculate a new DcrModel from a set of image residuals. 

811 

812 Parameters 

813 ---------- 

814 dcrModels : `lsst.pipe.tasks.DcrModel` 

815 Current model of the true sky after correcting chromatic effects. 

816 residualGeneratorList : `generator` of `numpy.ndarray` 

817 The residual image for the next subfilter, shifted for DCR. 

818 dcrBBox : `lsst.geom.box.Box2I` 

819 Sub-region of the coadd which includes a buffer to allow for DCR. 

820 statsCtrl : `lsst.afw.math.StatisticsControl` 

821 Statistics control object for coadd. 

822 gain : `float` 

823 Relative weight to give the new solution when updating the model. 

824 modelWeights : `numpy.ndarray` or `float` 

825 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

826 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

827 refImage : `lsst.afw.image.Image` 

828 A reference image used to supply the default pixel values. 

829 dcrWeights : `list` of `lsst.afw.image.Image` 

830 Per-pixel weights for each subfilter. 

831 Equal to 1/(number of unmasked images contributing to each pixel). 

832 

833 Returns 

834 ------- 

835 dcrModel : `lsst.pipe.tasks.DcrModel` 

836 New model of the true sky after correcting chromatic effects. 

837 """ 

838 newModelImages = [] 

839 for subfilter, model in enumerate(dcrModels): 

840 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList] 

841 residual = np.sum(residualsList, axis=0) 

842 residual *= dcrWeights[subfilter][dcrBBox].array 

843 # `MaskedImage`s only support in-place addition, so rename for readability 

844 newModel = model[dcrBBox].clone() 

845 newModel.array += residual 

846 # Catch any invalid values 

847 badPixels = ~np.isfinite(newModel.array) 

848 newModel.array[badPixels] = model[dcrBBox].array[badPixels] 

849 if self.config.regularizeModelIterations > 0: 

850 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox, 

851 self.config.regularizeModelIterations, 

852 self.config.regularizationWidth) 

853 newModelImages.append(newModel) 

854 if self.config.regularizeModelFrequency > 0: 

855 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl, 

856 self.config.regularizeModelFrequency, 

857 self.config.regularizationWidth) 

858 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain) 

859 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights) 

860 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength, 

861 dcrModels.bandwidth, dcrModels.psf, 

862 dcrModels.mask, dcrModels.variance) 

863 

864 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl): 

865 """Calculate a quality of fit metric for the matched templates. 

866 

867 Parameters 

868 ---------- 

869 dcrModels : `lsst.pipe.tasks.DcrModel` 

870 Best fit model of the true sky after correcting chromatic effects. 

871 subExposures : `dict` of `lsst.afw.image.ExposureF` 

872 The pre-loaded exposures for the current subregion. 

873 bbox : `lsst.geom.box.Box2I` 

874 Sub-region to coadd. 

875 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

876 The data references to the input warped exposures. 

877 weightList : `list` of `float` 

878 The weight to give each input exposure in the coadd. 

879 statsCtrl : `lsst.afw.math.StatisticsControl` 

880 Statistics control object for coadd. 

881 

882 Returns 

883 ------- 

884 convergenceMetric : `float` 

885 Quality of fit metric for all input exposures, within the sub-region. 

886 """ 

887 significanceImage = np.abs(dcrModels.getReferenceImage(bbox)) 

888 nSigma = 3. 

889 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl, 

890 bufferSize=self.bufferSize) 

891 if np.max(significanceImage) == 0: 

892 significanceImage += 1. 

893 weight = 0 

894 metric = 0. 

895 metricList = {} 

896 for warpExpRef, expWeight in zip(warpRefList, weightList): 

897 visit = warpExpRef.dataId["visit"] 

898 exposure = subExposures[visit][bbox] 

899 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl) 

900 metric += singleMetric 

901 metricList[visit] = singleMetric 

902 weight += 1. 

903 self.log.info("Individual metrics:\n%s", metricList) 

904 return 1.0 if weight == 0.0 else metric/weight 

905 

906 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl): 

907 """Calculate a quality of fit metric for a single matched template. 

908 

909 Parameters 

910 ---------- 

911 dcrModels : `lsst.pipe.tasks.DcrModel` 

912 Best fit model of the true sky after correcting chromatic effects. 

913 exposure : `lsst.afw.image.ExposureF` 

914 The input warped exposure to evaluate. 

915 significanceImage : `numpy.ndarray` 

916 Array of weights for each pixel corresponding to its significance 

917 for the convergence calculation. 

918 statsCtrl : `lsst.afw.math.StatisticsControl` 

919 Statistics control object for coadd. 

920 

921 Returns 

922 ------- 

923 convergenceMetric : `float` 

924 Quality of fit metric for one exposure, within the sub-region. 

925 """ 

926 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

927 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

928 bbox=exposure.getBBox(), 

929 order=self.config.imageInterpOrder, 

930 splitSubfilters=self.config.splitSubfilters, 

931 splitThreshold=self.config.splitThreshold, 

932 amplifyModel=self.config.accelerateModel) 

933 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage 

934 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2. 

935 

936 finitePixels = np.isfinite(diffVals) 

937 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0 

938 convergeMaskPixels = exposure.mask.array & convergeMask > 0 

939 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels 

940 if np.sum(usePixels) == 0: 

941 metric = 0. 

942 else: 

943 diffUse = diffVals[usePixels] 

944 refUse = refVals[usePixels] 

945 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse)) 

946 return metric 

947 

948 def stackCoadd(self, dcrCoadds): 

949 """Add a list of sub-band coadds together. 

950 

951 Parameters 

952 ---------- 

953 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

954 A list of coadd exposures, each exposure containing 

955 the model for one subfilter. 

956 

957 Returns 

958 ------- 

959 coaddExposure : `lsst.afw.image.ExposureF` 

960 A single coadd exposure that is the sum of the sub-bands. 

961 """ 

962 coaddExposure = dcrCoadds[0].clone() 

963 for coadd in dcrCoadds[1:]: 

964 coaddExposure.maskedImage += coadd.maskedImage 

965 return coaddExposure 

966 

967 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, 

968 mask=None, variance=None): 

969 """Create a list of coadd exposures from a list of masked images. 

970 

971 Parameters 

972 ---------- 

973 dcrModels : `lsst.pipe.tasks.DcrModel` 

974 Best fit model of the true sky after correcting chromatic effects. 

975 skyInfo : `lsst.pipe.base.Struct` 

976 Patch geometry information, from getSkyInfo. 

977 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

978 The data references to the input warped exposures. 

979 weightList : `list` of `float` 

980 The weight to give each input exposure in the coadd. 

981 calibration : `lsst.afw.Image.PhotoCalib`, optional 

982 Scale factor to set the photometric calibration of an exposure. 

983 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 

984 A record of the observations that are included in the coadd. 

985 mask : `lsst.afw.image.Mask`, optional 

986 Optional mask to override the values in the final coadd. 

987 variance : `lsst.afw.image.Image`, optional 

988 Optional variance plane to override the values in the final coadd. 

989 

990 Returns 

991 ------- 

992 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

993 A list of coadd exposures, each exposure containing 

994 the model for one subfilter. 

995 """ 

996 dcrCoadds = [] 

997 refModel = dcrModels.getReferenceImage() 

998 for model in dcrModels: 

999 if self.config.accelerateModel > 1: 

1000 model.array = (model.array - refModel)*self.config.accelerateModel + refModel 

1001 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

1002 if calibration is not None: 

1003 coaddExposure.setPhotoCalib(calibration) 

1004 if coaddInputs is not None: 

1005 coaddExposure.getInfo().setCoaddInputs(coaddInputs) 

1006 # Set the metadata for the coadd, including PSF and aperture corrections. 

1007 self.assembleMetadata(coaddExposure, warpRefList, weightList) 

1008 # Overwrite the PSF 

1009 coaddExposure.setPsf(dcrModels.psf) 

1010 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox]) 

1011 maskedImage = afwImage.MaskedImageF(dcrModels.bbox) 

1012 maskedImage.image = model 

1013 maskedImage.mask = dcrModels.mask 

1014 maskedImage.variance = dcrModels.variance 

1015 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox]) 

1016 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

1017 if mask is not None: 

1018 coaddExposure.setMask(mask) 

1019 if variance is not None: 

1020 coaddExposure.setVariance(variance) 

1021 dcrCoadds.append(coaddExposure) 

1022 return dcrCoadds 

1023 

1024 def calculateGain(self, convergenceList, gainList): 

1025 """Calculate the gain to use for the current iteration. 

1026 

1027 After calculating a new DcrModel, each value is averaged with the 

1028 value in the corresponding pixel from the previous iteration. This 

1029 reduces oscillating solutions that iterative techniques are plagued by, 

1030 and speeds convergence. By far the biggest changes to the model 

1031 happen in the first couple iterations, so we can also use a more 

1032 aggressive gain later when the model is changing slowly. 

1033 

1034 Parameters 

1035 ---------- 

1036 convergenceList : `list` of `float` 

1037 The quality of fit metric from each previous iteration. 

1038 gainList : `list` of `float` 

1039 The gains used in each previous iteration: appended with the new 

1040 gain value. 

1041 Gains are numbers between ``self.config.baseGain`` and 1. 

1042 

1043 Returns 

1044 ------- 

1045 gain : `float` 

1046 Relative weight to give the new solution when updating the model. 

1047 A value of 1.0 gives equal weight to both solutions. 

1048 

1049 Raises 

1050 ------ 

1051 ValueError 

1052 If ``len(convergenceList) != len(gainList)+1``. 

1053 """ 

1054 nIter = len(convergenceList) 

1055 if nIter != len(gainList) + 1: 

1056 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)." 

1057 % (len(convergenceList), len(gainList))) 

1058 

1059 if self.config.baseGain is None: 

1060 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters 

1061 # The more subfilters being modeled, the lower the gain should be. 

1062 baseGain = 1./(self.config.dcrNumSubfilters - 1) 

1063 else: 

1064 baseGain = self.config.baseGain 

1065 

1066 if self.config.useProgressiveGain and nIter > 2: 

1067 # To calculate the best gain to use, compare the past gains that have been used 

1068 # with the resulting convergences to estimate the best gain to use. 

1069 # Algorithmically, this is a Kalman filter. 

1070 # If forward modeling proceeds perfectly, the convergence metric should 

1071 # asymptotically approach a final value. 

1072 # We can estimate that value from the measured changes in convergence 

1073 # weighted by the gains used in each previous iteration. 

1074 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i] 

1075 for i in range(nIter - 1)] 

1076 # The convergence metric is strictly positive, so if the estimated final convergence is 

1077 # less than zero, force it to zero. 

1078 estFinalConv = np.array(estFinalConv) 

1079 estFinalConv[estFinalConv < 0] = 0 

1080 # Because the estimate may slowly change over time, only use the most recent measurements. 

1081 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):]) 

1082 lastGain = gainList[-1] 

1083 lastConv = convergenceList[-2] 

1084 newConv = convergenceList[-1] 

1085 # The predicted convergence is the value we would get if the new model calculated 

1086 # in the previous iteration was perfect. Recall that the updated model that is 

1087 # actually used is the gain-weighted average of the new and old model, 

1088 # so the convergence would be similarly weighted. 

1089 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain) 

1090 # If the measured and predicted convergence are very close, that indicates 

1091 # that our forward model is accurate and we can use a more aggressive gain 

1092 # If the measured convergence is significantly worse (or better!) than predicted, 

1093 # that indicates that the model is not converging as expected and 

1094 # we should use a more conservative gain. 

1095 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain)) 

1096 newGain = 1 - abs(delta) 

1097 # Average the gains to prevent oscillating solutions. 

1098 newGain = (newGain + lastGain)/2. 

1099 gain = max(baseGain, newGain) 

1100 else: 

1101 gain = baseGain 

1102 gainList.append(gain) 

1103 return gain 

1104 

1105 def calculateModelWeights(self, dcrModels, dcrBBox): 

1106 """Build an array that smoothly tapers to 0 away from detected sources. 

1107 

1108 Parameters 

1109 ---------- 

1110 dcrModels : `lsst.pipe.tasks.DcrModel` 

1111 Best fit model of the true sky after correcting chromatic effects. 

1112 dcrBBox : `lsst.geom.box.Box2I` 

1113 Sub-region of the coadd which includes a buffer to allow for DCR. 

1114 

1115 Returns 

1116 ------- 

1117 weights : `numpy.ndarray` or `float` 

1118 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1119 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1120 

1121 Raises 

1122 ------ 

1123 ValueError 

1124 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 

1125 """ 

1126 if not self.config.useModelWeights: 

1127 return 1.0 

1128 if self.config.modelWeightsWidth < 0: 

1129 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set") 

1130 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1131 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0 

1132 weights = np.zeros_like(dcrModels[0][dcrBBox].array) 

1133 weights[convergeMaskPixels] = 1. 

1134 weights = ndimage.gaussian_filter(weights, self.config.modelWeightsWidth) 

1135 weights /= np.max(weights) 

1136 return weights 

1137 

1138 def applyModelWeights(self, modelImages, refImage, modelWeights): 

1139 """Smoothly replace model pixel values with those from a 

1140 reference at locations away from detected sources. 

1141 

1142 Parameters 

1143 ---------- 

1144 modelImages : `list` of `lsst.afw.image.Image` 

1145 The new DCR model images from the current iteration. 

1146 The values will be modified in place. 

1147 refImage : `lsst.afw.image.MaskedImage` 

1148 A reference image used to supply the default pixel values. 

1149 modelWeights : `numpy.ndarray` or `float` 

1150 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1151 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1152 """ 

1153 if self.config.useModelWeights: 

1154 for model in modelImages: 

1155 model.array *= modelWeights 

1156 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters 

1157 

1158 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList): 

1159 """Pre-load sub-regions of a list of exposures. 

1160 

1161 Parameters 

1162 ---------- 

1163 bbox : `lsst.geom.box.Box2I` 

1164 Sub-region to coadd. 

1165 statsCtrl : `lsst.afw.math.StatisticsControl` 

1166 Statistics control object for coadd. 

1167 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1168 The data references to the input warped exposures. 

1169 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

1170 The image scalars correct for the zero point of the exposures. 

1171 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None` 

1172 Each element is dict with keys = mask plane name to add the spans to. 

1173 

1174 Returns 

1175 ------- 

1176 subExposures : `dict` 

1177 The `dict` keys are the visit IDs, 

1178 and the values are `lsst.afw.image.ExposureF` 

1179 The pre-loaded exposures for the current subregion. 

1180 The variance plane contains weights, and not the variance 

1181 """ 

1182 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList) 

1183 subExposures = {} 

1184 for warpExpRef, imageScaler, altMaskSpans in zipIterables: 

1185 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

1186 visit = warpExpRef.dataId["visit"] 

1187 if altMaskSpans is not None: 

1188 self.applyAltMaskPlanes(exposure.mask, altMaskSpans) 

1189 imageScaler.scaleMaskedImage(exposure.maskedImage) 

1190 # Note that the variance plane here is used to store weights, not the actual variance 

1191 exposure.variance.array[:, :] = 0. 

1192 # Set the weight of unmasked pixels to 1. 

1193 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1. 

1194 # Set the image value of masked pixels to zero. 

1195 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual`` 

1196 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0. 

1197 subExposures[visit] = exposure 

1198 return subExposures 

1199 

1200 def selectCoaddPsf(self, templateCoadd, warpRefList): 

1201 """Compute the PSF of the coadd from the exposures with the best seeing. 

1202 

1203 Parameters 

1204 ---------- 

1205 templateCoadd : `lsst.afw.image.ExposureF` 

1206 The initial coadd exposure before accounting for DCR. 

1207 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1208 The data references to the input warped exposures. 

1209 

1210 Returns 

1211 ------- 

1212 psf : `lsst.meas.algorithms.CoaddPsf` 

1213 The average PSF of the input exposures with the best seeing. 

1214 """ 

1215 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

1216 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit 

1217 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef`` 

1218 ccds = templateCoadd.getInfo().getCoaddInputs().ccds 

1219 templatePsf = templateCoadd.getPsf() 

1220 # Just need a rough estimate; average positions are fine 

1221 templateAvgPos = templatePsf.getAveragePosition() 

1222 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()*sigma2fwhm 

1223 psfSizes = np.zeros(len(ccds)) 

1224 ccdVisits = np.array(ccds["visit"]) 

1225 for warpExpRef in warpRefList: 

1226 psf = warpExpRef.get(component="psf") 

1227 visit = warpExpRef.dataId["visit"] 

1228 psfAvgPos = psf.getAveragePosition() 

1229 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm 

1230 psfSizes[ccdVisits == visit] = psfSize 

1231 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd 

1232 # The selected PSFs are those that have a FWHM less than or equal to the smaller 

1233 # of the mean or median FWHM of the input exposures. 

1234 sizeThreshold = min(np.median(psfSizes), psfRefSize) 

1235 goodPsfs = psfSizes <= sizeThreshold 

1236 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), 

1237 self.config.coaddPsf.makeControl()) 

1238 return psf