Coverage for python/lsst/pipe/tasks/dcrAssembleCoadd.py: 14%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

460 statements  

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23from math import ceil 

24import numpy as np 

25from scipy import ndimage 

26import lsst.geom as geom 

27import lsst.afw.image as afwImage 

28import lsst.afw.table as afwTable 

29import lsst.coadd.utils as coaddUtils 

30from lsst.daf.butler import DeferredDatasetHandle 

31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel 

32import lsst.meas.algorithms as measAlg 

33from lsst.meas.base import SingleFrameMeasurementTask 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.utils as utils 

37from lsst.utils.timer import timeMethod 

38from .assembleCoadd import (AssembleCoaddConnections, 

39 AssembleCoaddTask, 

40 CompareWarpAssembleCoaddConfig, 

41 CompareWarpAssembleCoaddTask) 

42from .coaddBase import makeSkyInfo 

43from .measurePsf import MeasurePsfTask 

44 

45__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"] 

46 

47 

48class DcrAssembleCoaddConnections(AssembleCoaddConnections, 

49 dimensions=("tract", "patch", "band", "skymap"), 

50 defaultTemplates={"inputWarpName": "deep", 

51 "inputCoaddName": "deep", 

52 "outputCoaddName": "dcr", 

53 "warpType": "direct", 

54 "warpTypeSuffix": "", 

55 "fakesType": ""}): 

56 inputWarps = pipeBase.connectionTypes.Input( 

57 doc=("Input list of warps to be assembled i.e. stacked." 

58 "Note that this will often be different than the inputCoaddName." 

59 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

60 name="{inputWarpName}Coadd_{warpType}Warp", 

61 storageClass="ExposureF", 

62 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

63 deferLoad=True, 

64 multiple=True 

65 ) 

66 templateExposure = pipeBase.connectionTypes.Input( 

67 doc="Input coadded exposure, produced by previous call to AssembleCoadd", 

68 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}", 

69 storageClass="ExposureF", 

70 dimensions=("tract", "patch", "skymap", "band"), 

71 ) 

72 dcrCoadds = pipeBase.connectionTypes.Output( 

73 doc="Output coadded exposure, produced by stacking input warps", 

74 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

75 storageClass="ExposureF", 

76 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

77 multiple=True, 

78 ) 

79 dcrNImages = pipeBase.connectionTypes.Output( 

80 doc="Output image of number of input images per pixel", 

81 name="{outputCoaddName}Coadd_nImage", 

82 storageClass="ImageU", 

83 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

84 multiple=True, 

85 ) 

86 

87 def __init__(self, *, config=None): 

88 super().__init__(config=config) 

89 if not config.doWrite: 

90 self.outputs.remove("dcrCoadds") 

91 if not config.doNImage: 

92 self.outputs.remove("dcrNImages") 

93 # Remove outputs inherited from ``AssembleCoaddConnections`` that are not used 

94 self.outputs.remove("coaddExposure") 

95 self.outputs.remove("nImage") 

96 

97 

98class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, 

99 pipelineConnections=DcrAssembleCoaddConnections): 

100 dcrNumSubfilters = pexConfig.Field( 

101 dtype=int, 

102 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.", 

103 default=3, 

104 ) 

105 maxNumIter = pexConfig.Field( 

106 dtype=int, 

107 optional=True, 

108 doc="Maximum number of iterations of forward modeling.", 

109 default=None, 

110 ) 

111 minNumIter = pexConfig.Field( 

112 dtype=int, 

113 optional=True, 

114 doc="Minimum number of iterations of forward modeling.", 

115 default=None, 

116 ) 

117 convergenceThreshold = pexConfig.Field( 

118 dtype=float, 

119 doc="Target relative change in convergence between iterations of forward modeling.", 

120 default=0.001, 

121 ) 

122 useConvergence = pexConfig.Field( 

123 dtype=bool, 

124 doc="Use convergence test as a forward modeling end condition?" 

125 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations", 

126 default=True, 

127 ) 

128 baseGain = pexConfig.Field( 

129 dtype=float, 

130 optional=True, 

131 doc="Relative weight to give the new solution vs. the last solution when updating the model." 

132 "A value of 1.0 gives equal weight to both solutions." 

133 "Small values imply slower convergence of the solution, but can " 

134 "help prevent overshooting and failures in the fit." 

135 "If ``baseGain`` is None, a conservative gain " 

136 "will be calculated from the number of subfilters. ", 

137 default=None, 

138 ) 

139 useProgressiveGain = pexConfig.Field( 

140 dtype=bool, 

141 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 

142 "When calculating the next gain, we use up to 5 previous gains and convergence values." 

143 "Can be set to False to force the model to change at the rate of ``baseGain``. ", 

144 default=True, 

145 ) 

146 doAirmassWeight = pexConfig.Field( 

147 dtype=bool, 

148 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.", 

149 default=False, 

150 ) 

151 modelWeightsWidth = pexConfig.Field( 

152 dtype=float, 

153 doc="Width of the region around detected sources to include in the DcrModel.", 

154 default=3, 

155 ) 

156 useModelWeights = pexConfig.Field( 

157 dtype=bool, 

158 doc="Width of the region around detected sources to include in the DcrModel.", 

159 default=True, 

160 ) 

161 splitSubfilters = pexConfig.Field( 

162 dtype=bool, 

163 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter." 

164 "Instead of at the midpoint", 

165 default=True, 

166 ) 

167 splitThreshold = pexConfig.Field( 

168 dtype=float, 

169 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 

170 "Set to 0 to always split the subfilters.", 

171 default=0.1, 

172 ) 

173 regularizeModelIterations = pexConfig.Field( 

174 dtype=float, 

175 doc="Maximum relative change of the model allowed between iterations." 

176 "Set to zero to disable.", 

177 default=2., 

178 ) 

179 regularizeModelFrequency = pexConfig.Field( 

180 dtype=float, 

181 doc="Maximum relative change of the model allowed between subfilters." 

182 "Set to zero to disable.", 

183 default=4., 

184 ) 

185 convergenceMaskPlanes = pexConfig.ListField( 

186 dtype=str, 

187 default=["DETECTED"], 

188 doc="Mask planes to use to calculate convergence." 

189 ) 

190 regularizationWidth = pexConfig.Field( 

191 dtype=int, 

192 default=2, 

193 doc="Minimum radius of a region to include in regularization, in pixels." 

194 ) 

195 imageInterpOrder = pexConfig.Field( 

196 dtype=int, 

197 doc="The order of the spline interpolation used to shift the image plane.", 

198 default=3, 

199 ) 

200 accelerateModel = pexConfig.Field( 

201 dtype=float, 

202 doc="Factor to amplify the differences between model planes by to speed convergence.", 

203 default=3, 

204 ) 

205 doCalculatePsf = pexConfig.Field( 

206 dtype=bool, 

207 doc="Set to detect stars and recalculate the PSF from the final coadd." 

208 "Otherwise the PSF is estimated from a selection of the best input exposures", 

209 default=False, 

210 ) 

211 detectPsfSources = pexConfig.ConfigurableField( 

212 target=measAlg.SourceDetectionTask, 

213 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.", 

214 ) 

215 measurePsfSources = pexConfig.ConfigurableField( 

216 target=SingleFrameMeasurementTask, 

217 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 

218 ) 

219 measurePsf = pexConfig.ConfigurableField( 

220 target=MeasurePsfTask, 

221 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.", 

222 ) 

223 effectiveWavelength = pexConfig.Field( 

224 doc="Effective wavelength of the filter, in nm." 

225 "Required if transmission curves aren't used." 

226 "Support for using transmission curves is to be added in DM-13668.", 

227 dtype=float, 

228 ) 

229 bandwidth = pexConfig.Field( 

230 doc="Bandwidth of the physical filter, in nm." 

231 "Required if transmission curves aren't used." 

232 "Support for using transmission curves is to be added in DM-13668.", 

233 dtype=float, 

234 ) 

235 

236 def setDefaults(self): 

237 CompareWarpAssembleCoaddConfig.setDefaults(self) 

238 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask) 

239 self.doNImage = True 

240 self.assembleStaticSkyModel.warpType = self.warpType 

241 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time 

242 self.assembleStaticSkyModel.doNImage = False 

243 self.assembleStaticSkyModel.doWrite = False 

244 self.detectPsfSources.returnOriginalFootprints = False 

245 self.detectPsfSources.thresholdPolarity = "positive" 

246 # Only use bright sources for PSF measurement 

247 self.detectPsfSources.thresholdValue = 50 

248 self.detectPsfSources.nSigmaToGrow = 2 

249 # A valid star for PSF measurement should at least fill 5x5 pixels 

250 self.detectPsfSources.minPixels = 25 

251 # Use the variance plane to calculate signal to noise 

252 self.detectPsfSources.thresholdType = "pixel_stdev" 

253 # The signal to noise limit is good enough, while the flux limit is set 

254 # in dimensionless units and may not be appropriate for all data sets. 

255 self.measurePsf.starSelector["objectSize"].doFluxLimit = False 

256 

257 

258class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask): 

259 """Assemble DCR coadded images from a set of warps. 

260 

261 Attributes 

262 ---------- 

263 bufferSize : `int` 

264 The number of pixels to grow each subregion by to allow for DCR. 

265 

266 Notes 

267 ----- 

268 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 

269 Warps (also called coadded temporary exposures), including the effects of 

270 Differential Chromatic Refraction (DCR). 

271 For full details of the mathematics and algorithm, please see 

272 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 

273 

274 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 

275 each subfilter used in the iterative calculation. 

276 It begins by dividing the bandpass-defining filter into N equal bandwidth 

277 "subfilters", and divides the flux in each pixel from an initial coadd 

278 equally into each as a "dcrModel". Because the airmass and parallactic 

279 angle of each individual exposure is known, we can calculate the shift 

280 relative to the center of the band in each subfilter due to DCR. For each 

281 exposure we apply this shift as a linear transformation to the dcrModels 

282 and stack the results to produce a DCR-matched exposure. The matched 

283 exposures are subtracted from the input exposures to produce a set of 

284 residual images, and these residuals are reverse shifted for each 

285 exposures' subfilters and stacked. The shifted and stacked residuals are 

286 added to the dcrModels to produce a new estimate of the flux in each pixel 

287 within each subfilter. The dcrModels are solved for iteratively, which 

288 continues until the solution from a new iteration improves by less than 

289 a set percentage, or a maximum number of iterations is reached. 

290 Two forms of regularization are employed to reduce unphysical results. 

291 First, the new solution is averaged with the solution from the previous 

292 iteration, which mitigates oscillating solutions where the model 

293 overshoots with alternating very high and low values. 

294 Second, a common degeneracy when the data have a limited range of airmass or 

295 parallactic angle values is for one subfilter to be fit with very low or 

296 negative values, while another subfilter is fit with very high values. This 

297 typically appears in the form of holes next to sources in one subfilter, 

298 and corresponding extended wings in another. Because each subfilter has 

299 a narrow bandwidth we assume that physical sources that are above the noise 

300 level will not vary in flux by more than a factor of `frequencyClampFactor` 

301 between subfilters, and pixels that have flux deviations larger than that 

302 factor will have the excess flux distributed evenly among all subfilters. 

303 If `splitSubfilters` is set, then each subfilter will be further sub- 

304 divided during the forward modeling step (only). This approximates using 

305 a higher number of subfilters that may be necessary for high airmass 

306 observations, but does not increase the number of free parameters in the 

307 fit. This is needed when there are high airmass observations which would 

308 otherwise have significant DCR even within a subfilter. Because calculating 

309 the shifted images takes most of the time, splitting the subfilters is 

310 turned off by way of the `splitThreshold` option for low-airmass 

311 observations that do not suffer from DCR within a subfilter. 

312 """ 

313 

314 ConfigClass = DcrAssembleCoaddConfig 

315 _DefaultName = "dcrAssembleCoadd" 

316 

317 def __init__(self, *args, **kwargs): 

318 super().__init__(*args, **kwargs) 

319 if self.config.doCalculatePsf: 

320 self.schema = afwTable.SourceTable.makeMinimalSchema() 

321 self.makeSubtask("detectPsfSources", schema=self.schema) 

322 self.makeSubtask("measurePsfSources", schema=self.schema) 

323 self.makeSubtask("measurePsf", schema=self.schema) 

324 

325 @utils.inheritDoc(pipeBase.PipelineTask) 

326 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

327 # Docstring to be formatted with info from PipelineTask.runQuantum 

328 """ 

329 Notes 

330 ----- 

331 Assemble a coadd from a set of Warps. 

332 

333 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

334 Analogous to `runDataRef`, it prepares all the data products to be 

335 passed to `run`, and processes the results before returning a struct 

336 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

337 Therefore, its inputs are accessed subregion by subregion 

338 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

339 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

340 correspond to an update in `runDataRef` while both entry points 

341 are used. 

342 """ 

343 inputData = butlerQC.get(inputRefs) 

344 

345 # Construct skyInfo expected by run 

346 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

347 skyMap = inputData["skyMap"] 

348 outputDataId = butlerQC.quantum.dataId 

349 

350 inputData['skyInfo'] = makeSkyInfo(skyMap, 

351 tractId=outputDataId['tract'], 

352 patchId=outputDataId['patch']) 

353 

354 # Construct list of input Deferred Datasets 

355 # These quack a bit like like Gen2 DataRefs 

356 warpRefList = inputData['inputWarps'] 

357 # Perform same middle steps as `runDataRef` does 

358 inputs = self.prepareInputs(warpRefList) 

359 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

360 self.getTempExpDatasetName(self.warpType)) 

361 if len(inputs.tempExpRefList) == 0: 

362 self.log.warning("No coadd temporary exposures found") 

363 return 

364 

365 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

366 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

367 inputs.weightList, supplementaryData=supplementaryData) 

368 

369 inputData.setdefault('brightObjectMask', None) 

370 for subfilter in range(self.config.dcrNumSubfilters): 

371 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

372 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf()) 

373 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId) 

374 

375 if self.config.doWrite: 

376 butlerQC.put(retStruct, outputRefs) 

377 return retStruct 

378 

379 @timeMethod 

380 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

381 """Assemble a coadd from a set of warps. 

382 

383 Coadd a set of Warps. Compute weights to be applied to each Warp and 

384 find scalings to match the photometric zeropoint to a reference Warp. 

385 Assemble the Warps using run method. 

386 Forward model chromatic effects across multiple subfilters, 

387 and subtract from the input Warps to build sets of residuals. 

388 Use the residuals to construct a new ``DcrModel`` for each subfilter, 

389 and iterate until the model converges. 

390 Interpolate over NaNs and optionally write the coadd to disk. 

391 Return the coadded exposure. 

392 

393 Parameters 

394 ---------- 

395 dataRef : `lsst.daf.persistence.ButlerDataRef` 

396 Data reference defining the patch for coaddition and the 

397 reference Warp 

398 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 

399 List of data references to warps. Data to be coadded will be 

400 selected from this list based on overlap with the patch defined by 

401 the data reference. 

402 

403 Returns 

404 ------- 

405 results : `lsst.pipe.base.Struct` 

406 The Struct contains the following fields: 

407 

408 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

409 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

410 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

411 - ``dcrNImages``: `list` of exposure count images for each subfilter 

412 """ 

413 if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList): 

414 raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList") 

415 

416 skyInfo = self.getSkyInfo(dataRef) 

417 if warpRefList is None: 

418 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

419 if len(calExpRefList) == 0: 

420 self.log.warning("No exposures to coadd") 

421 return 

422 self.log.info("Coadding %d exposures", len(calExpRefList)) 

423 

424 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

425 

426 inputData = self.prepareInputs(warpRefList) 

427 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

428 self.getTempExpDatasetName(self.warpType)) 

429 if len(inputData.tempExpRefList) == 0: 

430 self.log.warning("No coadd temporary exposures found") 

431 return 

432 

433 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

434 

435 results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

436 inputData.weightList, supplementaryData=supplementaryData) 

437 if results is None: 

438 self.log.warning("Could not construct DcrModel for patch %s: no data to coadd.", 

439 skyInfo.patchInfo.getIndex()) 

440 return 

441 

442 if self.config.doCalculatePsf: 

443 self.measureCoaddPsf(results.coaddExposure) 

444 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

445 for subfilter in range(self.config.dcrNumSubfilters): 

446 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

447 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf()) 

448 self.processResults(results.dcrCoadds[subfilter], 

449 brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

450 if self.config.doWrite: 

451 self.log.info("Persisting dcrCoadd") 

452 dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter, 

453 numSubfilters=self.config.dcrNumSubfilters) 

454 if self.config.doNImage and results.dcrNImages is not None: 

455 dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter, 

456 numSubfilters=self.config.dcrNumSubfilters) 

457 

458 return results 

459 

460 @utils.inheritDoc(AssembleCoaddTask) 

461 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

462 """Load the previously-generated template coadd. 

463 

464 This can be removed entirely once we no longer support the Gen 2 butler. 

465 

466 Returns 

467 ------- 

468 templateCoadd : `lsst.pipe.base.Struct` 

469 Result struct with components: 

470 

471 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`) 

472 """ 

473 templateCoadd = butlerQC.get(inputRefs.templateExposure) 

474 

475 return pipeBase.Struct(templateCoadd=templateCoadd) 

476 

477 def measureCoaddPsf(self, coaddExposure): 

478 """Detect sources on the coadd exposure and measure the final PSF. 

479 

480 Parameters 

481 ---------- 

482 coaddExposure : `lsst.afw.image.Exposure` 

483 The final coadded exposure. 

484 """ 

485 table = afwTable.SourceTable.make(self.schema) 

486 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False) 

487 coaddSources = detResults.sources 

488 self.measurePsfSources.run( 

489 measCat=coaddSources, 

490 exposure=coaddExposure 

491 ) 

492 # Measure the PSF on the stacked subfilter coadds if possible. 

493 # We should already have a decent estimate of the coadd PSF, however, 

494 # so in case of any errors simply log them as a warning and use the 

495 # default PSF. 

496 try: 

497 psfResults = self.measurePsf.run(coaddExposure, coaddSources) 

498 except Exception as e: 

499 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e) 

500 else: 

501 coaddExposure.setPsf(psfResults.psf) 

502 

503 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList): 

504 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 

505 

506 Sets the property ``bufferSize``. 

507 

508 Parameters 

509 ---------- 

510 templateCoadd : `lsst.afw.image.ExposureF` 

511 The initial coadd exposure before accounting for DCR. 

512 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

513 `lsst.daf.persistence.ButlerDataRef` 

514 The data references to the input warped exposures. 

515 weightList : `list` of `float` 

516 The weight to give each input exposure in the coadd 

517 Will be modified in place if ``doAirmassWeight`` is set. 

518 

519 Returns 

520 ------- 

521 dcrModels : `lsst.pipe.tasks.DcrModel` 

522 Best fit model of the true sky after correcting chromatic effects. 

523 

524 Raises 

525 ------ 

526 NotImplementedError 

527 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 

528 """ 

529 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

530 filterLabel = templateCoadd.getFilterLabel() 

531 tempExpName = self.getTempExpDatasetName(self.warpType) 

532 dcrShifts = [] 

533 airmassDict = {} 

534 angleDict = {} 

535 psfSizeDict = {} 

536 for visitNum, warpExpRef in enumerate(warpRefList): 

537 if isinstance(warpExpRef, DeferredDatasetHandle): 

538 # Gen 3 API 

539 visitInfo = warpExpRef.get(component="visitInfo") 

540 psf = warpExpRef.get(component="psf") 

541 else: 

542 # Gen 2 API. Delete this when Gen 2 retired 

543 visitInfo = warpExpRef.get(tempExpName + "_visitInfo") 

544 psf = warpExpRef.get(tempExpName).getPsf() 

545 visit = warpExpRef.dataId["visit"] 

546 # Just need a rough estimate; average positions are fine 

547 psfAvgPos = psf.getAveragePosition() 

548 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm 

549 airmass = visitInfo.getBoresightAirmass() 

550 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees() 

551 airmassDict[visit] = airmass 

552 angleDict[visit] = parallacticAngle 

553 psfSizeDict[visit] = psfSize 

554 if self.config.doAirmassWeight: 

555 weightList[visitNum] *= airmass 

556 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(), 

557 self.config.effectiveWavelength, 

558 self.config.bandwidth, 

559 self.config.dcrNumSubfilters)))) 

560 self.log.info("Selected airmasses:\n%s", airmassDict) 

561 self.log.info("Selected parallactic angles:\n%s", angleDict) 

562 self.log.info("Selected PSF sizes:\n%s", psfSizeDict) 

563 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1) 

564 try: 

565 psf = self.selectCoaddPsf(templateCoadd, warpRefList) 

566 except Exception as e: 

567 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e) 

568 else: 

569 psf = templateCoadd.getPsf() 

570 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage, 

571 self.config.dcrNumSubfilters, 

572 effectiveWavelength=self.config.effectiveWavelength, 

573 bandwidth=self.config.bandwidth, 

574 wcs=templateCoadd.getWcs(), 

575 filterLabel=filterLabel, 

576 psf=psf) 

577 return dcrModels 

578 

579 @timeMethod 

580 def run(self, skyInfo, warpRefList, imageScalerList, weightList, 

581 supplementaryData=None): 

582 """Assemble the coadd. 

583 

584 Requires additional inputs Struct ``supplementaryData`` to contain a 

585 ``templateCoadd`` that serves as the model of the static sky. 

586 

587 Find artifacts and apply them to the warps' masks creating a list of 

588 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 

589 Then pass these alternative masks to the base class's assemble method. 

590 

591 Divide the ``templateCoadd`` evenly between each subfilter of a 

592 ``DcrModel`` as the starting best estimate of the true wavelength- 

593 dependent sky. Forward model the ``DcrModel`` using the known 

594 chromatic effects in each subfilter and calculate a convergence metric 

595 based on how well the modeled template matches the input warps. If 

596 the convergence has not yet reached the desired threshold, then shift 

597 and stack the residual images to build a new ``DcrModel``. Apply 

598 conditioning to prevent oscillating solutions between iterations or 

599 between subfilters. 

600 

601 Once the ``DcrModel`` reaches convergence or the maximum number of 

602 iterations has been reached, fill the metadata for each subfilter 

603 image and make them proper ``coaddExposure``s. 

604 

605 Parameters 

606 ---------- 

607 skyInfo : `lsst.pipe.base.Struct` 

608 Patch geometry information, from getSkyInfo 

609 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

610 `lsst.daf.persistence.ButlerDataRef` 

611 The data references to the input warped exposures. 

612 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

613 The image scalars correct for the zero point of the exposures. 

614 weightList : `list` of `float` 

615 The weight to give each input exposure in the coadd 

616 supplementaryData : `lsst.pipe.base.Struct` 

617 Result struct returned by ``makeSupplementaryData`` with components: 

618 

619 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 

620 

621 Returns 

622 ------- 

623 result : `lsst.pipe.base.Struct` 

624 Result struct with components: 

625 

626 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

627 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

628 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

629 - ``dcrNImages``: `list` of exposure count images for each subfilter 

630 """ 

631 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters 

632 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3 

633 templateCoadd = supplementaryData.templateCoadd 

634 baseMask = templateCoadd.mask.clone() 

635 # The variance plane is for each subfilter 

636 # and should be proportionately lower than the full-band image 

637 baseVariance = templateCoadd.variance.clone() 

638 baseVariance /= self.config.dcrNumSubfilters 

639 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList) 

640 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask. 

641 templateCoadd.setMask(baseMask) 

642 badMaskPlanes = self.config.badMaskPlanes[:] 

643 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes`` 

644 # This is because pixels in observations that are significantly affect by DCR 

645 # are likely to have many pixels that are both "DETECTED" and "CLIPPED", 

646 # but those are necessary to constrain the DCR model. 

647 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes) 

648 

649 stats = self.prepareStats(mask=badPixelMask) 

650 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList) 

651 if self.config.doNImage: 

652 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList, 

653 spanSetMaskList, stats.ctrl) 

654 nImage = afwImage.ImageU(skyInfo.bbox) 

655 # Note that this nImage will be a factor of dcrNumSubfilters higher than 

656 # the nImage returned by assembleCoadd for most pixels. This is because each 

657 # subfilter may have a different nImage, and fractional values are not allowed. 

658 for dcrNImage in dcrNImages: 

659 nImage += dcrNImage 

660 else: 

661 dcrNImages = None 

662 

663 subregionSize = geom.Extent2I(*self.config.subregionSize) 

664 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) 

665 * ceil(skyInfo.bbox.getWidth()/subregionSize[0])) 

666 subIter = 0 

667 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

668 modelIter = 0 

669 subIter += 1 

670 self.log.info("Computing coadd over patch %s subregion %s of %s: %s", 

671 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox) 

672 dcrBBox = geom.Box2I(subBBox) 

673 dcrBBox.grow(self.bufferSize) 

674 dcrBBox.clip(dcrModels.bbox) 

675 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox) 

676 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList, 

677 imageScalerList, spanSetMaskList) 

678 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

679 warpRefList, weightList, stats.ctrl) 

680 self.log.info("Initial convergence : %s", convergenceMetric) 

681 convergenceList = [convergenceMetric] 

682 gainList = [] 

683 convergenceCheck = 1. 

684 refImage = templateCoadd.image 

685 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter): 

686 gain = self.calculateGain(convergenceList, gainList) 

687 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList, 

688 stats.ctrl, convergenceMetric, gain, 

689 modelWeights, refImage, dcrWeights) 

690 if self.config.useConvergence: 

691 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

692 warpRefList, weightList, stats.ctrl) 

693 if convergenceMetric == 0: 

694 self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is " 

695 "most likely due to there being no valid data in the region.", 

696 skyInfo.patchInfo.getIndex(), subIter) 

697 break 

698 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric 

699 if (convergenceCheck < 0) & (modelIter > minNumIter): 

700 self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum " 

701 "iterations or desired convergence improvement of %s." 

702 " Divergence: %s", 

703 skyInfo.patchInfo.getIndex(), subIter, 

704 self.config.convergenceThreshold, convergenceCheck) 

705 break 

706 convergenceList.append(convergenceMetric) 

707 if modelIter > maxNumIter: 

708 if self.config.useConvergence: 

709 self.log.warning("Coadd patch %s subregion %s reached maximum iterations " 

710 "before reaching desired convergence improvement of %s." 

711 " Final convergence improvement: %s", 

712 skyInfo.patchInfo.getIndex(), subIter, 

713 self.config.convergenceThreshold, convergenceCheck) 

714 break 

715 

716 if self.config.useConvergence: 

717 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)", 

718 modelIter, convergenceMetric, 100.*convergenceCheck, gain) 

719 modelIter += 1 

720 else: 

721 if self.config.useConvergence: 

722 self.log.info("Coadd patch %s subregion %s finished with " 

723 "convergence metric %s after %s iterations", 

724 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter) 

725 else: 

726 self.log.info("Coadd patch %s subregion %s finished after %s iterations", 

727 skyInfo.patchInfo.getIndex(), subIter, modelIter) 

728 if self.config.useConvergence and convergenceMetric > 0: 

729 self.log.info("Final convergence improvement was %.4f%% overall", 

730 100*(convergenceList[0] - convergenceMetric)/convergenceMetric) 

731 

732 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList, 

733 calibration=self.scaleZeroPoint.getPhotoCalib(), 

734 coaddInputs=templateCoadd.getInfo().getCoaddInputs(), 

735 mask=baseMask, 

736 variance=baseVariance) 

737 coaddExposure = self.stackCoadd(dcrCoadds) 

738 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

739 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages) 

740 

741 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl): 

742 """Calculate the number of exposures contributing to each subfilter. 

743 

744 Parameters 

745 ---------- 

746 dcrModels : `lsst.pipe.tasks.DcrModel` 

747 Best fit model of the true sky after correcting chromatic effects. 

748 bbox : `lsst.geom.box.Box2I` 

749 Bounding box of the patch to coadd. 

750 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

751 `lsst.daf.persistence.ButlerDataRef` 

752 The data references to the input warped exposures. 

753 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

754 Each element of the `dict` contains the new mask plane name 

755 (e.g. "CLIPPED and/or "NO_DATA") as the key, 

756 and the list of SpanSets to apply to the mask. 

757 statsCtrl : `lsst.afw.math.StatisticsControl` 

758 Statistics control object for coadd 

759 

760 Returns 

761 ------- 

762 dcrNImages : `list` of `lsst.afw.image.ImageU` 

763 List of exposure count images for each subfilter 

764 dcrWeights : `list` of `lsst.afw.image.ImageF` 

765 Per-pixel weights for each subfilter. 

766 Equal to 1/(number of unmasked images contributing to each pixel). 

767 """ 

768 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

769 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

770 tempExpName = self.getTempExpDatasetName(self.warpType) 

771 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList): 

772 if isinstance(warpExpRef, DeferredDatasetHandle): 

773 # Gen 3 API 

774 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

775 else: 

776 # Gen 2 API. Delete this when Gen 2 retired 

777 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

778 visitInfo = exposure.getInfo().getVisitInfo() 

779 wcs = exposure.getInfo().getWcs() 

780 mask = exposure.mask 

781 if altMaskSpans is not None: 

782 self.applyAltMaskPlanes(mask, altMaskSpans) 

783 weightImage = np.zeros_like(exposure.image.array) 

784 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1. 

785 # The weights must be shifted in exactly the same way as the residuals, 

786 # because they will be used as the denominator in the weighted average of residuals. 

787 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, 

788 dcrModels.effectiveWavelength, dcrModels.bandwidth) 

789 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights): 

790 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype) 

791 dcrWeight.array += shiftedWeights 

792 # Exclude any pixels that don't have at least one exposure contributing in all subfilters 

793 weightsThreshold = 1. 

794 goodPix = dcrWeights[0].array > weightsThreshold 

795 for weights in dcrWeights[1:]: 

796 goodPix = (weights.array > weightsThreshold) & goodPix 

797 for subfilter in range(self.config.dcrNumSubfilters): 

798 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix] 

799 dcrWeights[subfilter].array[~goodPix] = 0. 

800 dcrNImages[subfilter].array[~goodPix] = 0 

801 return (dcrNImages, dcrWeights) 

802 

803 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, 

804 statsCtrl, convergenceMetric, 

805 gain, modelWeights, refImage, dcrWeights): 

806 """Assemble the DCR coadd for a sub-region. 

807 

808 Build a DCR-matched template for each input exposure, then shift the 

809 residuals according to the DCR in each subfilter. 

810 Stack the shifted residuals and apply them as a correction to the 

811 solution from the previous iteration. 

812 Restrict the new model solutions from varying by more than a factor of 

813 `modelClampFactor` from the last solution, and additionally restrict the 

814 individual subfilter models from varying by more than a factor of 

815 `frequencyClampFactor` from their average. 

816 Finally, mitigate potentially oscillating solutions by averaging the new 

817 solution with the solution from the previous iteration, weighted by 

818 their convergence metric. 

819 

820 Parameters 

821 ---------- 

822 dcrModels : `lsst.pipe.tasks.DcrModel` 

823 Best fit model of the true sky after correcting chromatic effects. 

824 subExposures : `dict` of `lsst.afw.image.ExposureF` 

825 The pre-loaded exposures for the current subregion. 

826 bbox : `lsst.geom.box.Box2I` 

827 Bounding box of the subregion to coadd. 

828 dcrBBox : `lsst.geom.box.Box2I` 

829 Sub-region of the coadd which includes a buffer to allow for DCR. 

830 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

831 `lsst.daf.persistence.ButlerDataRef` 

832 The data references to the input warped exposures. 

833 statsCtrl : `lsst.afw.math.StatisticsControl` 

834 Statistics control object for coadd 

835 convergenceMetric : `float` 

836 Quality of fit metric for the matched templates of the input images. 

837 gain : `float`, optional 

838 Relative weight to give the new solution when updating the model. 

839 modelWeights : `numpy.ndarray` or `float` 

840 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

841 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

842 refImage : `lsst.afw.image.Image` 

843 A reference image used to supply the default pixel values. 

844 dcrWeights : `list` of `lsst.afw.image.Image` 

845 Per-pixel weights for each subfilter. 

846 Equal to 1/(number of unmasked images contributing to each pixel). 

847 """ 

848 residualGeneratorList = [] 

849 

850 for warpExpRef in warpRefList: 

851 visit = warpExpRef.dataId["visit"] 

852 exposure = subExposures[visit] 

853 visitInfo = exposure.getInfo().getVisitInfo() 

854 wcs = exposure.getInfo().getWcs() 

855 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

856 bbox=exposure.getBBox(), 

857 order=self.config.imageInterpOrder, 

858 splitSubfilters=self.config.splitSubfilters, 

859 splitThreshold=self.config.splitThreshold, 

860 amplifyModel=self.config.accelerateModel) 

861 residual = exposure.image.array - templateImage.array 

862 # Note that the variance plane here is used to store weights, not the actual variance 

863 residual *= exposure.variance.array 

864 # The residuals are stored as a list of generators. 

865 # This allows the residual for a given subfilter and exposure to be created 

866 # on the fly, instead of needing to store them all in memory. 

867 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, 

868 dcrModels.effectiveWavelength, 

869 dcrModels.bandwidth)) 

870 

871 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

872 gain=gain, 

873 modelWeights=modelWeights, 

874 refImage=refImage, 

875 dcrWeights=dcrWeights) 

876 dcrModels.assign(dcrSubModelOut, bbox) 

877 

878 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth): 

879 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 

880 

881 Parameters 

882 ---------- 

883 residual : `numpy.ndarray` 

884 The residual masked image for one exposure, 

885 after subtracting the matched template 

886 visitInfo : `lsst.afw.image.VisitInfo` 

887 Metadata for the exposure. 

888 wcs : `lsst.afw.geom.SkyWcs` 

889 Coordinate system definition (wcs) for the exposure. 

890 

891 Yields 

892 ------ 

893 residualImage : `numpy.ndarray` 

894 The residual image for the next subfilter, shifted for DCR. 

895 """ 

896 if self.config.imageInterpOrder > 1: 

897 # Pre-calculate the spline-filtered residual image, so that step can be 

898 # skipped in the shift calculation in `applyDcr`. 

899 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder) 

900 else: 

901 # No need to prefilter if order=1 (it will also raise an error) 

902 filteredResidual = residual 

903 # Note that `splitSubfilters` is always turned off in the reverse direction. 

904 # This option introduces additional blurring if applied to the residuals. 

905 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters, 

906 splitSubfilters=False) 

907 for dcr in dcrShift: 

908 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False, 

909 doPrefilter=False, order=self.config.imageInterpOrder) 

910 

911 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

912 gain, modelWeights, refImage, dcrWeights): 

913 """Calculate a new DcrModel from a set of image residuals. 

914 

915 Parameters 

916 ---------- 

917 dcrModels : `lsst.pipe.tasks.DcrModel` 

918 Current model of the true sky after correcting chromatic effects. 

919 residualGeneratorList : `generator` of `numpy.ndarray` 

920 The residual image for the next subfilter, shifted for DCR. 

921 dcrBBox : `lsst.geom.box.Box2I` 

922 Sub-region of the coadd which includes a buffer to allow for DCR. 

923 statsCtrl : `lsst.afw.math.StatisticsControl` 

924 Statistics control object for coadd 

925 gain : `float` 

926 Relative weight to give the new solution when updating the model. 

927 modelWeights : `numpy.ndarray` or `float` 

928 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

929 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

930 refImage : `lsst.afw.image.Image` 

931 A reference image used to supply the default pixel values. 

932 dcrWeights : `list` of `lsst.afw.image.Image` 

933 Per-pixel weights for each subfilter. 

934 Equal to 1/(number of unmasked images contributing to each pixel). 

935 

936 Returns 

937 ------- 

938 dcrModel : `lsst.pipe.tasks.DcrModel` 

939 New model of the true sky after correcting chromatic effects. 

940 """ 

941 newModelImages = [] 

942 for subfilter, model in enumerate(dcrModels): 

943 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList] 

944 residual = np.sum(residualsList, axis=0) 

945 residual *= dcrWeights[subfilter][dcrBBox].array 

946 # `MaskedImage`s only support in-place addition, so rename for readability 

947 newModel = model[dcrBBox].clone() 

948 newModel.array += residual 

949 # Catch any invalid values 

950 badPixels = ~np.isfinite(newModel.array) 

951 newModel.array[badPixels] = model[dcrBBox].array[badPixels] 

952 if self.config.regularizeModelIterations > 0: 

953 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox, 

954 self.config.regularizeModelIterations, 

955 self.config.regularizationWidth) 

956 newModelImages.append(newModel) 

957 if self.config.regularizeModelFrequency > 0: 

958 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl, 

959 self.config.regularizeModelFrequency, 

960 self.config.regularizationWidth) 

961 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain) 

962 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights) 

963 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength, 

964 dcrModels.bandwidth, dcrModels.psf, 

965 dcrModels.mask, dcrModels.variance) 

966 

967 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl): 

968 """Calculate a quality of fit metric for the matched templates. 

969 

970 Parameters 

971 ---------- 

972 dcrModels : `lsst.pipe.tasks.DcrModel` 

973 Best fit model of the true sky after correcting chromatic effects. 

974 subExposures : `dict` of `lsst.afw.image.ExposureF` 

975 The pre-loaded exposures for the current subregion. 

976 bbox : `lsst.geom.box.Box2I` 

977 Sub-region to coadd 

978 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

979 `lsst.daf.persistence.ButlerDataRef` 

980 The data references to the input warped exposures. 

981 weightList : `list` of `float` 

982 The weight to give each input exposure in the coadd 

983 statsCtrl : `lsst.afw.math.StatisticsControl` 

984 Statistics control object for coadd 

985 

986 Returns 

987 ------- 

988 convergenceMetric : `float` 

989 Quality of fit metric for all input exposures, within the sub-region 

990 """ 

991 significanceImage = np.abs(dcrModels.getReferenceImage(bbox)) 

992 nSigma = 3. 

993 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl, 

994 bufferSize=self.bufferSize) 

995 if np.max(significanceImage) == 0: 

996 significanceImage += 1. 

997 weight = 0 

998 metric = 0. 

999 metricList = {} 

1000 for warpExpRef, expWeight in zip(warpRefList, weightList): 

1001 visit = warpExpRef.dataId["visit"] 

1002 exposure = subExposures[visit][bbox] 

1003 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl) 

1004 metric += singleMetric 

1005 metricList[visit] = singleMetric 

1006 weight += 1. 

1007 self.log.info("Individual metrics:\n%s", metricList) 

1008 return 1.0 if weight == 0.0 else metric/weight 

1009 

1010 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl): 

1011 """Calculate a quality of fit metric for a single matched template. 

1012 

1013 Parameters 

1014 ---------- 

1015 dcrModels : `lsst.pipe.tasks.DcrModel` 

1016 Best fit model of the true sky after correcting chromatic effects. 

1017 exposure : `lsst.afw.image.ExposureF` 

1018 The input warped exposure to evaluate. 

1019 significanceImage : `numpy.ndarray` 

1020 Array of weights for each pixel corresponding to its significance 

1021 for the convergence calculation. 

1022 statsCtrl : `lsst.afw.math.StatisticsControl` 

1023 Statistics control object for coadd 

1024 

1025 Returns 

1026 ------- 

1027 convergenceMetric : `float` 

1028 Quality of fit metric for one exposure, within the sub-region. 

1029 """ 

1030 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1031 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

1032 bbox=exposure.getBBox(), 

1033 order=self.config.imageInterpOrder, 

1034 splitSubfilters=self.config.splitSubfilters, 

1035 splitThreshold=self.config.splitThreshold, 

1036 amplifyModel=self.config.accelerateModel) 

1037 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage 

1038 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2. 

1039 

1040 finitePixels = np.isfinite(diffVals) 

1041 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0 

1042 convergeMaskPixels = exposure.mask.array & convergeMask > 0 

1043 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels 

1044 if np.sum(usePixels) == 0: 

1045 metric = 0. 

1046 else: 

1047 diffUse = diffVals[usePixels] 

1048 refUse = refVals[usePixels] 

1049 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse)) 

1050 return metric 

1051 

1052 def stackCoadd(self, dcrCoadds): 

1053 """Add a list of sub-band coadds together. 

1054 

1055 Parameters 

1056 ---------- 

1057 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1058 A list of coadd exposures, each exposure containing 

1059 the model for one subfilter. 

1060 

1061 Returns 

1062 ------- 

1063 coaddExposure : `lsst.afw.image.ExposureF` 

1064 A single coadd exposure that is the sum of the sub-bands. 

1065 """ 

1066 coaddExposure = dcrCoadds[0].clone() 

1067 for coadd in dcrCoadds[1:]: 

1068 coaddExposure.maskedImage += coadd.maskedImage 

1069 return coaddExposure 

1070 

1071 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, 

1072 mask=None, variance=None): 

1073 """Create a list of coadd exposures from a list of masked images. 

1074 

1075 Parameters 

1076 ---------- 

1077 dcrModels : `lsst.pipe.tasks.DcrModel` 

1078 Best fit model of the true sky after correcting chromatic effects. 

1079 skyInfo : `lsst.pipe.base.Struct` 

1080 Patch geometry information, from getSkyInfo 

1081 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1082 `lsst.daf.persistence.ButlerDataRef` 

1083 The data references to the input warped exposures. 

1084 weightList : `list` of `float` 

1085 The weight to give each input exposure in the coadd 

1086 calibration : `lsst.afw.Image.PhotoCalib`, optional 

1087 Scale factor to set the photometric calibration of an exposure. 

1088 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 

1089 A record of the observations that are included in the coadd. 

1090 mask : `lsst.afw.image.Mask`, optional 

1091 Optional mask to override the values in the final coadd. 

1092 variance : `lsst.afw.image.Image`, optional 

1093 Optional variance plane to override the values in the final coadd. 

1094 

1095 Returns 

1096 ------- 

1097 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1098 A list of coadd exposures, each exposure containing 

1099 the model for one subfilter. 

1100 """ 

1101 dcrCoadds = [] 

1102 refModel = dcrModels.getReferenceImage() 

1103 for model in dcrModels: 

1104 if self.config.accelerateModel > 1: 

1105 model.array = (model.array - refModel)*self.config.accelerateModel + refModel 

1106 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

1107 if calibration is not None: 

1108 coaddExposure.setPhotoCalib(calibration) 

1109 if coaddInputs is not None: 

1110 coaddExposure.getInfo().setCoaddInputs(coaddInputs) 

1111 # Set the metadata for the coadd, including PSF and aperture corrections. 

1112 self.assembleMetadata(coaddExposure, warpRefList, weightList) 

1113 # Overwrite the PSF 

1114 coaddExposure.setPsf(dcrModels.psf) 

1115 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox]) 

1116 maskedImage = afwImage.MaskedImageF(dcrModels.bbox) 

1117 maskedImage.image = model 

1118 maskedImage.mask = dcrModels.mask 

1119 maskedImage.variance = dcrModels.variance 

1120 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox]) 

1121 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

1122 if mask is not None: 

1123 coaddExposure.setMask(mask) 

1124 if variance is not None: 

1125 coaddExposure.setVariance(variance) 

1126 dcrCoadds.append(coaddExposure) 

1127 return dcrCoadds 

1128 

1129 def calculateGain(self, convergenceList, gainList): 

1130 """Calculate the gain to use for the current iteration. 

1131 

1132 After calculating a new DcrModel, each value is averaged with the 

1133 value in the corresponding pixel from the previous iteration. This 

1134 reduces oscillating solutions that iterative techniques are plagued by, 

1135 and speeds convergence. By far the biggest changes to the model 

1136 happen in the first couple iterations, so we can also use a more 

1137 aggressive gain later when the model is changing slowly. 

1138 

1139 Parameters 

1140 ---------- 

1141 convergenceList : `list` of `float` 

1142 The quality of fit metric from each previous iteration. 

1143 gainList : `list` of `float` 

1144 The gains used in each previous iteration: appended with the new 

1145 gain value. 

1146 Gains are numbers between ``self.config.baseGain`` and 1. 

1147 

1148 Returns 

1149 ------- 

1150 gain : `float` 

1151 Relative weight to give the new solution when updating the model. 

1152 A value of 1.0 gives equal weight to both solutions. 

1153 

1154 Raises 

1155 ------ 

1156 ValueError 

1157 If ``len(convergenceList) != len(gainList)+1``. 

1158 """ 

1159 nIter = len(convergenceList) 

1160 if nIter != len(gainList) + 1: 

1161 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)." 

1162 % (len(convergenceList), len(gainList))) 

1163 

1164 if self.config.baseGain is None: 

1165 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters 

1166 # The more subfilters being modeled, the lower the gain should be. 

1167 baseGain = 1./(self.config.dcrNumSubfilters - 1) 

1168 else: 

1169 baseGain = self.config.baseGain 

1170 

1171 if self.config.useProgressiveGain and nIter > 2: 

1172 # To calculate the best gain to use, compare the past gains that have been used 

1173 # with the resulting convergences to estimate the best gain to use. 

1174 # Algorithmically, this is a Kalman filter. 

1175 # If forward modeling proceeds perfectly, the convergence metric should 

1176 # asymptotically approach a final value. 

1177 # We can estimate that value from the measured changes in convergence 

1178 # weighted by the gains used in each previous iteration. 

1179 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i] 

1180 for i in range(nIter - 1)] 

1181 # The convergence metric is strictly positive, so if the estimated final convergence is 

1182 # less than zero, force it to zero. 

1183 estFinalConv = np.array(estFinalConv) 

1184 estFinalConv[estFinalConv < 0] = 0 

1185 # Because the estimate may slowly change over time, only use the most recent measurements. 

1186 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):]) 

1187 lastGain = gainList[-1] 

1188 lastConv = convergenceList[-2] 

1189 newConv = convergenceList[-1] 

1190 # The predicted convergence is the value we would get if the new model calculated 

1191 # in the previous iteration was perfect. Recall that the updated model that is 

1192 # actually used is the gain-weighted average of the new and old model, 

1193 # so the convergence would be similarly weighted. 

1194 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain) 

1195 # If the measured and predicted convergence are very close, that indicates 

1196 # that our forward model is accurate and we can use a more aggressive gain 

1197 # If the measured convergence is significantly worse (or better!) than predicted, 

1198 # that indicates that the model is not converging as expected and 

1199 # we should use a more conservative gain. 

1200 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain)) 

1201 newGain = 1 - abs(delta) 

1202 # Average the gains to prevent oscillating solutions. 

1203 newGain = (newGain + lastGain)/2. 

1204 gain = max(baseGain, newGain) 

1205 else: 

1206 gain = baseGain 

1207 gainList.append(gain) 

1208 return gain 

1209 

1210 def calculateModelWeights(self, dcrModels, dcrBBox): 

1211 """Build an array that smoothly tapers to 0 away from detected sources. 

1212 

1213 Parameters 

1214 ---------- 

1215 dcrModels : `lsst.pipe.tasks.DcrModel` 

1216 Best fit model of the true sky after correcting chromatic effects. 

1217 dcrBBox : `lsst.geom.box.Box2I` 

1218 Sub-region of the coadd which includes a buffer to allow for DCR. 

1219 

1220 Returns 

1221 ------- 

1222 weights : `numpy.ndarray` or `float` 

1223 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1224 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1225 

1226 Raises 

1227 ------ 

1228 ValueError 

1229 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 

1230 """ 

1231 if not self.config.useModelWeights: 

1232 return 1.0 

1233 if self.config.modelWeightsWidth < 0: 

1234 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set") 

1235 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1236 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0 

1237 weights = np.zeros_like(dcrModels[0][dcrBBox].array) 

1238 weights[convergeMaskPixels] = 1. 

1239 weights = ndimage.gaussian_filter(weights, self.config.modelWeightsWidth) 

1240 weights /= np.max(weights) 

1241 return weights 

1242 

1243 def applyModelWeights(self, modelImages, refImage, modelWeights): 

1244 """Smoothly replace model pixel values with those from a 

1245 reference at locations away from detected sources. 

1246 

1247 Parameters 

1248 ---------- 

1249 modelImages : `list` of `lsst.afw.image.Image` 

1250 The new DCR model images from the current iteration. 

1251 The values will be modified in place. 

1252 refImage : `lsst.afw.image.MaskedImage` 

1253 A reference image used to supply the default pixel values. 

1254 modelWeights : `numpy.ndarray` or `float` 

1255 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1256 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1257 """ 

1258 if self.config.useModelWeights: 

1259 for model in modelImages: 

1260 model.array *= modelWeights 

1261 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters 

1262 

1263 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList): 

1264 """Pre-load sub-regions of a list of exposures. 

1265 

1266 Parameters 

1267 ---------- 

1268 bbox : `lsst.geom.box.Box2I` 

1269 Sub-region to coadd 

1270 statsCtrl : `lsst.afw.math.StatisticsControl` 

1271 Statistics control object for coadd 

1272 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1273 `lsst.daf.persistence.ButlerDataRef` 

1274 The data references to the input warped exposures. 

1275 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

1276 The image scalars correct for the zero point of the exposures. 

1277 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

1278 Each element is dict with keys = mask plane name to add the spans to 

1279 

1280 Returns 

1281 ------- 

1282 subExposures : `dict` 

1283 The `dict` keys are the visit IDs, 

1284 and the values are `lsst.afw.image.ExposureF` 

1285 The pre-loaded exposures for the current subregion. 

1286 The variance plane contains weights, and not the variance 

1287 """ 

1288 tempExpName = self.getTempExpDatasetName(self.warpType) 

1289 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList) 

1290 subExposures = {} 

1291 for warpExpRef, imageScaler, altMaskSpans in zipIterables: 

1292 if isinstance(warpExpRef, DeferredDatasetHandle): 

1293 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

1294 else: 

1295 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

1296 visit = warpExpRef.dataId["visit"] 

1297 if altMaskSpans is not None: 

1298 self.applyAltMaskPlanes(exposure.mask, altMaskSpans) 

1299 imageScaler.scaleMaskedImage(exposure.maskedImage) 

1300 # Note that the variance plane here is used to store weights, not the actual variance 

1301 exposure.variance.array[:, :] = 0. 

1302 # Set the weight of unmasked pixels to 1. 

1303 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1. 

1304 # Set the image value of masked pixels to zero. 

1305 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual`` 

1306 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0. 

1307 subExposures[visit] = exposure 

1308 return subExposures 

1309 

1310 def selectCoaddPsf(self, templateCoadd, warpRefList): 

1311 """Compute the PSF of the coadd from the exposures with the best seeing. 

1312 

1313 Parameters 

1314 ---------- 

1315 templateCoadd : `lsst.afw.image.ExposureF` 

1316 The initial coadd exposure before accounting for DCR. 

1317 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1318 `lsst.daf.persistence.ButlerDataRef` 

1319 The data references to the input warped exposures. 

1320 

1321 Returns 

1322 ------- 

1323 psf : `lsst.meas.algorithms.CoaddPsf` 

1324 The average PSF of the input exposures with the best seeing. 

1325 """ 

1326 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

1327 tempExpName = self.getTempExpDatasetName(self.warpType) 

1328 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit 

1329 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef`` 

1330 ccds = templateCoadd.getInfo().getCoaddInputs().ccds 

1331 templatePsf = templateCoadd.getPsf() 

1332 # Just need a rough estimate; average positions are fine 

1333 templateAvgPos = templatePsf.getAveragePosition() 

1334 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()*sigma2fwhm 

1335 psfSizes = np.zeros(len(ccds)) 

1336 ccdVisits = np.array(ccds["visit"]) 

1337 for warpExpRef in warpRefList: 

1338 if isinstance(warpExpRef, DeferredDatasetHandle): 

1339 # Gen 3 API 

1340 psf = warpExpRef.get(component="psf") 

1341 else: 

1342 # Gen 2 API. Delete this when Gen 2 retired 

1343 psf = warpExpRef.get(tempExpName).getPsf() 

1344 visit = warpExpRef.dataId["visit"] 

1345 psfAvgPos = psf.getAveragePosition() 

1346 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm 

1347 psfSizes[ccdVisits == visit] = psfSize 

1348 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd 

1349 # The selected PSFs are those that have a FWHM less than or equal to the smaller 

1350 # of the mean or median FWHM of the input exposures. 

1351 sizeThreshold = min(np.median(psfSizes), psfRefSize) 

1352 goodPsfs = psfSizes <= sizeThreshold 

1353 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), 

1354 self.config.coaddPsf.makeControl()) 

1355 return psf