Coverage for python/lsst/pipe/tasks/dcrAssembleCoadd.py: 15%

452 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-17 08:57 +0000

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23from math import ceil 

24import numpy as np 

25from scipy import ndimage 

26import lsst.geom as geom 

27import lsst.afw.image as afwImage 

28import lsst.afw.table as afwTable 

29import lsst.coadd.utils as coaddUtils 

30from lsst.daf.butler import DeferredDatasetHandle 

31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel 

32import lsst.meas.algorithms as measAlg 

33from lsst.meas.base import SingleFrameMeasurementTask 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.utils as utils 

37from lsst.skymap import BaseSkyMap 

38from .assembleCoadd import (AssembleCoaddTask, 

39 CompareWarpAssembleCoaddConfig, 

40 CompareWarpAssembleCoaddTask) 

41from .coaddBase import makeSkyInfo 

42from .measurePsf import MeasurePsfTask 

43 

44__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"] 

45 

46 

47class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

48 dimensions=("tract", "patch", "band", "skymap"), 

49 defaultTemplates={"inputCoaddName": "deep", 

50 "outputCoaddName": "dcr", 

51 "warpType": "direct", 

52 "warpTypeSuffix": "", 

53 "fakesType": ""}): 

54 inputWarps = pipeBase.connectionTypes.Input( 

55 doc=("Input list of warps to be assembled i.e. stacked." 

56 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

57 name="{inputCoaddName}Coadd_{warpType}Warp", 

58 storageClass="ExposureF", 

59 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

60 deferLoad=True, 

61 multiple=True 

62 ) 

63 skyMap = pipeBase.connectionTypes.Input( 

64 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

65 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

66 storageClass="SkyMap", 

67 dimensions=("skymap", ), 

68 ) 

69 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

70 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

71 " BRIGHT_OBJECT."), 

72 name="brightObjectMask", 

73 storageClass="ObjectMaskCatalog", 

74 dimensions=("tract", "patch", "skymap", "band"), 

75 ) 

76 templateExposure = pipeBase.connectionTypes.Input( 

77 doc="Input coadded exposure, produced by previous call to AssembleCoadd", 

78 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}", 

79 storageClass="ExposureF", 

80 dimensions=("tract", "patch", "skymap", "band"), 

81 ) 

82 dcrCoadds = pipeBase.connectionTypes.Output( 

83 doc="Output coadded exposure, produced by stacking input warps", 

84 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

85 storageClass="ExposureF", 

86 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

87 multiple=True, 

88 ) 

89 dcrNImages = pipeBase.connectionTypes.Output( 

90 doc="Output image of number of input images per pixel", 

91 name="{outputCoaddName}Coadd_nImage", 

92 storageClass="ImageU", 

93 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

94 multiple=True, 

95 ) 

96 

97 def __init__(self, *, config=None): 

98 super().__init__(config=config) 

99 if not config.doWrite: 

100 self.outputs.remove("dcrCoadds") 

101 

102 

103class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, 

104 pipelineConnections=DcrAssembleCoaddConnections): 

105 dcrNumSubfilters = pexConfig.Field( 

106 dtype=int, 

107 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.", 

108 default=3, 

109 ) 

110 maxNumIter = pexConfig.Field( 

111 dtype=int, 

112 optional=True, 

113 doc="Maximum number of iterations of forward modeling.", 

114 default=None, 

115 ) 

116 minNumIter = pexConfig.Field( 

117 dtype=int, 

118 optional=True, 

119 doc="Minimum number of iterations of forward modeling.", 

120 default=None, 

121 ) 

122 convergenceThreshold = pexConfig.Field( 

123 dtype=float, 

124 doc="Target relative change in convergence between iterations of forward modeling.", 

125 default=0.001, 

126 ) 

127 useConvergence = pexConfig.Field( 

128 dtype=bool, 

129 doc="Use convergence test as a forward modeling end condition?" 

130 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations", 

131 default=True, 

132 ) 

133 baseGain = pexConfig.Field( 

134 dtype=float, 

135 optional=True, 

136 doc="Relative weight to give the new solution vs. the last solution when updating the model." 

137 "A value of 1.0 gives equal weight to both solutions." 

138 "Small values imply slower convergence of the solution, but can " 

139 "help prevent overshooting and failures in the fit." 

140 "If ``baseGain`` is None, a conservative gain " 

141 "will be calculated from the number of subfilters. ", 

142 default=None, 

143 ) 

144 useProgressiveGain = pexConfig.Field( 

145 dtype=bool, 

146 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 

147 "When calculating the next gain, we use up to 5 previous gains and convergence values." 

148 "Can be set to False to force the model to change at the rate of ``baseGain``. ", 

149 default=True, 

150 ) 

151 doAirmassWeight = pexConfig.Field( 

152 dtype=bool, 

153 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.", 

154 default=False, 

155 ) 

156 modelWeightsWidth = pexConfig.Field( 

157 dtype=float, 

158 doc="Width of the region around detected sources to include in the DcrModel.", 

159 default=3, 

160 ) 

161 useModelWeights = pexConfig.Field( 

162 dtype=bool, 

163 doc="Width of the region around detected sources to include in the DcrModel.", 

164 default=True, 

165 ) 

166 splitSubfilters = pexConfig.Field( 

167 dtype=bool, 

168 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter." 

169 "Instead of at the midpoint", 

170 default=True, 

171 ) 

172 splitThreshold = pexConfig.Field( 

173 dtype=float, 

174 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 

175 "Set to 0 to always split the subfilters.", 

176 default=0.1, 

177 ) 

178 regularizeModelIterations = pexConfig.Field( 

179 dtype=float, 

180 doc="Maximum relative change of the model allowed between iterations." 

181 "Set to zero to disable.", 

182 default=2., 

183 ) 

184 regularizeModelFrequency = pexConfig.Field( 

185 dtype=float, 

186 doc="Maximum relative change of the model allowed between subfilters." 

187 "Set to zero to disable.", 

188 default=4., 

189 ) 

190 convergenceMaskPlanes = pexConfig.ListField( 

191 dtype=str, 

192 default=["DETECTED"], 

193 doc="Mask planes to use to calculate convergence." 

194 ) 

195 regularizationWidth = pexConfig.Field( 

196 dtype=int, 

197 default=2, 

198 doc="Minimum radius of a region to include in regularization, in pixels." 

199 ) 

200 imageInterpOrder = pexConfig.Field( 

201 dtype=int, 

202 doc="The order of the spline interpolation used to shift the image plane.", 

203 default=3, 

204 ) 

205 accelerateModel = pexConfig.Field( 

206 dtype=float, 

207 doc="Factor to amplify the differences between model planes by to speed convergence.", 

208 default=3, 

209 ) 

210 doCalculatePsf = pexConfig.Field( 

211 dtype=bool, 

212 doc="Set to detect stars and recalculate the PSF from the final coadd." 

213 "Otherwise the PSF is estimated from a selection of the best input exposures", 

214 default=False, 

215 ) 

216 detectPsfSources = pexConfig.ConfigurableField( 

217 target=measAlg.SourceDetectionTask, 

218 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.", 

219 ) 

220 measurePsfSources = pexConfig.ConfigurableField( 

221 target=SingleFrameMeasurementTask, 

222 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 

223 ) 

224 measurePsf = pexConfig.ConfigurableField( 

225 target=MeasurePsfTask, 

226 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.", 

227 ) 

228 effectiveWavelength = pexConfig.Field( 

229 doc="Effective wavelength of the filter, in nm." 

230 "Required if transmission curves aren't used." 

231 "Support for using transmission curves is to be added in DM-13668.", 

232 dtype=float, 

233 ) 

234 bandwidth = pexConfig.Field( 

235 doc="Bandwidth of the physical filter, in nm." 

236 "Required if transmission curves aren't used." 

237 "Support for using transmission curves is to be added in DM-13668.", 

238 dtype=float, 

239 ) 

240 

241 def setDefaults(self): 

242 CompareWarpAssembleCoaddConfig.setDefaults(self) 

243 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask) 

244 self.doNImage = True 

245 self.assembleStaticSkyModel.warpType = self.warpType 

246 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time 

247 self.assembleStaticSkyModel.doNImage = False 

248 self.assembleStaticSkyModel.doWrite = False 

249 self.detectPsfSources.returnOriginalFootprints = False 

250 self.detectPsfSources.thresholdPolarity = "positive" 

251 # Only use bright sources for PSF measurement 

252 self.detectPsfSources.thresholdValue = 50 

253 self.detectPsfSources.nSigmaToGrow = 2 

254 # A valid star for PSF measurement should at least fill 5x5 pixels 

255 self.detectPsfSources.minPixels = 25 

256 # Use the variance plane to calculate signal to noise 

257 self.detectPsfSources.thresholdType = "pixel_stdev" 

258 # The signal to noise limit is good enough, while the flux limit is set 

259 # in dimensionless units and may not be appropriate for all data sets. 

260 self.measurePsf.starSelector["objectSize"].doFluxLimit = False 

261 

262 

263class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask): 

264 """Assemble DCR coadded images from a set of warps. 

265 

266 Attributes 

267 ---------- 

268 bufferSize : `int` 

269 The number of pixels to grow each subregion by to allow for DCR. 

270 

271 Notes 

272 ----- 

273 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 

274 Warps (also called coadded temporary exposures), including the effects of 

275 Differential Chromatic Refraction (DCR). 

276 For full details of the mathematics and algorithm, please see 

277 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 

278 

279 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 

280 each subfilter used in the iterative calculation. 

281 It begins by dividing the bandpass-defining filter into N equal bandwidth 

282 "subfilters", and divides the flux in each pixel from an initial coadd 

283 equally into each as a "dcrModel". Because the airmass and parallactic 

284 angle of each individual exposure is known, we can calculate the shift 

285 relative to the center of the band in each subfilter due to DCR. For each 

286 exposure we apply this shift as a linear transformation to the dcrModels 

287 and stack the results to produce a DCR-matched exposure. The matched 

288 exposures are subtracted from the input exposures to produce a set of 

289 residual images, and these residuals are reverse shifted for each 

290 exposures' subfilters and stacked. The shifted and stacked residuals are 

291 added to the dcrModels to produce a new estimate of the flux in each pixel 

292 within each subfilter. The dcrModels are solved for iteratively, which 

293 continues until the solution from a new iteration improves by less than 

294 a set percentage, or a maximum number of iterations is reached. 

295 Two forms of regularization are employed to reduce unphysical results. 

296 First, the new solution is averaged with the solution from the previous 

297 iteration, which mitigates oscillating solutions where the model 

298 overshoots with alternating very high and low values. 

299 Second, a common degeneracy when the data have a limited range of airmass or 

300 parallactic angle values is for one subfilter to be fit with very low or 

301 negative values, while another subfilter is fit with very high values. This 

302 typically appears in the form of holes next to sources in one subfilter, 

303 and corresponding extended wings in another. Because each subfilter has 

304 a narrow bandwidth we assume that physical sources that are above the noise 

305 level will not vary in flux by more than a factor of `frequencyClampFactor` 

306 between subfilters, and pixels that have flux deviations larger than that 

307 factor will have the excess flux distributed evenly among all subfilters. 

308 If `splitSubfilters` is set, then each subfilter will be further sub- 

309 divided during the forward modeling step (only). This approximates using 

310 a higher number of subfilters that may be necessary for high airmass 

311 observations, but does not increase the number of free parameters in the 

312 fit. This is needed when there are high airmass observations which would 

313 otherwise have significant DCR even within a subfilter. Because calculating 

314 the shifted images takes most of the time, splitting the subfilters is 

315 turned off by way of the `splitThreshold` option for low-airmass 

316 observations that do not suffer from DCR within a subfilter. 

317 """ 

318 

319 ConfigClass = DcrAssembleCoaddConfig 

320 _DefaultName = "dcrAssembleCoadd" 

321 

322 def __init__(self, *args, **kwargs): 

323 super().__init__(*args, **kwargs) 

324 if self.config.doCalculatePsf: 

325 self.schema = afwTable.SourceTable.makeMinimalSchema() 

326 self.makeSubtask("detectPsfSources", schema=self.schema) 

327 self.makeSubtask("measurePsfSources", schema=self.schema) 

328 self.makeSubtask("measurePsf", schema=self.schema) 

329 

330 @utils.inheritDoc(pipeBase.PipelineTask) 

331 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

332 # Docstring to be formatted with info from PipelineTask.runQuantum 

333 """ 

334 Notes 

335 ----- 

336 Assemble a coadd from a set of Warps. 

337 

338 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

339 Analogous to `runDataRef`, it prepares all the data products to be 

340 passed to `run`, and processes the results before returning a struct 

341 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

342 Therefore, its inputs are accessed subregion by subregion 

343 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

344 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

345 correspond to an update in `runDataRef` while both entry points 

346 are used. 

347 """ 

348 inputData = butlerQC.get(inputRefs) 

349 

350 # Construct skyInfo expected by run 

351 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

352 skyMap = inputData["skyMap"] 

353 outputDataId = butlerQC.quantum.dataId 

354 

355 inputData['skyInfo'] = makeSkyInfo(skyMap, 

356 tractId=outputDataId['tract'], 

357 patchId=outputDataId['patch']) 

358 

359 # Construct list of input Deferred Datasets 

360 # These quack a bit like like Gen2 DataRefs 

361 warpRefList = inputData['inputWarps'] 

362 # Perform same middle steps as `runDataRef` does 

363 inputs = self.prepareInputs(warpRefList) 

364 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

365 self.getTempExpDatasetName(self.warpType)) 

366 if len(inputs.tempExpRefList) == 0: 

367 self.log.warning("No coadd temporary exposures found") 

368 return 

369 

370 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

371 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

372 inputs.weightList, supplementaryData=supplementaryData) 

373 

374 inputData.setdefault('brightObjectMask', None) 

375 for subfilter in range(self.config.dcrNumSubfilters): 

376 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

377 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf()) 

378 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId) 

379 

380 if self.config.doWrite: 

381 butlerQC.put(retStruct, outputRefs) 

382 return retStruct 

383 

384 @pipeBase.timeMethod 

385 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

386 """Assemble a coadd from a set of warps. 

387 

388 Coadd a set of Warps. Compute weights to be applied to each Warp and 

389 find scalings to match the photometric zeropoint to a reference Warp. 

390 Assemble the Warps using run method. 

391 Forward model chromatic effects across multiple subfilters, 

392 and subtract from the input Warps to build sets of residuals. 

393 Use the residuals to construct a new ``DcrModel`` for each subfilter, 

394 and iterate until the model converges. 

395 Interpolate over NaNs and optionally write the coadd to disk. 

396 Return the coadded exposure. 

397 

398 Parameters 

399 ---------- 

400 dataRef : `lsst.daf.persistence.ButlerDataRef` 

401 Data reference defining the patch for coaddition and the 

402 reference Warp 

403 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 

404 List of data references to warps. Data to be coadded will be 

405 selected from this list based on overlap with the patch defined by 

406 the data reference. 

407 

408 Returns 

409 ------- 

410 results : `lsst.pipe.base.Struct` 

411 The Struct contains the following fields: 

412 

413 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

414 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

415 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

416 - ``dcrNImages``: `list` of exposure count images for each subfilter 

417 """ 

418 if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList): 

419 raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList") 

420 

421 skyInfo = self.getSkyInfo(dataRef) 

422 if warpRefList is None: 

423 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

424 if len(calExpRefList) == 0: 

425 self.log.warning("No exposures to coadd") 

426 return 

427 self.log.info("Coadding %d exposures", len(calExpRefList)) 

428 

429 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

430 

431 inputData = self.prepareInputs(warpRefList) 

432 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

433 self.getTempExpDatasetName(self.warpType)) 

434 if len(inputData.tempExpRefList) == 0: 

435 self.log.warning("No coadd temporary exposures found") 

436 return 

437 

438 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

439 

440 results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

441 inputData.weightList, supplementaryData=supplementaryData) 

442 if results is None: 

443 self.log.warning("Could not construct DcrModel for patch %s: no data to coadd.", 

444 skyInfo.patchInfo.getIndex()) 

445 return 

446 

447 if self.config.doCalculatePsf: 

448 self.measureCoaddPsf(results.coaddExposure) 

449 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

450 for subfilter in range(self.config.dcrNumSubfilters): 

451 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

452 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf()) 

453 self.processResults(results.dcrCoadds[subfilter], 

454 brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

455 if self.config.doWrite: 

456 self.log.info("Persisting dcrCoadd") 

457 dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter, 

458 numSubfilters=self.config.dcrNumSubfilters) 

459 if self.config.doNImage and results.dcrNImages is not None: 

460 dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter, 

461 numSubfilters=self.config.dcrNumSubfilters) 

462 

463 return results 

464 

465 @utils.inheritDoc(AssembleCoaddTask) 

466 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

467 """Load the previously-generated template coadd. 

468 

469 This can be removed entirely once we no longer support the Gen 2 butler. 

470 

471 Returns 

472 ------- 

473 templateCoadd : `lsst.pipe.base.Struct` 

474 Result struct with components: 

475 

476 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`) 

477 """ 

478 templateCoadd = butlerQC.get(inputRefs.templateExposure) 

479 

480 return pipeBase.Struct(templateCoadd=templateCoadd) 

481 

482 def measureCoaddPsf(self, coaddExposure): 

483 """Detect sources on the coadd exposure and measure the final PSF. 

484 

485 Parameters 

486 ---------- 

487 coaddExposure : `lsst.afw.image.Exposure` 

488 The final coadded exposure. 

489 """ 

490 table = afwTable.SourceTable.make(self.schema) 

491 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False) 

492 coaddSources = detResults.sources 

493 self.measurePsfSources.run( 

494 measCat=coaddSources, 

495 exposure=coaddExposure 

496 ) 

497 # Measure the PSF on the stacked subfilter coadds if possible. 

498 # We should already have a decent estimate of the coadd PSF, however, 

499 # so in case of any errors simply log them as a warning and use the 

500 # default PSF. 

501 try: 

502 psfResults = self.measurePsf.run(coaddExposure, coaddSources) 

503 except Exception as e: 

504 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e) 

505 else: 

506 coaddExposure.setPsf(psfResults.psf) 

507 

508 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList): 

509 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 

510 

511 Sets the property ``bufferSize``. 

512 

513 Parameters 

514 ---------- 

515 templateCoadd : `lsst.afw.image.ExposureF` 

516 The initial coadd exposure before accounting for DCR. 

517 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

518 `lsst.daf.persistence.ButlerDataRef` 

519 The data references to the input warped exposures. 

520 weightList : `list` of `float` 

521 The weight to give each input exposure in the coadd 

522 Will be modified in place if ``doAirmassWeight`` is set. 

523 

524 Returns 

525 ------- 

526 dcrModels : `lsst.pipe.tasks.DcrModel` 

527 Best fit model of the true sky after correcting chromatic effects. 

528 

529 Raises 

530 ------ 

531 NotImplementedError 

532 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 

533 """ 

534 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

535 filterLabel = templateCoadd.getFilterLabel() 

536 tempExpName = self.getTempExpDatasetName(self.warpType) 

537 dcrShifts = [] 

538 airmassDict = {} 

539 angleDict = {} 

540 psfSizeDict = {} 

541 for visitNum, warpExpRef in enumerate(warpRefList): 

542 if isinstance(warpExpRef, DeferredDatasetHandle): 

543 # Gen 3 API 

544 visitInfo = warpExpRef.get(component="visitInfo") 

545 psf = warpExpRef.get(component="psf") 

546 else: 

547 # Gen 2 API. Delete this when Gen 2 retired 

548 visitInfo = warpExpRef.get(tempExpName + "_visitInfo") 

549 psf = warpExpRef.get(tempExpName).getPsf() 

550 visit = warpExpRef.dataId["visit"] 

551 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm 

552 airmass = visitInfo.getBoresightAirmass() 

553 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees() 

554 airmassDict[visit] = airmass 

555 angleDict[visit] = parallacticAngle 

556 psfSizeDict[visit] = psfSize 

557 if self.config.doAirmassWeight: 

558 weightList[visitNum] *= airmass 

559 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(), 

560 self.config.effectiveWavelength, 

561 self.config.bandwidth, 

562 self.config.dcrNumSubfilters)))) 

563 self.log.info("Selected airmasses:\n%s", airmassDict) 

564 self.log.info("Selected parallactic angles:\n%s", angleDict) 

565 self.log.info("Selected PSF sizes:\n%s", psfSizeDict) 

566 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1) 

567 try: 

568 psf = self.selectCoaddPsf(templateCoadd, warpRefList) 

569 except Exception as e: 

570 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e) 

571 else: 

572 psf = templateCoadd.getPsf() 

573 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage, 

574 self.config.dcrNumSubfilters, 

575 effectiveWavelength=self.config.effectiveWavelength, 

576 bandwidth=self.config.bandwidth, 

577 filterLabel=filterLabel, 

578 psf=psf) 

579 return dcrModels 

580 

581 @pipeBase.timeMethod 

582 def run(self, skyInfo, warpRefList, imageScalerList, weightList, 

583 supplementaryData=None): 

584 """Assemble the coadd. 

585 

586 Requires additional inputs Struct ``supplementaryData`` to contain a 

587 ``templateCoadd`` that serves as the model of the static sky. 

588 

589 Find artifacts and apply them to the warps' masks creating a list of 

590 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 

591 Then pass these alternative masks to the base class's assemble method. 

592 

593 Divide the ``templateCoadd`` evenly between each subfilter of a 

594 ``DcrModel`` as the starting best estimate of the true wavelength- 

595 dependent sky. Forward model the ``DcrModel`` using the known 

596 chromatic effects in each subfilter and calculate a convergence metric 

597 based on how well the modeled template matches the input warps. If 

598 the convergence has not yet reached the desired threshold, then shift 

599 and stack the residual images to build a new ``DcrModel``. Apply 

600 conditioning to prevent oscillating solutions between iterations or 

601 between subfilters. 

602 

603 Once the ``DcrModel`` reaches convergence or the maximum number of 

604 iterations has been reached, fill the metadata for each subfilter 

605 image and make them proper ``coaddExposure``s. 

606 

607 Parameters 

608 ---------- 

609 skyInfo : `lsst.pipe.base.Struct` 

610 Patch geometry information, from getSkyInfo 

611 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

612 `lsst.daf.persistence.ButlerDataRef` 

613 The data references to the input warped exposures. 

614 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

615 The image scalars correct for the zero point of the exposures. 

616 weightList : `list` of `float` 

617 The weight to give each input exposure in the coadd 

618 supplementaryData : `lsst.pipe.base.Struct` 

619 Result struct returned by ``makeSupplementaryData`` with components: 

620 

621 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 

622 

623 Returns 

624 ------- 

625 result : `lsst.pipe.base.Struct` 

626 Result struct with components: 

627 

628 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

629 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

630 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

631 - ``dcrNImages``: `list` of exposure count images for each subfilter 

632 """ 

633 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters 

634 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3 

635 templateCoadd = supplementaryData.templateCoadd 

636 baseMask = templateCoadd.mask.clone() 

637 # The variance plane is for each subfilter 

638 # and should be proportionately lower than the full-band image 

639 baseVariance = templateCoadd.variance.clone() 

640 baseVariance /= self.config.dcrNumSubfilters 

641 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList) 

642 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask. 

643 templateCoadd.setMask(baseMask) 

644 badMaskPlanes = self.config.badMaskPlanes[:] 

645 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes`` 

646 # This is because pixels in observations that are significantly affect by DCR 

647 # are likely to have many pixels that are both "DETECTED" and "CLIPPED", 

648 # but those are necessary to constrain the DCR model. 

649 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes) 

650 

651 stats = self.prepareStats(mask=badPixelMask) 

652 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList) 

653 if self.config.doNImage: 

654 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList, 

655 spanSetMaskList, stats.ctrl) 

656 nImage = afwImage.ImageU(skyInfo.bbox) 

657 # Note that this nImage will be a factor of dcrNumSubfilters higher than 

658 # the nImage returned by assembleCoadd for most pixels. This is because each 

659 # subfilter may have a different nImage, and fractional values are not allowed. 

660 for dcrNImage in dcrNImages: 

661 nImage += dcrNImage 

662 else: 

663 dcrNImages = None 

664 

665 subregionSize = geom.Extent2I(*self.config.subregionSize) 

666 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) 

667 * ceil(skyInfo.bbox.getWidth()/subregionSize[0])) 

668 subIter = 0 

669 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

670 modelIter = 0 

671 subIter += 1 

672 self.log.info("Computing coadd over patch %s subregion %s of %s: %s", 

673 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox) 

674 dcrBBox = geom.Box2I(subBBox) 

675 dcrBBox.grow(self.bufferSize) 

676 dcrBBox.clip(dcrModels.bbox) 

677 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox) 

678 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList, 

679 imageScalerList, spanSetMaskList) 

680 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

681 warpRefList, weightList, stats.ctrl) 

682 self.log.info("Initial convergence : %s", convergenceMetric) 

683 convergenceList = [convergenceMetric] 

684 gainList = [] 

685 convergenceCheck = 1. 

686 refImage = templateCoadd.image 

687 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter): 

688 gain = self.calculateGain(convergenceList, gainList) 

689 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList, 

690 stats.ctrl, convergenceMetric, gain, 

691 modelWeights, refImage, dcrWeights) 

692 if self.config.useConvergence: 

693 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

694 warpRefList, weightList, stats.ctrl) 

695 if convergenceMetric == 0: 

696 self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is " 

697 "most likely due to there being no valid data in the region.", 

698 skyInfo.patchInfo.getIndex(), subIter) 

699 break 

700 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric 

701 if (convergenceCheck < 0) & (modelIter > minNumIter): 

702 self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum " 

703 "iterations or desired convergence improvement of %s." 

704 " Divergence: %s", 

705 skyInfo.patchInfo.getIndex(), subIter, 

706 self.config.convergenceThreshold, convergenceCheck) 

707 break 

708 convergenceList.append(convergenceMetric) 

709 if modelIter > maxNumIter: 

710 if self.config.useConvergence: 

711 self.log.warning("Coadd patch %s subregion %s reached maximum iterations " 

712 "before reaching desired convergence improvement of %s." 

713 " Final convergence improvement: %s", 

714 skyInfo.patchInfo.getIndex(), subIter, 

715 self.config.convergenceThreshold, convergenceCheck) 

716 break 

717 

718 if self.config.useConvergence: 

719 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)", 

720 modelIter, convergenceMetric, 100.*convergenceCheck, gain) 

721 modelIter += 1 

722 else: 

723 if self.config.useConvergence: 

724 self.log.info("Coadd patch %s subregion %s finished with " 

725 "convergence metric %s after %s iterations", 

726 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter) 

727 else: 

728 self.log.info("Coadd patch %s subregion %s finished after %s iterations", 

729 skyInfo.patchInfo.getIndex(), subIter, modelIter) 

730 if self.config.useConvergence and convergenceMetric > 0: 

731 self.log.info("Final convergence improvement was %.4f%% overall", 

732 100*(convergenceList[0] - convergenceMetric)/convergenceMetric) 

733 

734 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList, 

735 calibration=self.scaleZeroPoint.getPhotoCalib(), 

736 coaddInputs=templateCoadd.getInfo().getCoaddInputs(), 

737 mask=baseMask, 

738 variance=baseVariance) 

739 coaddExposure = self.stackCoadd(dcrCoadds) 

740 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

741 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages) 

742 

743 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl): 

744 """Calculate the number of exposures contributing to each subfilter. 

745 

746 Parameters 

747 ---------- 

748 dcrModels : `lsst.pipe.tasks.DcrModel` 

749 Best fit model of the true sky after correcting chromatic effects. 

750 bbox : `lsst.geom.box.Box2I` 

751 Bounding box of the patch to coadd. 

752 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

753 `lsst.daf.persistence.ButlerDataRef` 

754 The data references to the input warped exposures. 

755 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

756 Each element of the `dict` contains the new mask plane name 

757 (e.g. "CLIPPED and/or "NO_DATA") as the key, 

758 and the list of SpanSets to apply to the mask. 

759 statsCtrl : `lsst.afw.math.StatisticsControl` 

760 Statistics control object for coadd 

761 

762 Returns 

763 ------- 

764 dcrNImages : `list` of `lsst.afw.image.ImageU` 

765 List of exposure count images for each subfilter 

766 dcrWeights : `list` of `lsst.afw.image.ImageF` 

767 Per-pixel weights for each subfilter. 

768 Equal to 1/(number of unmasked images contributing to each pixel). 

769 """ 

770 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

771 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

772 tempExpName = self.getTempExpDatasetName(self.warpType) 

773 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList): 

774 if isinstance(warpExpRef, DeferredDatasetHandle): 

775 # Gen 3 API 

776 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

777 else: 

778 # Gen 2 API. Delete this when Gen 2 retired 

779 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

780 visitInfo = exposure.getInfo().getVisitInfo() 

781 wcs = exposure.getInfo().getWcs() 

782 mask = exposure.mask 

783 if altMaskSpans is not None: 

784 self.applyAltMaskPlanes(mask, altMaskSpans) 

785 weightImage = np.zeros_like(exposure.image.array) 

786 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1. 

787 # The weights must be shifted in exactly the same way as the residuals, 

788 # because they will be used as the denominator in the weighted average of residuals. 

789 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, 

790 dcrModels.effectiveWavelength, dcrModels.bandwidth) 

791 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights): 

792 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype) 

793 dcrWeight.array += shiftedWeights 

794 # Exclude any pixels that don't have at least one exposure contributing in all subfilters 

795 weightsThreshold = 1. 

796 goodPix = dcrWeights[0].array > weightsThreshold 

797 for weights in dcrWeights[1:]: 

798 goodPix = (weights.array > weightsThreshold) & goodPix 

799 for subfilter in range(self.config.dcrNumSubfilters): 

800 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix] 

801 dcrWeights[subfilter].array[~goodPix] = 0. 

802 dcrNImages[subfilter].array[~goodPix] = 0 

803 return (dcrNImages, dcrWeights) 

804 

805 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, 

806 statsCtrl, convergenceMetric, 

807 gain, modelWeights, refImage, dcrWeights): 

808 """Assemble the DCR coadd for a sub-region. 

809 

810 Build a DCR-matched template for each input exposure, then shift the 

811 residuals according to the DCR in each subfilter. 

812 Stack the shifted residuals and apply them as a correction to the 

813 solution from the previous iteration. 

814 Restrict the new model solutions from varying by more than a factor of 

815 `modelClampFactor` from the last solution, and additionally restrict the 

816 individual subfilter models from varying by more than a factor of 

817 `frequencyClampFactor` from their average. 

818 Finally, mitigate potentially oscillating solutions by averaging the new 

819 solution with the solution from the previous iteration, weighted by 

820 their convergence metric. 

821 

822 Parameters 

823 ---------- 

824 dcrModels : `lsst.pipe.tasks.DcrModel` 

825 Best fit model of the true sky after correcting chromatic effects. 

826 subExposures : `dict` of `lsst.afw.image.ExposureF` 

827 The pre-loaded exposures for the current subregion. 

828 bbox : `lsst.geom.box.Box2I` 

829 Bounding box of the subregion to coadd. 

830 dcrBBox : `lsst.geom.box.Box2I` 

831 Sub-region of the coadd which includes a buffer to allow for DCR. 

832 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

833 `lsst.daf.persistence.ButlerDataRef` 

834 The data references to the input warped exposures. 

835 statsCtrl : `lsst.afw.math.StatisticsControl` 

836 Statistics control object for coadd 

837 convergenceMetric : `float` 

838 Quality of fit metric for the matched templates of the input images. 

839 gain : `float`, optional 

840 Relative weight to give the new solution when updating the model. 

841 modelWeights : `numpy.ndarray` or `float` 

842 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

843 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

844 refImage : `lsst.afw.image.Image` 

845 A reference image used to supply the default pixel values. 

846 dcrWeights : `list` of `lsst.afw.image.Image` 

847 Per-pixel weights for each subfilter. 

848 Equal to 1/(number of unmasked images contributing to each pixel). 

849 """ 

850 residualGeneratorList = [] 

851 

852 for warpExpRef in warpRefList: 

853 visit = warpExpRef.dataId["visit"] 

854 exposure = subExposures[visit] 

855 visitInfo = exposure.getInfo().getVisitInfo() 

856 wcs = exposure.getInfo().getWcs() 

857 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

858 order=self.config.imageInterpOrder, 

859 splitSubfilters=self.config.splitSubfilters, 

860 splitThreshold=self.config.splitThreshold, 

861 amplifyModel=self.config.accelerateModel) 

862 residual = exposure.image.array - templateImage.array 

863 # Note that the variance plane here is used to store weights, not the actual variance 

864 residual *= exposure.variance.array 

865 # The residuals are stored as a list of generators. 

866 # This allows the residual for a given subfilter and exposure to be created 

867 # on the fly, instead of needing to store them all in memory. 

868 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, 

869 dcrModels.effectiveWavelength, 

870 dcrModels.bandwidth)) 

871 

872 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

873 gain=gain, 

874 modelWeights=modelWeights, 

875 refImage=refImage, 

876 dcrWeights=dcrWeights) 

877 dcrModels.assign(dcrSubModelOut, bbox) 

878 

879 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth): 

880 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 

881 

882 Parameters 

883 ---------- 

884 residual : `numpy.ndarray` 

885 The residual masked image for one exposure, 

886 after subtracting the matched template 

887 visitInfo : `lsst.afw.image.VisitInfo` 

888 Metadata for the exposure. 

889 wcs : `lsst.afw.geom.SkyWcs` 

890 Coordinate system definition (wcs) for the exposure. 

891 

892 Yields 

893 ------ 

894 residualImage : `numpy.ndarray` 

895 The residual image for the next subfilter, shifted for DCR. 

896 """ 

897 # Pre-calculate the spline-filtered residual image, so that step can be 

898 # skipped in the shift calculation in `applyDcr`. 

899 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder) 

900 # Note that `splitSubfilters` is always turned off in the reverse direction. 

901 # This option introduces additional blurring if applied to the residuals. 

902 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters, 

903 splitSubfilters=False) 

904 for dcr in dcrShift: 

905 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False, 

906 doPrefilter=False, order=self.config.imageInterpOrder) 

907 

908 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

909 gain, modelWeights, refImage, dcrWeights): 

910 """Calculate a new DcrModel from a set of image residuals. 

911 

912 Parameters 

913 ---------- 

914 dcrModels : `lsst.pipe.tasks.DcrModel` 

915 Current model of the true sky after correcting chromatic effects. 

916 residualGeneratorList : `generator` of `numpy.ndarray` 

917 The residual image for the next subfilter, shifted for DCR. 

918 dcrBBox : `lsst.geom.box.Box2I` 

919 Sub-region of the coadd which includes a buffer to allow for DCR. 

920 statsCtrl : `lsst.afw.math.StatisticsControl` 

921 Statistics control object for coadd 

922 gain : `float` 

923 Relative weight to give the new solution when updating the model. 

924 modelWeights : `numpy.ndarray` or `float` 

925 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

926 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

927 refImage : `lsst.afw.image.Image` 

928 A reference image used to supply the default pixel values. 

929 dcrWeights : `list` of `lsst.afw.image.Image` 

930 Per-pixel weights for each subfilter. 

931 Equal to 1/(number of unmasked images contributing to each pixel). 

932 

933 Returns 

934 ------- 

935 dcrModel : `lsst.pipe.tasks.DcrModel` 

936 New model of the true sky after correcting chromatic effects. 

937 """ 

938 newModelImages = [] 

939 for subfilter, model in enumerate(dcrModels): 

940 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList] 

941 residual = np.sum(residualsList, axis=0) 

942 residual *= dcrWeights[subfilter][dcrBBox].array 

943 # `MaskedImage`s only support in-place addition, so rename for readability 

944 newModel = model[dcrBBox].clone() 

945 newModel.array += residual 

946 # Catch any invalid values 

947 badPixels = ~np.isfinite(newModel.array) 

948 newModel.array[badPixels] = model[dcrBBox].array[badPixels] 

949 if self.config.regularizeModelIterations > 0: 

950 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox, 

951 self.config.regularizeModelIterations, 

952 self.config.regularizationWidth) 

953 newModelImages.append(newModel) 

954 if self.config.regularizeModelFrequency > 0: 

955 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl, 

956 self.config.regularizeModelFrequency, 

957 self.config.regularizationWidth) 

958 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain) 

959 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights) 

960 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength, 

961 dcrModels.bandwidth, dcrModels.psf, 

962 dcrModels.mask, dcrModels.variance) 

963 

964 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl): 

965 """Calculate a quality of fit metric for the matched templates. 

966 

967 Parameters 

968 ---------- 

969 dcrModels : `lsst.pipe.tasks.DcrModel` 

970 Best fit model of the true sky after correcting chromatic effects. 

971 subExposures : `dict` of `lsst.afw.image.ExposureF` 

972 The pre-loaded exposures for the current subregion. 

973 bbox : `lsst.geom.box.Box2I` 

974 Sub-region to coadd 

975 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

976 `lsst.daf.persistence.ButlerDataRef` 

977 The data references to the input warped exposures. 

978 weightList : `list` of `float` 

979 The weight to give each input exposure in the coadd 

980 statsCtrl : `lsst.afw.math.StatisticsControl` 

981 Statistics control object for coadd 

982 

983 Returns 

984 ------- 

985 convergenceMetric : `float` 

986 Quality of fit metric for all input exposures, within the sub-region 

987 """ 

988 significanceImage = np.abs(dcrModels.getReferenceImage(bbox)) 

989 nSigma = 3. 

990 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl, 

991 bufferSize=self.bufferSize) 

992 if np.max(significanceImage) == 0: 

993 significanceImage += 1. 

994 weight = 0 

995 metric = 0. 

996 metricList = {} 

997 for warpExpRef, expWeight in zip(warpRefList, weightList): 

998 visit = warpExpRef.dataId["visit"] 

999 exposure = subExposures[visit][bbox] 

1000 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl) 

1001 metric += singleMetric 

1002 metricList[visit] = singleMetric 

1003 weight += 1. 

1004 self.log.info("Individual metrics:\n%s", metricList) 

1005 return 1.0 if weight == 0.0 else metric/weight 

1006 

1007 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl): 

1008 """Calculate a quality of fit metric for a single matched template. 

1009 

1010 Parameters 

1011 ---------- 

1012 dcrModels : `lsst.pipe.tasks.DcrModel` 

1013 Best fit model of the true sky after correcting chromatic effects. 

1014 exposure : `lsst.afw.image.ExposureF` 

1015 The input warped exposure to evaluate. 

1016 significanceImage : `numpy.ndarray` 

1017 Array of weights for each pixel corresponding to its significance 

1018 for the convergence calculation. 

1019 statsCtrl : `lsst.afw.math.StatisticsControl` 

1020 Statistics control object for coadd 

1021 

1022 Returns 

1023 ------- 

1024 convergenceMetric : `float` 

1025 Quality of fit metric for one exposure, within the sub-region. 

1026 """ 

1027 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1028 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

1029 order=self.config.imageInterpOrder, 

1030 splitSubfilters=self.config.splitSubfilters, 

1031 splitThreshold=self.config.splitThreshold, 

1032 amplifyModel=self.config.accelerateModel) 

1033 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage 

1034 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2. 

1035 

1036 finitePixels = np.isfinite(diffVals) 

1037 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0 

1038 convergeMaskPixels = exposure.mask.array & convergeMask > 0 

1039 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels 

1040 if np.sum(usePixels) == 0: 

1041 metric = 0. 

1042 else: 

1043 diffUse = diffVals[usePixels] 

1044 refUse = refVals[usePixels] 

1045 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse)) 

1046 return metric 

1047 

1048 def stackCoadd(self, dcrCoadds): 

1049 """Add a list of sub-band coadds together. 

1050 

1051 Parameters 

1052 ---------- 

1053 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1054 A list of coadd exposures, each exposure containing 

1055 the model for one subfilter. 

1056 

1057 Returns 

1058 ------- 

1059 coaddExposure : `lsst.afw.image.ExposureF` 

1060 A single coadd exposure that is the sum of the sub-bands. 

1061 """ 

1062 coaddExposure = dcrCoadds[0].clone() 

1063 for coadd in dcrCoadds[1:]: 

1064 coaddExposure.maskedImage += coadd.maskedImage 

1065 return coaddExposure 

1066 

1067 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, 

1068 mask=None, variance=None): 

1069 """Create a list of coadd exposures from a list of masked images. 

1070 

1071 Parameters 

1072 ---------- 

1073 dcrModels : `lsst.pipe.tasks.DcrModel` 

1074 Best fit model of the true sky after correcting chromatic effects. 

1075 skyInfo : `lsst.pipe.base.Struct` 

1076 Patch geometry information, from getSkyInfo 

1077 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1078 `lsst.daf.persistence.ButlerDataRef` 

1079 The data references to the input warped exposures. 

1080 weightList : `list` of `float` 

1081 The weight to give each input exposure in the coadd 

1082 calibration : `lsst.afw.Image.PhotoCalib`, optional 

1083 Scale factor to set the photometric calibration of an exposure. 

1084 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 

1085 A record of the observations that are included in the coadd. 

1086 mask : `lsst.afw.image.Mask`, optional 

1087 Optional mask to override the values in the final coadd. 

1088 variance : `lsst.afw.image.Image`, optional 

1089 Optional variance plane to override the values in the final coadd. 

1090 

1091 Returns 

1092 ------- 

1093 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1094 A list of coadd exposures, each exposure containing 

1095 the model for one subfilter. 

1096 """ 

1097 dcrCoadds = [] 

1098 refModel = dcrModels.getReferenceImage() 

1099 for model in dcrModels: 

1100 if self.config.accelerateModel > 1: 

1101 model.array = (model.array - refModel)*self.config.accelerateModel + refModel 

1102 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

1103 if calibration is not None: 

1104 coaddExposure.setPhotoCalib(calibration) 

1105 if coaddInputs is not None: 

1106 coaddExposure.getInfo().setCoaddInputs(coaddInputs) 

1107 # Set the metadata for the coadd, including PSF and aperture corrections. 

1108 self.assembleMetadata(coaddExposure, warpRefList, weightList) 

1109 # Overwrite the PSF 

1110 coaddExposure.setPsf(dcrModels.psf) 

1111 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox]) 

1112 maskedImage = afwImage.MaskedImageF(dcrModels.bbox) 

1113 maskedImage.image = model 

1114 maskedImage.mask = dcrModels.mask 

1115 maskedImage.variance = dcrModels.variance 

1116 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox]) 

1117 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

1118 if mask is not None: 

1119 coaddExposure.setMask(mask) 

1120 if variance is not None: 

1121 coaddExposure.setVariance(variance) 

1122 dcrCoadds.append(coaddExposure) 

1123 return dcrCoadds 

1124 

1125 def calculateGain(self, convergenceList, gainList): 

1126 """Calculate the gain to use for the current iteration. 

1127 

1128 After calculating a new DcrModel, each value is averaged with the 

1129 value in the corresponding pixel from the previous iteration. This 

1130 reduces oscillating solutions that iterative techniques are plagued by, 

1131 and speeds convergence. By far the biggest changes to the model 

1132 happen in the first couple iterations, so we can also use a more 

1133 aggressive gain later when the model is changing slowly. 

1134 

1135 Parameters 

1136 ---------- 

1137 convergenceList : `list` of `float` 

1138 The quality of fit metric from each previous iteration. 

1139 gainList : `list` of `float` 

1140 The gains used in each previous iteration: appended with the new 

1141 gain value. 

1142 Gains are numbers between ``self.config.baseGain`` and 1. 

1143 

1144 Returns 

1145 ------- 

1146 gain : `float` 

1147 Relative weight to give the new solution when updating the model. 

1148 A value of 1.0 gives equal weight to both solutions. 

1149 

1150 Raises 

1151 ------ 

1152 ValueError 

1153 If ``len(convergenceList) != len(gainList)+1``. 

1154 """ 

1155 nIter = len(convergenceList) 

1156 if nIter != len(gainList) + 1: 

1157 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)." 

1158 % (len(convergenceList), len(gainList))) 

1159 

1160 if self.config.baseGain is None: 

1161 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters 

1162 # The more subfilters being modeled, the lower the gain should be. 

1163 baseGain = 1./(self.config.dcrNumSubfilters - 1) 

1164 else: 

1165 baseGain = self.config.baseGain 

1166 

1167 if self.config.useProgressiveGain and nIter > 2: 

1168 # To calculate the best gain to use, compare the past gains that have been used 

1169 # with the resulting convergences to estimate the best gain to use. 

1170 # Algorithmically, this is a Kalman filter. 

1171 # If forward modeling proceeds perfectly, the convergence metric should 

1172 # asymptotically approach a final value. 

1173 # We can estimate that value from the measured changes in convergence 

1174 # weighted by the gains used in each previous iteration. 

1175 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i] 

1176 for i in range(nIter - 1)] 

1177 # The convergence metric is strictly positive, so if the estimated final convergence is 

1178 # less than zero, force it to zero. 

1179 estFinalConv = np.array(estFinalConv) 

1180 estFinalConv[estFinalConv < 0] = 0 

1181 # Because the estimate may slowly change over time, only use the most recent measurements. 

1182 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):]) 

1183 lastGain = gainList[-1] 

1184 lastConv = convergenceList[-2] 

1185 newConv = convergenceList[-1] 

1186 # The predicted convergence is the value we would get if the new model calculated 

1187 # in the previous iteration was perfect. Recall that the updated model that is 

1188 # actually used is the gain-weighted average of the new and old model, 

1189 # so the convergence would be similarly weighted. 

1190 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain) 

1191 # If the measured and predicted convergence are very close, that indicates 

1192 # that our forward model is accurate and we can use a more aggressive gain 

1193 # If the measured convergence is significantly worse (or better!) than predicted, 

1194 # that indicates that the model is not converging as expected and 

1195 # we should use a more conservative gain. 

1196 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain)) 

1197 newGain = 1 - abs(delta) 

1198 # Average the gains to prevent oscillating solutions. 

1199 newGain = (newGain + lastGain)/2. 

1200 gain = max(baseGain, newGain) 

1201 else: 

1202 gain = baseGain 

1203 gainList.append(gain) 

1204 return gain 

1205 

1206 def calculateModelWeights(self, dcrModels, dcrBBox): 

1207 """Build an array that smoothly tapers to 0 away from detected sources. 

1208 

1209 Parameters 

1210 ---------- 

1211 dcrModels : `lsst.pipe.tasks.DcrModel` 

1212 Best fit model of the true sky after correcting chromatic effects. 

1213 dcrBBox : `lsst.geom.box.Box2I` 

1214 Sub-region of the coadd which includes a buffer to allow for DCR. 

1215 

1216 Returns 

1217 ------- 

1218 weights : `numpy.ndarray` or `float` 

1219 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1220 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1221 

1222 Raises 

1223 ------ 

1224 ValueError 

1225 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 

1226 """ 

1227 if not self.config.useModelWeights: 

1228 return 1.0 

1229 if self.config.modelWeightsWidth < 0: 

1230 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set") 

1231 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1232 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0 

1233 weights = np.zeros_like(dcrModels[0][dcrBBox].array) 

1234 weights[convergeMaskPixels] = 1. 

1235 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth) 

1236 weights /= np.max(weights) 

1237 return weights 

1238 

1239 def applyModelWeights(self, modelImages, refImage, modelWeights): 

1240 """Smoothly replace model pixel values with those from a 

1241 reference at locations away from detected sources. 

1242 

1243 Parameters 

1244 ---------- 

1245 modelImages : `list` of `lsst.afw.image.Image` 

1246 The new DCR model images from the current iteration. 

1247 The values will be modified in place. 

1248 refImage : `lsst.afw.image.MaskedImage` 

1249 A reference image used to supply the default pixel values. 

1250 modelWeights : `numpy.ndarray` or `float` 

1251 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1252 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1253 """ 

1254 if self.config.useModelWeights: 

1255 for model in modelImages: 

1256 model.array *= modelWeights 

1257 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters 

1258 

1259 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList): 

1260 """Pre-load sub-regions of a list of exposures. 

1261 

1262 Parameters 

1263 ---------- 

1264 bbox : `lsst.geom.box.Box2I` 

1265 Sub-region to coadd 

1266 statsCtrl : `lsst.afw.math.StatisticsControl` 

1267 Statistics control object for coadd 

1268 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1269 `lsst.daf.persistence.ButlerDataRef` 

1270 The data references to the input warped exposures. 

1271 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

1272 The image scalars correct for the zero point of the exposures. 

1273 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

1274 Each element is dict with keys = mask plane name to add the spans to 

1275 

1276 Returns 

1277 ------- 

1278 subExposures : `dict` 

1279 The `dict` keys are the visit IDs, 

1280 and the values are `lsst.afw.image.ExposureF` 

1281 The pre-loaded exposures for the current subregion. 

1282 The variance plane contains weights, and not the variance 

1283 """ 

1284 tempExpName = self.getTempExpDatasetName(self.warpType) 

1285 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList) 

1286 subExposures = {} 

1287 for warpExpRef, imageScaler, altMaskSpans in zipIterables: 

1288 if isinstance(warpExpRef, DeferredDatasetHandle): 

1289 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

1290 else: 

1291 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

1292 visit = warpExpRef.dataId["visit"] 

1293 if altMaskSpans is not None: 

1294 self.applyAltMaskPlanes(exposure.mask, altMaskSpans) 

1295 imageScaler.scaleMaskedImage(exposure.maskedImage) 

1296 # Note that the variance plane here is used to store weights, not the actual variance 

1297 exposure.variance.array[:, :] = 0. 

1298 # Set the weight of unmasked pixels to 1. 

1299 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1. 

1300 # Set the image value of masked pixels to zero. 

1301 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual`` 

1302 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0. 

1303 subExposures[visit] = exposure 

1304 return subExposures 

1305 

1306 def selectCoaddPsf(self, templateCoadd, warpRefList): 

1307 """Compute the PSF of the coadd from the exposures with the best seeing. 

1308 

1309 Parameters 

1310 ---------- 

1311 templateCoadd : `lsst.afw.image.ExposureF` 

1312 The initial coadd exposure before accounting for DCR. 

1313 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1314 `lsst.daf.persistence.ButlerDataRef` 

1315 The data references to the input warped exposures. 

1316 

1317 Returns 

1318 ------- 

1319 psf : `lsst.meas.algorithms.CoaddPsf` 

1320 The average PSF of the input exposures with the best seeing. 

1321 """ 

1322 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

1323 tempExpName = self.getTempExpDatasetName(self.warpType) 

1324 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit 

1325 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef`` 

1326 ccds = templateCoadd.getInfo().getCoaddInputs().ccds 

1327 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm 

1328 psfSizes = np.zeros(len(ccds)) 

1329 ccdVisits = np.array(ccds["visit"]) 

1330 for warpExpRef in warpRefList: 

1331 if isinstance(warpExpRef, DeferredDatasetHandle): 

1332 # Gen 3 API 

1333 psf = warpExpRef.get(component="psf") 

1334 else: 

1335 # Gen 2 API. Delete this when Gen 2 retired 

1336 psf = warpExpRef.get(tempExpName).getPsf() 

1337 visit = warpExpRef.dataId["visit"] 

1338 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm 

1339 psfSizes[ccdVisits == visit] = psfSize 

1340 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd 

1341 # The selected PSFs are those that have a FWHM less than or equal to the smaller 

1342 # of the mean or median FWHM of the input exposures. 

1343 sizeThreshold = min(np.median(psfSizes), psfRefSize) 

1344 goodPsfs = psfSizes <= sizeThreshold 

1345 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), 

1346 self.config.coaddPsf.makeControl()) 

1347 return psf