Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23from math import ceil 

24import numpy as np 

25from scipy import ndimage 

26import lsst.geom as geom 

27import lsst.afw.image as afwImage 

28import lsst.afw.table as afwTable 

29import lsst.coadd.utils as coaddUtils 

30from lsst.daf.butler import DeferredDatasetHandle 

31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel 

32import lsst.meas.algorithms as measAlg 

33from lsst.meas.base import SingleFrameMeasurementTask 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.utils as utils 

37from .assembleCoadd import (AssembleCoaddTask, 

38 CompareWarpAssembleCoaddConfig, 

39 CompareWarpAssembleCoaddTask) 

40from .coaddBase import makeSkyInfo 

41from .measurePsf import MeasurePsfTask 

42 

43__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"] 

44 

45 

46class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("tract", "patch", "band", "skymap"), 

48 defaultTemplates={"inputCoaddName": "deep", 

49 "outputCoaddName": "dcr", 

50 "warpType": "direct", 

51 "warpTypeSuffix": "", 

52 "fakesType": ""}): 

53 inputWarps = pipeBase.connectionTypes.Input( 

54 doc=("Input list of warps to be assembled i.e. stacked." 

55 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

56 name="{inputCoaddName}Coadd_{warpType}Warp", 

57 storageClass="ExposureF", 

58 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

59 deferLoad=True, 

60 multiple=True 

61 ) 

62 skyMap = pipeBase.connectionTypes.Input( 

63 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

64 name="{inputCoaddName}Coadd_skyMap", 

65 storageClass="SkyMap", 

66 dimensions=("skymap", ), 

67 ) 

68 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

69 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

70 " BRIGHT_OBJECT."), 

71 name="brightObjectMask", 

72 storageClass="ObjectMaskCatalog", 

73 dimensions=("tract", "patch", "skymap", "band"), 

74 ) 

75 templateExposure = pipeBase.connectionTypes.Input( 

76 doc="Input coadded exposure, produced by previous call to AssembleCoadd", 

77 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}", 

78 storageClass="ExposureF", 

79 dimensions=("tract", "patch", "skymap", "band"), 

80 ) 

81 dcrCoadds = pipeBase.connectionTypes.Output( 

82 doc="Output coadded exposure, produced by stacking input warps", 

83 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

84 storageClass="ExposureF", 

85 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

86 multiple=True, 

87 ) 

88 dcrNImages = pipeBase.connectionTypes.Output( 

89 doc="Output image of number of input images per pixel", 

90 name="{outputCoaddName}Coadd_nImage", 

91 storageClass="ImageU", 

92 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

93 multiple=True, 

94 ) 

95 

96 def __init__(self, *, config=None): 

97 super().__init__(config=config) 

98 if not config.doWrite: 

99 self.outputs.remove("dcrCoadds") 

100 

101 

102class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, 

103 pipelineConnections=DcrAssembleCoaddConnections): 

104 dcrNumSubfilters = pexConfig.Field( 

105 dtype=int, 

106 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.", 

107 default=3, 

108 ) 

109 maxNumIter = pexConfig.Field( 

110 dtype=int, 

111 optional=True, 

112 doc="Maximum number of iterations of forward modeling.", 

113 default=None, 

114 ) 

115 minNumIter = pexConfig.Field( 

116 dtype=int, 

117 optional=True, 

118 doc="Minimum number of iterations of forward modeling.", 

119 default=None, 

120 ) 

121 convergenceThreshold = pexConfig.Field( 

122 dtype=float, 

123 doc="Target relative change in convergence between iterations of forward modeling.", 

124 default=0.001, 

125 ) 

126 useConvergence = pexConfig.Field( 

127 dtype=bool, 

128 doc="Use convergence test as a forward modeling end condition?" 

129 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations", 

130 default=True, 

131 ) 

132 baseGain = pexConfig.Field( 

133 dtype=float, 

134 optional=True, 

135 doc="Relative weight to give the new solution vs. the last solution when updating the model." 

136 "A value of 1.0 gives equal weight to both solutions." 

137 "Small values imply slower convergence of the solution, but can " 

138 "help prevent overshooting and failures in the fit." 

139 "If ``baseGain`` is None, a conservative gain " 

140 "will be calculated from the number of subfilters. ", 

141 default=None, 

142 ) 

143 useProgressiveGain = pexConfig.Field( 

144 dtype=bool, 

145 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 

146 "When calculating the next gain, we use up to 5 previous gains and convergence values." 

147 "Can be set to False to force the model to change at the rate of ``baseGain``. ", 

148 default=True, 

149 ) 

150 doAirmassWeight = pexConfig.Field( 

151 dtype=bool, 

152 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.", 

153 default=False, 

154 ) 

155 modelWeightsWidth = pexConfig.Field( 

156 dtype=float, 

157 doc="Width of the region around detected sources to include in the DcrModel.", 

158 default=3, 

159 ) 

160 useModelWeights = pexConfig.Field( 

161 dtype=bool, 

162 doc="Width of the region around detected sources to include in the DcrModel.", 

163 default=True, 

164 ) 

165 splitSubfilters = pexConfig.Field( 

166 dtype=bool, 

167 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter." 

168 "Instead of at the midpoint", 

169 default=True, 

170 ) 

171 splitThreshold = pexConfig.Field( 

172 dtype=float, 

173 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 

174 "Set to 0 to always split the subfilters.", 

175 default=0.1, 

176 ) 

177 regularizeModelIterations = pexConfig.Field( 

178 dtype=float, 

179 doc="Maximum relative change of the model allowed between iterations." 

180 "Set to zero to disable.", 

181 default=2., 

182 ) 

183 regularizeModelFrequency = pexConfig.Field( 

184 dtype=float, 

185 doc="Maximum relative change of the model allowed between subfilters." 

186 "Set to zero to disable.", 

187 default=4., 

188 ) 

189 convergenceMaskPlanes = pexConfig.ListField( 

190 dtype=str, 

191 default=["DETECTED"], 

192 doc="Mask planes to use to calculate convergence." 

193 ) 

194 regularizationWidth = pexConfig.Field( 

195 dtype=int, 

196 default=2, 

197 doc="Minimum radius of a region to include in regularization, in pixels." 

198 ) 

199 imageInterpOrder = pexConfig.Field( 

200 dtype=int, 

201 doc="The order of the spline interpolation used to shift the image plane.", 

202 default=3, 

203 ) 

204 accelerateModel = pexConfig.Field( 

205 dtype=float, 

206 doc="Factor to amplify the differences between model planes by to speed convergence.", 

207 default=3, 

208 ) 

209 doCalculatePsf = pexConfig.Field( 

210 dtype=bool, 

211 doc="Set to detect stars and recalculate the PSF from the final coadd." 

212 "Otherwise the PSF is estimated from a selection of the best input exposures", 

213 default=False, 

214 ) 

215 detectPsfSources = pexConfig.ConfigurableField( 

216 target=measAlg.SourceDetectionTask, 

217 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.", 

218 ) 

219 measurePsfSources = pexConfig.ConfigurableField( 

220 target=SingleFrameMeasurementTask, 

221 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 

222 ) 

223 measurePsf = pexConfig.ConfigurableField( 

224 target=MeasurePsfTask, 

225 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.", 

226 ) 

227 effectiveWavelength = pexConfig.Field( 

228 doc="Effective wavelength of the filter, in nm." 

229 "Required if transmission curves aren't used." 

230 "Support for using transmission curves is to be added in DM-13668.", 

231 dtype=float, 

232 ) 

233 bandwidth = pexConfig.Field( 

234 doc="Bandwidth of the physical filter, in nm." 

235 "Required if transmission curves aren't used." 

236 "Support for using transmission curves is to be added in DM-13668.", 

237 dtype=float, 

238 ) 

239 

240 def setDefaults(self): 

241 CompareWarpAssembleCoaddConfig.setDefaults(self) 

242 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask) 

243 self.doNImage = True 

244 self.assembleStaticSkyModel.warpType = self.warpType 

245 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time 

246 self.assembleStaticSkyModel.doNImage = False 

247 self.assembleStaticSkyModel.doWrite = False 

248 self.detectPsfSources.returnOriginalFootprints = False 

249 self.detectPsfSources.thresholdPolarity = "positive" 

250 # Only use bright sources for PSF measurement 

251 self.detectPsfSources.thresholdValue = 50 

252 self.detectPsfSources.nSigmaToGrow = 2 

253 # A valid star for PSF measurement should at least fill 5x5 pixels 

254 self.detectPsfSources.minPixels = 25 

255 # Use the variance plane to calculate signal to noise 

256 self.detectPsfSources.thresholdType = "pixel_stdev" 

257 # The signal to noise limit is good enough, while the flux limit is set 

258 # in dimensionless units and may not be appropriate for all data sets. 

259 self.measurePsf.starSelector["objectSize"].doFluxLimit = False 

260 

261 

262class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask): 

263 """Assemble DCR coadded images from a set of warps. 

264 

265 Attributes 

266 ---------- 

267 bufferSize : `int` 

268 The number of pixels to grow each subregion by to allow for DCR. 

269 

270 Notes 

271 ----- 

272 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 

273 Warps (also called coadded temporary exposures), including the effects of 

274 Differential Chromatic Refraction (DCR). 

275 For full details of the mathematics and algorithm, please see 

276 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 

277 

278 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 

279 each subfilter used in the iterative calculation. 

280 It begins by dividing the bandpass-defining filter into N equal bandwidth 

281 "subfilters", and divides the flux in each pixel from an initial coadd 

282 equally into each as a "dcrModel". Because the airmass and parallactic 

283 angle of each individual exposure is known, we can calculate the shift 

284 relative to the center of the band in each subfilter due to DCR. For each 

285 exposure we apply this shift as a linear transformation to the dcrModels 

286 and stack the results to produce a DCR-matched exposure. The matched 

287 exposures are subtracted from the input exposures to produce a set of 

288 residual images, and these residuals are reverse shifted for each 

289 exposures' subfilters and stacked. The shifted and stacked residuals are 

290 added to the dcrModels to produce a new estimate of the flux in each pixel 

291 within each subfilter. The dcrModels are solved for iteratively, which 

292 continues until the solution from a new iteration improves by less than 

293 a set percentage, or a maximum number of iterations is reached. 

294 Two forms of regularization are employed to reduce unphysical results. 

295 First, the new solution is averaged with the solution from the previous 

296 iteration, which mitigates oscillating solutions where the model 

297 overshoots with alternating very high and low values. 

298 Second, a common degeneracy when the data have a limited range of airmass or 

299 parallactic angle values is for one subfilter to be fit with very low or 

300 negative values, while another subfilter is fit with very high values. This 

301 typically appears in the form of holes next to sources in one subfilter, 

302 and corresponding extended wings in another. Because each subfilter has 

303 a narrow bandwidth we assume that physical sources that are above the noise 

304 level will not vary in flux by more than a factor of `frequencyClampFactor` 

305 between subfilters, and pixels that have flux deviations larger than that 

306 factor will have the excess flux distributed evenly among all subfilters. 

307 If `splitSubfilters` is set, then each subfilter will be further sub- 

308 divided during the forward modeling step (only). This approximates using 

309 a higher number of subfilters that may be necessary for high airmass 

310 observations, but does not increase the number of free parameters in the 

311 fit. This is needed when there are high airmass observations which would 

312 otherwise have significant DCR even within a subfilter. Because calculating 

313 the shifted images takes most of the time, splitting the subfilters is 

314 turned off by way of the `splitThreshold` option for low-airmass 

315 observations that do not suffer from DCR within a subfilter. 

316 """ 

317 

318 ConfigClass = DcrAssembleCoaddConfig 

319 _DefaultName = "dcrAssembleCoadd" 

320 

321 def __init__(self, *args, **kwargs): 

322 super().__init__(*args, **kwargs) 

323 if self.config.doCalculatePsf: 

324 self.schema = afwTable.SourceTable.makeMinimalSchema() 

325 self.makeSubtask("detectPsfSources", schema=self.schema) 

326 self.makeSubtask("measurePsfSources", schema=self.schema) 

327 self.makeSubtask("measurePsf", schema=self.schema) 

328 

329 @utils.inheritDoc(pipeBase.PipelineTask) 

330 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

331 # Docstring to be formatted with info from PipelineTask.runQuantum 

332 """ 

333 Notes 

334 ----- 

335 Assemble a coadd from a set of Warps. 

336 

337 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

338 Analogous to `runDataRef`, it prepares all the data products to be 

339 passed to `run`, and processes the results before returning a struct 

340 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

341 Therefore, its inputs are accessed subregion by subregion 

342 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

343 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

344 correspond to an update in `runDataRef` while both entry points 

345 are used. 

346 """ 

347 inputData = butlerQC.get(inputRefs) 

348 

349 # Construct skyInfo expected by run 

350 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

351 skyMap = inputData["skyMap"] 

352 outputDataId = butlerQC.quantum.dataId 

353 

354 inputData['skyInfo'] = makeSkyInfo(skyMap, 

355 tractId=outputDataId['tract'], 

356 patchId=outputDataId['patch']) 

357 

358 # Construct list of input Deferred Datasets 

359 # These quack a bit like like Gen2 DataRefs 

360 warpRefList = inputData['inputWarps'] 

361 # Perform same middle steps as `runDataRef` does 

362 inputs = self.prepareInputs(warpRefList) 

363 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

364 self.getTempExpDatasetName(self.warpType)) 

365 if len(inputs.tempExpRefList) == 0: 

366 self.log.warn("No coadd temporary exposures found") 

367 return 

368 

369 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

370 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

371 inputs.weightList, supplementaryData=supplementaryData) 

372 

373 inputData.setdefault('brightObjectMask', None) 

374 for subfilter in range(self.config.dcrNumSubfilters): 

375 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

376 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf()) 

377 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId) 

378 

379 if self.config.doWrite: 

380 butlerQC.put(retStruct, outputRefs) 

381 return retStruct 

382 

383 @pipeBase.timeMethod 

384 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

385 """Assemble a coadd from a set of warps. 

386 

387 Coadd a set of Warps. Compute weights to be applied to each Warp and 

388 find scalings to match the photometric zeropoint to a reference Warp. 

389 Assemble the Warps using run method. 

390 Forward model chromatic effects across multiple subfilters, 

391 and subtract from the input Warps to build sets of residuals. 

392 Use the residuals to construct a new ``DcrModel`` for each subfilter, 

393 and iterate until the model converges. 

394 Interpolate over NaNs and optionally write the coadd to disk. 

395 Return the coadded exposure. 

396 

397 Parameters 

398 ---------- 

399 dataRef : `lsst.daf.persistence.ButlerDataRef` 

400 Data reference defining the patch for coaddition and the 

401 reference Warp 

402 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 

403 List of data references to warps. Data to be coadded will be 

404 selected from this list based on overlap with the patch defined by 

405 the data reference. 

406 

407 Returns 

408 ------- 

409 results : `lsst.pipe.base.Struct` 

410 The Struct contains the following fields: 

411 

412 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

413 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

414 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

415 - ``dcrNImages``: `list` of exposure count images for each subfilter 

416 """ 

417 if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList): 

418 raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList") 

419 

420 skyInfo = self.getSkyInfo(dataRef) 

421 if warpRefList is None: 

422 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

423 if len(calExpRefList) == 0: 

424 self.log.warn("No exposures to coadd") 

425 return 

426 self.log.info("Coadding %d exposures", len(calExpRefList)) 

427 

428 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

429 

430 inputData = self.prepareInputs(warpRefList) 

431 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

432 self.getTempExpDatasetName(self.warpType)) 

433 if len(inputData.tempExpRefList) == 0: 

434 self.log.warn("No coadd temporary exposures found") 

435 return 

436 

437 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

438 

439 results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

440 inputData.weightList, supplementaryData=supplementaryData) 

441 if results is None: 

442 self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.", 

443 skyInfo.patchInfo.getIndex()) 

444 return 

445 

446 if self.config.doCalculatePsf: 

447 self.measureCoaddPsf(results.coaddExposure) 

448 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

449 for subfilter in range(self.config.dcrNumSubfilters): 

450 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

451 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf()) 

452 self.processResults(results.dcrCoadds[subfilter], 

453 brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

454 if self.config.doWrite: 

455 self.log.info("Persisting dcrCoadd") 

456 dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter, 

457 numSubfilters=self.config.dcrNumSubfilters) 

458 if self.config.doNImage and results.dcrNImages is not None: 

459 dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter, 

460 numSubfilters=self.config.dcrNumSubfilters) 

461 

462 return results 

463 

464 @utils.inheritDoc(AssembleCoaddTask) 

465 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

466 """Load the previously-generated template coadd. 

467 

468 This can be removed entirely once we no longer support the Gen 2 butler. 

469 

470 Returns 

471 ------- 

472 templateCoadd : `lsst.pipe.base.Struct` 

473 Result struct with components: 

474 

475 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`) 

476 """ 

477 templateCoadd = butlerQC.get(inputRefs.templateExposure) 

478 

479 return pipeBase.Struct(templateCoadd=templateCoadd) 

480 

481 def measureCoaddPsf(self, coaddExposure): 

482 """Detect sources on the coadd exposure and measure the final PSF. 

483 

484 Parameters 

485 ---------- 

486 coaddExposure : `lsst.afw.image.Exposure` 

487 The final coadded exposure. 

488 """ 

489 table = afwTable.SourceTable.make(self.schema) 

490 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False) 

491 coaddSources = detResults.sources 

492 self.measurePsfSources.run( 

493 measCat=coaddSources, 

494 exposure=coaddExposure 

495 ) 

496 # Measure the PSF on the stacked subfilter coadds if possible. 

497 # We should already have a decent estimate of the coadd PSF, however, 

498 # so in case of any errors simply log them as a warning and use the 

499 # default PSF. 

500 try: 

501 psfResults = self.measurePsf.run(coaddExposure, coaddSources) 

502 except Exception as e: 

503 self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e) 

504 else: 

505 coaddExposure.setPsf(psfResults.psf) 

506 

507 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList): 

508 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 

509 

510 Sets the property ``bufferSize``. 

511 

512 Parameters 

513 ---------- 

514 templateCoadd : `lsst.afw.image.ExposureF` 

515 The initial coadd exposure before accounting for DCR. 

516 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

517 `lsst.daf.persistence.ButlerDataRef` 

518 The data references to the input warped exposures. 

519 weightList : `list` of `float` 

520 The weight to give each input exposure in the coadd 

521 Will be modified in place if ``doAirmassWeight`` is set. 

522 

523 Returns 

524 ------- 

525 dcrModels : `lsst.pipe.tasks.DcrModel` 

526 Best fit model of the true sky after correcting chromatic effects. 

527 

528 Raises 

529 ------ 

530 NotImplementedError 

531 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 

532 """ 

533 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

534 filterInfo = templateCoadd.getFilter() 

535 tempExpName = self.getTempExpDatasetName(self.warpType) 

536 dcrShifts = [] 

537 airmassDict = {} 

538 angleDict = {} 

539 psfSizeDict = {} 

540 for visitNum, warpExpRef in enumerate(warpRefList): 

541 if isinstance(warpExpRef, DeferredDatasetHandle): 

542 # Gen 3 API 

543 visitInfo = warpExpRef.get(component="visitInfo") 

544 psf = warpExpRef.get(component="psf") 

545 else: 

546 # Gen 2 API. Delete this when Gen 2 retired 

547 visitInfo = warpExpRef.get(tempExpName + "_visitInfo") 

548 psf = warpExpRef.get(tempExpName).getPsf() 

549 visit = warpExpRef.dataId["visit"] 

550 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm 

551 airmass = visitInfo.getBoresightAirmass() 

552 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees() 

553 airmassDict[visit] = airmass 

554 angleDict[visit] = parallacticAngle 

555 psfSizeDict[visit] = psfSize 

556 if self.config.doAirmassWeight: 

557 weightList[visitNum] *= airmass 

558 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(), 

559 self.config.effectiveWavelength, 

560 self.config.bandwidth, 

561 self.config.dcrNumSubfilters)))) 

562 self.log.info("Selected airmasses:\n%s", airmassDict) 

563 self.log.info("Selected parallactic angles:\n%s", angleDict) 

564 self.log.info("Selected PSF sizes:\n%s", psfSizeDict) 

565 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1) 

566 try: 

567 psf = self.selectCoaddPsf(templateCoadd, warpRefList) 

568 except Exception as e: 

569 self.log.warn("Unable to calculate restricted PSF, using default coadd PSF: %s" % e) 

570 else: 

571 psf = templateCoadd.getPsf() 

572 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage, 

573 self.config.dcrNumSubfilters, 

574 effectiveWavelength=self.config.effectiveWavelength, 

575 bandwidth=self.config.bandwidth, 

576 filterInfo=filterInfo, 

577 psf=psf) 

578 return dcrModels 

579 

580 @pipeBase.timeMethod 

581 def run(self, skyInfo, warpRefList, imageScalerList, weightList, 

582 supplementaryData=None): 

583 """Assemble the coadd. 

584 

585 Requires additional inputs Struct ``supplementaryData`` to contain a 

586 ``templateCoadd`` that serves as the model of the static sky. 

587 

588 Find artifacts and apply them to the warps' masks creating a list of 

589 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 

590 Then pass these alternative masks to the base class's assemble method. 

591 

592 Divide the ``templateCoadd`` evenly between each subfilter of a 

593 ``DcrModel`` as the starting best estimate of the true wavelength- 

594 dependent sky. Forward model the ``DcrModel`` using the known 

595 chromatic effects in each subfilter and calculate a convergence metric 

596 based on how well the modeled template matches the input warps. If 

597 the convergence has not yet reached the desired threshold, then shift 

598 and stack the residual images to build a new ``DcrModel``. Apply 

599 conditioning to prevent oscillating solutions between iterations or 

600 between subfilters. 

601 

602 Once the ``DcrModel`` reaches convergence or the maximum number of 

603 iterations has been reached, fill the metadata for each subfilter 

604 image and make them proper ``coaddExposure``s. 

605 

606 Parameters 

607 ---------- 

608 skyInfo : `lsst.pipe.base.Struct` 

609 Patch geometry information, from getSkyInfo 

610 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

611 `lsst.daf.persistence.ButlerDataRef` 

612 The data references to the input warped exposures. 

613 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

614 The image scalars correct for the zero point of the exposures. 

615 weightList : `list` of `float` 

616 The weight to give each input exposure in the coadd 

617 supplementaryData : `lsst.pipe.base.Struct` 

618 Result struct returned by ``makeSupplementaryData`` with components: 

619 

620 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 

621 

622 Returns 

623 ------- 

624 result : `lsst.pipe.base.Struct` 

625 Result struct with components: 

626 

627 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

628 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

629 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

630 - ``dcrNImages``: `list` of exposure count images for each subfilter 

631 """ 

632 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters 

633 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3 

634 templateCoadd = supplementaryData.templateCoadd 

635 baseMask = templateCoadd.mask.clone() 

636 # The variance plane is for each subfilter 

637 # and should be proportionately lower than the full-band image 

638 baseVariance = templateCoadd.variance.clone() 

639 baseVariance /= self.config.dcrNumSubfilters 

640 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList) 

641 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask. 

642 templateCoadd.setMask(baseMask) 

643 badMaskPlanes = self.config.badMaskPlanes[:] 

644 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes`` 

645 # This is because pixels in observations that are significantly affect by DCR 

646 # are likely to have many pixels that are both "DETECTED" and "CLIPPED", 

647 # but those are necessary to constrain the DCR model. 

648 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes) 

649 

650 stats = self.prepareStats(mask=badPixelMask) 

651 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList) 

652 if self.config.doNImage: 

653 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList, 

654 spanSetMaskList, stats.ctrl) 

655 nImage = afwImage.ImageU(skyInfo.bbox) 

656 # Note that this nImage will be a factor of dcrNumSubfilters higher than 

657 # the nImage returned by assembleCoadd for most pixels. This is because each 

658 # subfilter may have a different nImage, and fractional values are not allowed. 

659 for dcrNImage in dcrNImages: 

660 nImage += dcrNImage 

661 else: 

662 dcrNImages = None 

663 

664 subregionSize = geom.Extent2I(*self.config.subregionSize) 

665 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) 

666 * ceil(skyInfo.bbox.getWidth()/subregionSize[0])) 

667 subIter = 0 

668 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

669 modelIter = 0 

670 subIter += 1 

671 self.log.info("Computing coadd over patch %s subregion %s of %s: %s", 

672 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox) 

673 dcrBBox = geom.Box2I(subBBox) 

674 dcrBBox.grow(self.bufferSize) 

675 dcrBBox.clip(dcrModels.bbox) 

676 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox) 

677 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList, 

678 imageScalerList, spanSetMaskList) 

679 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

680 warpRefList, weightList, stats.ctrl) 

681 self.log.info("Initial convergence : %s", convergenceMetric) 

682 convergenceList = [convergenceMetric] 

683 gainList = [] 

684 convergenceCheck = 1. 

685 refImage = templateCoadd.image 

686 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter): 

687 gain = self.calculateGain(convergenceList, gainList) 

688 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList, 

689 stats.ctrl, convergenceMetric, gain, 

690 modelWeights, refImage, dcrWeights) 

691 if self.config.useConvergence: 

692 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

693 warpRefList, weightList, stats.ctrl) 

694 if convergenceMetric == 0: 

695 self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is " 

696 "most likely due to there being no valid data in the region.", 

697 skyInfo.patchInfo.getIndex(), subIter) 

698 break 

699 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric 

700 if (convergenceCheck < 0) & (modelIter > minNumIter): 

701 self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum " 

702 "iterations or desired convergence improvement of %s." 

703 " Divergence: %s", 

704 skyInfo.patchInfo.getIndex(), subIter, 

705 self.config.convergenceThreshold, convergenceCheck) 

706 break 

707 convergenceList.append(convergenceMetric) 

708 if modelIter > maxNumIter: 

709 if self.config.useConvergence: 

710 self.log.warn("Coadd patch %s subregion %s reached maximum iterations " 

711 "before reaching desired convergence improvement of %s." 

712 " Final convergence improvement: %s", 

713 skyInfo.patchInfo.getIndex(), subIter, 

714 self.config.convergenceThreshold, convergenceCheck) 

715 break 

716 

717 if self.config.useConvergence: 

718 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)", 

719 modelIter, convergenceMetric, 100.*convergenceCheck, gain) 

720 modelIter += 1 

721 else: 

722 if self.config.useConvergence: 

723 self.log.info("Coadd patch %s subregion %s finished with " 

724 "convergence metric %s after %s iterations", 

725 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter) 

726 else: 

727 self.log.info("Coadd patch %s subregion %s finished after %s iterations", 

728 skyInfo.patchInfo.getIndex(), subIter, modelIter) 

729 if self.config.useConvergence and convergenceMetric > 0: 

730 self.log.info("Final convergence improvement was %.4f%% overall", 

731 100*(convergenceList[0] - convergenceMetric)/convergenceMetric) 

732 

733 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList, 

734 calibration=self.scaleZeroPoint.getPhotoCalib(), 

735 coaddInputs=templateCoadd.getInfo().getCoaddInputs(), 

736 mask=baseMask, 

737 variance=baseVariance) 

738 coaddExposure = self.stackCoadd(dcrCoadds) 

739 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

740 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages) 

741 

742 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl): 

743 """Calculate the number of exposures contributing to each subfilter. 

744 

745 Parameters 

746 ---------- 

747 dcrModels : `lsst.pipe.tasks.DcrModel` 

748 Best fit model of the true sky after correcting chromatic effects. 

749 bbox : `lsst.geom.box.Box2I` 

750 Bounding box of the patch to coadd. 

751 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

752 `lsst.daf.persistence.ButlerDataRef` 

753 The data references to the input warped exposures. 

754 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

755 Each element of the `dict` contains the new mask plane name 

756 (e.g. "CLIPPED and/or "NO_DATA") as the key, 

757 and the list of SpanSets to apply to the mask. 

758 statsCtrl : `lsst.afw.math.StatisticsControl` 

759 Statistics control object for coadd 

760 

761 Returns 

762 ------- 

763 dcrNImages : `list` of `lsst.afw.image.ImageU` 

764 List of exposure count images for each subfilter 

765 dcrWeights : `list` of `lsst.afw.image.ImageF` 

766 Per-pixel weights for each subfilter. 

767 Equal to 1/(number of unmasked images contributing to each pixel). 

768 """ 

769 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

770 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

771 tempExpName = self.getTempExpDatasetName(self.warpType) 

772 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList): 

773 if isinstance(warpExpRef, DeferredDatasetHandle): 

774 # Gen 3 API 

775 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

776 else: 

777 # Gen 2 API. Delete this when Gen 2 retired 

778 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

779 visitInfo = exposure.getInfo().getVisitInfo() 

780 wcs = exposure.getInfo().getWcs() 

781 mask = exposure.mask 

782 if altMaskSpans is not None: 

783 self.applyAltMaskPlanes(mask, altMaskSpans) 

784 weightImage = np.zeros_like(exposure.image.array) 

785 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1. 

786 # The weights must be shifted in exactly the same way as the residuals, 

787 # because they will be used as the denominator in the weighted average of residuals. 

788 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, 

789 dcrModels.effectiveWavelength, dcrModels.bandwidth) 

790 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights): 

791 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype) 

792 dcrWeight.array += shiftedWeights 

793 # Exclude any pixels that don't have at least one exposure contributing in all subfilters 

794 weightsThreshold = 1. 

795 goodPix = dcrWeights[0].array > weightsThreshold 

796 for weights in dcrWeights[1:]: 

797 goodPix = (weights.array > weightsThreshold) & goodPix 

798 for subfilter in range(self.config.dcrNumSubfilters): 

799 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix] 

800 dcrWeights[subfilter].array[~goodPix] = 0. 

801 dcrNImages[subfilter].array[~goodPix] = 0 

802 return (dcrNImages, dcrWeights) 

803 

804 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, 

805 statsCtrl, convergenceMetric, 

806 gain, modelWeights, refImage, dcrWeights): 

807 """Assemble the DCR coadd for a sub-region. 

808 

809 Build a DCR-matched template for each input exposure, then shift the 

810 residuals according to the DCR in each subfilter. 

811 Stack the shifted residuals and apply them as a correction to the 

812 solution from the previous iteration. 

813 Restrict the new model solutions from varying by more than a factor of 

814 `modelClampFactor` from the last solution, and additionally restrict the 

815 individual subfilter models from varying by more than a factor of 

816 `frequencyClampFactor` from their average. 

817 Finally, mitigate potentially oscillating solutions by averaging the new 

818 solution with the solution from the previous iteration, weighted by 

819 their convergence metric. 

820 

821 Parameters 

822 ---------- 

823 dcrModels : `lsst.pipe.tasks.DcrModel` 

824 Best fit model of the true sky after correcting chromatic effects. 

825 subExposures : `dict` of `lsst.afw.image.ExposureF` 

826 The pre-loaded exposures for the current subregion. 

827 bbox : `lsst.geom.box.Box2I` 

828 Bounding box of the subregion to coadd. 

829 dcrBBox : `lsst.geom.box.Box2I` 

830 Sub-region of the coadd which includes a buffer to allow for DCR. 

831 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

832 `lsst.daf.persistence.ButlerDataRef` 

833 The data references to the input warped exposures. 

834 statsCtrl : `lsst.afw.math.StatisticsControl` 

835 Statistics control object for coadd 

836 convergenceMetric : `float` 

837 Quality of fit metric for the matched templates of the input images. 

838 gain : `float`, optional 

839 Relative weight to give the new solution when updating the model. 

840 modelWeights : `numpy.ndarray` or `float` 

841 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

842 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

843 refImage : `lsst.afw.image.Image` 

844 A reference image used to supply the default pixel values. 

845 dcrWeights : `list` of `lsst.afw.image.Image` 

846 Per-pixel weights for each subfilter. 

847 Equal to 1/(number of unmasked images contributing to each pixel). 

848 """ 

849 residualGeneratorList = [] 

850 

851 for warpExpRef in warpRefList: 

852 visit = warpExpRef.dataId["visit"] 

853 exposure = subExposures[visit] 

854 visitInfo = exposure.getInfo().getVisitInfo() 

855 wcs = exposure.getInfo().getWcs() 

856 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

857 order=self.config.imageInterpOrder, 

858 splitSubfilters=self.config.splitSubfilters, 

859 splitThreshold=self.config.splitThreshold, 

860 amplifyModel=self.config.accelerateModel) 

861 residual = exposure.image.array - templateImage.array 

862 # Note that the variance plane here is used to store weights, not the actual variance 

863 residual *= exposure.variance.array 

864 # The residuals are stored as a list of generators. 

865 # This allows the residual for a given subfilter and exposure to be created 

866 # on the fly, instead of needing to store them all in memory. 

867 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, 

868 dcrModels.effectiveWavelength, 

869 dcrModels.bandwidth)) 

870 

871 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

872 gain=gain, 

873 modelWeights=modelWeights, 

874 refImage=refImage, 

875 dcrWeights=dcrWeights) 

876 dcrModels.assign(dcrSubModelOut, bbox) 

877 

878 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth): 

879 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 

880 

881 Parameters 

882 ---------- 

883 residual : `numpy.ndarray` 

884 The residual masked image for one exposure, 

885 after subtracting the matched template 

886 visitInfo : `lsst.afw.image.VisitInfo` 

887 Metadata for the exposure. 

888 wcs : `lsst.afw.geom.SkyWcs` 

889 Coordinate system definition (wcs) for the exposure. 

890 filterInfo : `lsst.afw.image.Filter` 

891 The filter definition, set in the current instruments' obs package. 

892 Note: this object will be changed in DM-21333. 

893 

894 Yields 

895 ------ 

896 residualImage : `numpy.ndarray` 

897 The residual image for the next subfilter, shifted for DCR. 

898 """ 

899 # Pre-calculate the spline-filtered residual image, so that step can be 

900 # skipped in the shift calculation in `applyDcr`. 

901 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder) 

902 # Note that `splitSubfilters` is always turned off in the reverse direction. 

903 # This option introduces additional blurring if applied to the residuals. 

904 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters, 

905 splitSubfilters=False) 

906 for dcr in dcrShift: 

907 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False, 

908 doPrefilter=False, order=self.config.imageInterpOrder) 

909 

910 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

911 gain, modelWeights, refImage, dcrWeights): 

912 """Calculate a new DcrModel from a set of image residuals. 

913 

914 Parameters 

915 ---------- 

916 dcrModels : `lsst.pipe.tasks.DcrModel` 

917 Current model of the true sky after correcting chromatic effects. 

918 residualGeneratorList : `generator` of `numpy.ndarray` 

919 The residual image for the next subfilter, shifted for DCR. 

920 dcrBBox : `lsst.geom.box.Box2I` 

921 Sub-region of the coadd which includes a buffer to allow for DCR. 

922 statsCtrl : `lsst.afw.math.StatisticsControl` 

923 Statistics control object for coadd 

924 gain : `float` 

925 Relative weight to give the new solution when updating the model. 

926 modelWeights : `numpy.ndarray` or `float` 

927 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

928 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

929 refImage : `lsst.afw.image.Image` 

930 A reference image used to supply the default pixel values. 

931 dcrWeights : `list` of `lsst.afw.image.Image` 

932 Per-pixel weights for each subfilter. 

933 Equal to 1/(number of unmasked images contributing to each pixel). 

934 

935 Returns 

936 ------- 

937 dcrModel : `lsst.pipe.tasks.DcrModel` 

938 New model of the true sky after correcting chromatic effects. 

939 """ 

940 newModelImages = [] 

941 for subfilter, model in enumerate(dcrModels): 

942 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList] 

943 residual = np.sum(residualsList, axis=0) 

944 residual *= dcrWeights[subfilter][dcrBBox].array 

945 # `MaskedImage`s only support in-place addition, so rename for readability 

946 newModel = model[dcrBBox].clone() 

947 newModel.array += residual 

948 # Catch any invalid values 

949 badPixels = ~np.isfinite(newModel.array) 

950 newModel.array[badPixels] = model[dcrBBox].array[badPixels] 

951 if self.config.regularizeModelIterations > 0: 

952 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox, 

953 self.config.regularizeModelIterations, 

954 self.config.regularizationWidth) 

955 newModelImages.append(newModel) 

956 if self.config.regularizeModelFrequency > 0: 

957 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl, 

958 self.config.regularizeModelFrequency, 

959 self.config.regularizationWidth) 

960 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain) 

961 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights) 

962 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength, 

963 dcrModels.bandwidth, dcrModels.psf, 

964 dcrModels.mask, dcrModels.variance) 

965 

966 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl): 

967 """Calculate a quality of fit metric for the matched templates. 

968 

969 Parameters 

970 ---------- 

971 dcrModels : `lsst.pipe.tasks.DcrModel` 

972 Best fit model of the true sky after correcting chromatic effects. 

973 subExposures : `dict` of `lsst.afw.image.ExposureF` 

974 The pre-loaded exposures for the current subregion. 

975 bbox : `lsst.geom.box.Box2I` 

976 Sub-region to coadd 

977 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

978 `lsst.daf.persistence.ButlerDataRef` 

979 The data references to the input warped exposures. 

980 weightList : `list` of `float` 

981 The weight to give each input exposure in the coadd 

982 statsCtrl : `lsst.afw.math.StatisticsControl` 

983 Statistics control object for coadd 

984 

985 Returns 

986 ------- 

987 convergenceMetric : `float` 

988 Quality of fit metric for all input exposures, within the sub-region 

989 """ 

990 significanceImage = np.abs(dcrModels.getReferenceImage(bbox)) 

991 nSigma = 3. 

992 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl, 

993 bufferSize=self.bufferSize) 

994 if np.max(significanceImage) == 0: 

995 significanceImage += 1. 

996 weight = 0 

997 metric = 0. 

998 metricList = {} 

999 for warpExpRef, expWeight in zip(warpRefList, weightList): 

1000 visit = warpExpRef.dataId["visit"] 

1001 exposure = subExposures[visit][bbox] 

1002 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl) 

1003 metric += singleMetric 

1004 metricList[visit] = singleMetric 

1005 weight += 1. 

1006 self.log.info("Individual metrics:\n%s", metricList) 

1007 return 1.0 if weight == 0.0 else metric/weight 

1008 

1009 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl): 

1010 """Calculate a quality of fit metric for a single matched template. 

1011 

1012 Parameters 

1013 ---------- 

1014 dcrModels : `lsst.pipe.tasks.DcrModel` 

1015 Best fit model of the true sky after correcting chromatic effects. 

1016 exposure : `lsst.afw.image.ExposureF` 

1017 The input warped exposure to evaluate. 

1018 significanceImage : `numpy.ndarray` 

1019 Array of weights for each pixel corresponding to its significance 

1020 for the convergence calculation. 

1021 statsCtrl : `lsst.afw.math.StatisticsControl` 

1022 Statistics control object for coadd 

1023 

1024 Returns 

1025 ------- 

1026 convergenceMetric : `float` 

1027 Quality of fit metric for one exposure, within the sub-region. 

1028 """ 

1029 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1030 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

1031 order=self.config.imageInterpOrder, 

1032 splitSubfilters=self.config.splitSubfilters, 

1033 splitThreshold=self.config.splitThreshold, 

1034 amplifyModel=self.config.accelerateModel) 

1035 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage 

1036 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2. 

1037 

1038 finitePixels = np.isfinite(diffVals) 

1039 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0 

1040 convergeMaskPixels = exposure.mask.array & convergeMask > 0 

1041 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels 

1042 if np.sum(usePixels) == 0: 

1043 metric = 0. 

1044 else: 

1045 diffUse = diffVals[usePixels] 

1046 refUse = refVals[usePixels] 

1047 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse)) 

1048 return metric 

1049 

1050 def stackCoadd(self, dcrCoadds): 

1051 """Add a list of sub-band coadds together. 

1052 

1053 Parameters 

1054 ---------- 

1055 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1056 A list of coadd exposures, each exposure containing 

1057 the model for one subfilter. 

1058 

1059 Returns 

1060 ------- 

1061 coaddExposure : `lsst.afw.image.ExposureF` 

1062 A single coadd exposure that is the sum of the sub-bands. 

1063 """ 

1064 coaddExposure = dcrCoadds[0].clone() 

1065 for coadd in dcrCoadds[1:]: 

1066 coaddExposure.maskedImage += coadd.maskedImage 

1067 return coaddExposure 

1068 

1069 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, 

1070 mask=None, variance=None): 

1071 """Create a list of coadd exposures from a list of masked images. 

1072 

1073 Parameters 

1074 ---------- 

1075 dcrModels : `lsst.pipe.tasks.DcrModel` 

1076 Best fit model of the true sky after correcting chromatic effects. 

1077 skyInfo : `lsst.pipe.base.Struct` 

1078 Patch geometry information, from getSkyInfo 

1079 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1080 `lsst.daf.persistence.ButlerDataRef` 

1081 The data references to the input warped exposures. 

1082 weightList : `list` of `float` 

1083 The weight to give each input exposure in the coadd 

1084 calibration : `lsst.afw.Image.PhotoCalib`, optional 

1085 Scale factor to set the photometric calibration of an exposure. 

1086 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 

1087 A record of the observations that are included in the coadd. 

1088 mask : `lsst.afw.image.Mask`, optional 

1089 Optional mask to override the values in the final coadd. 

1090 variance : `lsst.afw.image.Image`, optional 

1091 Optional variance plane to override the values in the final coadd. 

1092 

1093 Returns 

1094 ------- 

1095 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1096 A list of coadd exposures, each exposure containing 

1097 the model for one subfilter. 

1098 """ 

1099 dcrCoadds = [] 

1100 refModel = dcrModels.getReferenceImage() 

1101 for model in dcrModels: 

1102 if self.config.accelerateModel > 1: 

1103 model.array = (model.array - refModel)*self.config.accelerateModel + refModel 

1104 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

1105 if calibration is not None: 

1106 coaddExposure.setPhotoCalib(calibration) 

1107 if coaddInputs is not None: 

1108 coaddExposure.getInfo().setCoaddInputs(coaddInputs) 

1109 # Set the metadata for the coadd, including PSF and aperture corrections. 

1110 self.assembleMetadata(coaddExposure, warpRefList, weightList) 

1111 # Overwrite the PSF 

1112 coaddExposure.setPsf(dcrModels.psf) 

1113 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox]) 

1114 maskedImage = afwImage.MaskedImageF(dcrModels.bbox) 

1115 maskedImage.image = model 

1116 maskedImage.mask = dcrModels.mask 

1117 maskedImage.variance = dcrModels.variance 

1118 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox]) 

1119 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

1120 if mask is not None: 

1121 coaddExposure.setMask(mask) 

1122 if variance is not None: 

1123 coaddExposure.setVariance(variance) 

1124 dcrCoadds.append(coaddExposure) 

1125 return dcrCoadds 

1126 

1127 def calculateGain(self, convergenceList, gainList): 

1128 """Calculate the gain to use for the current iteration. 

1129 

1130 After calculating a new DcrModel, each value is averaged with the 

1131 value in the corresponding pixel from the previous iteration. This 

1132 reduces oscillating solutions that iterative techniques are plagued by, 

1133 and speeds convergence. By far the biggest changes to the model 

1134 happen in the first couple iterations, so we can also use a more 

1135 aggressive gain later when the model is changing slowly. 

1136 

1137 Parameters 

1138 ---------- 

1139 convergenceList : `list` of `float` 

1140 The quality of fit metric from each previous iteration. 

1141 gainList : `list` of `float` 

1142 The gains used in each previous iteration: appended with the new 

1143 gain value. 

1144 Gains are numbers between ``self.config.baseGain`` and 1. 

1145 

1146 Returns 

1147 ------- 

1148 gain : `float` 

1149 Relative weight to give the new solution when updating the model. 

1150 A value of 1.0 gives equal weight to both solutions. 

1151 

1152 Raises 

1153 ------ 

1154 ValueError 

1155 If ``len(convergenceList) != len(gainList)+1``. 

1156 """ 

1157 nIter = len(convergenceList) 

1158 if nIter != len(gainList) + 1: 

1159 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)." 

1160 % (len(convergenceList), len(gainList))) 

1161 

1162 if self.config.baseGain is None: 

1163 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters 

1164 # The more subfilters being modeled, the lower the gain should be. 

1165 baseGain = 1./(self.config.dcrNumSubfilters - 1) 

1166 else: 

1167 baseGain = self.config.baseGain 

1168 

1169 if self.config.useProgressiveGain and nIter > 2: 

1170 # To calculate the best gain to use, compare the past gains that have been used 

1171 # with the resulting convergences to estimate the best gain to use. 

1172 # Algorithmically, this is a Kalman filter. 

1173 # If forward modeling proceeds perfectly, the convergence metric should 

1174 # asymptotically approach a final value. 

1175 # We can estimate that value from the measured changes in convergence 

1176 # weighted by the gains used in each previous iteration. 

1177 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i] 

1178 for i in range(nIter - 1)] 

1179 # The convergence metric is strictly positive, so if the estimated final convergence is 

1180 # less than zero, force it to zero. 

1181 estFinalConv = np.array(estFinalConv) 

1182 estFinalConv[estFinalConv < 0] = 0 

1183 # Because the estimate may slowly change over time, only use the most recent measurements. 

1184 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):]) 

1185 lastGain = gainList[-1] 

1186 lastConv = convergenceList[-2] 

1187 newConv = convergenceList[-1] 

1188 # The predicted convergence is the value we would get if the new model calculated 

1189 # in the previous iteration was perfect. Recall that the updated model that is 

1190 # actually used is the gain-weighted average of the new and old model, 

1191 # so the convergence would be similarly weighted. 

1192 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain) 

1193 # If the measured and predicted convergence are very close, that indicates 

1194 # that our forward model is accurate and we can use a more aggressive gain 

1195 # If the measured convergence is significantly worse (or better!) than predicted, 

1196 # that indicates that the model is not converging as expected and 

1197 # we should use a more conservative gain. 

1198 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain)) 

1199 newGain = 1 - abs(delta) 

1200 # Average the gains to prevent oscillating solutions. 

1201 newGain = (newGain + lastGain)/2. 

1202 gain = max(baseGain, newGain) 

1203 else: 

1204 gain = baseGain 

1205 gainList.append(gain) 

1206 return gain 

1207 

1208 def calculateModelWeights(self, dcrModels, dcrBBox): 

1209 """Build an array that smoothly tapers to 0 away from detected sources. 

1210 

1211 Parameters 

1212 ---------- 

1213 dcrModels : `lsst.pipe.tasks.DcrModel` 

1214 Best fit model of the true sky after correcting chromatic effects. 

1215 dcrBBox : `lsst.geom.box.Box2I` 

1216 Sub-region of the coadd which includes a buffer to allow for DCR. 

1217 

1218 Returns 

1219 ------- 

1220 weights : `numpy.ndarray` or `float` 

1221 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1222 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1223 

1224 Raises 

1225 ------ 

1226 ValueError 

1227 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 

1228 """ 

1229 if not self.config.useModelWeights: 

1230 return 1.0 

1231 if self.config.modelWeightsWidth < 0: 

1232 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set") 

1233 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1234 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0 

1235 weights = np.zeros_like(dcrModels[0][dcrBBox].array) 

1236 weights[convergeMaskPixels] = 1. 

1237 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth) 

1238 weights /= np.max(weights) 

1239 return weights 

1240 

1241 def applyModelWeights(self, modelImages, refImage, modelWeights): 

1242 """Smoothly replace model pixel values with those from a 

1243 reference at locations away from detected sources. 

1244 

1245 Parameters 

1246 ---------- 

1247 modelImages : `list` of `lsst.afw.image.Image` 

1248 The new DCR model images from the current iteration. 

1249 The values will be modified in place. 

1250 refImage : `lsst.afw.image.MaskedImage` 

1251 A reference image used to supply the default pixel values. 

1252 modelWeights : `numpy.ndarray` or `float` 

1253 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1254 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1255 """ 

1256 if self.config.useModelWeights: 

1257 for model in modelImages: 

1258 model.array *= modelWeights 

1259 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters 

1260 

1261 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList): 

1262 """Pre-load sub-regions of a list of exposures. 

1263 

1264 Parameters 

1265 ---------- 

1266 bbox : `lsst.geom.box.Box2I` 

1267 Sub-region to coadd 

1268 statsCtrl : `lsst.afw.math.StatisticsControl` 

1269 Statistics control object for coadd 

1270 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1271 `lsst.daf.persistence.ButlerDataRef` 

1272 The data references to the input warped exposures. 

1273 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

1274 The image scalars correct for the zero point of the exposures. 

1275 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

1276 Each element is dict with keys = mask plane name to add the spans to 

1277 

1278 Returns 

1279 ------- 

1280 subExposures : `dict` 

1281 The `dict` keys are the visit IDs, 

1282 and the values are `lsst.afw.image.ExposureF` 

1283 The pre-loaded exposures for the current subregion. 

1284 The variance plane contains weights, and not the variance 

1285 """ 

1286 tempExpName = self.getTempExpDatasetName(self.warpType) 

1287 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList) 

1288 subExposures = {} 

1289 for warpExpRef, imageScaler, altMaskSpans in zipIterables: 

1290 if isinstance(warpExpRef, DeferredDatasetHandle): 

1291 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

1292 else: 

1293 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

1294 visit = warpExpRef.dataId["visit"] 

1295 if altMaskSpans is not None: 

1296 self.applyAltMaskPlanes(exposure.mask, altMaskSpans) 

1297 imageScaler.scaleMaskedImage(exposure.maskedImage) 

1298 # Note that the variance plane here is used to store weights, not the actual variance 

1299 exposure.variance.array[:, :] = 0. 

1300 # Set the weight of unmasked pixels to 1. 

1301 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1. 

1302 # Set the image value of masked pixels to zero. 

1303 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual`` 

1304 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0. 

1305 subExposures[visit] = exposure 

1306 return subExposures 

1307 

1308 def selectCoaddPsf(self, templateCoadd, warpRefList): 

1309 """Compute the PSF of the coadd from the exposures with the best seeing. 

1310 

1311 Parameters 

1312 ---------- 

1313 templateCoadd : `lsst.afw.image.ExposureF` 

1314 The initial coadd exposure before accounting for DCR. 

1315 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1316 `lsst.daf.persistence.ButlerDataRef` 

1317 The data references to the input warped exposures. 

1318 

1319 Returns 

1320 ------- 

1321 psf : `lsst.meas.algorithms.CoaddPsf` 

1322 The average PSF of the input exposures with the best seeing. 

1323 """ 

1324 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

1325 tempExpName = self.getTempExpDatasetName(self.warpType) 

1326 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit 

1327 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef`` 

1328 ccds = templateCoadd.getInfo().getCoaddInputs().ccds 

1329 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm 

1330 psfSizes = np.zeros(len(ccds)) 

1331 ccdVisits = np.array(ccds["visit"]) 

1332 for warpExpRef in warpRefList: 

1333 if isinstance(warpExpRef, DeferredDatasetHandle): 

1334 # Gen 3 API 

1335 psf = warpExpRef.get(component="psf") 

1336 else: 

1337 # Gen 2 API. Delete this when Gen 2 retired 

1338 psf = warpExpRef.get(tempExpName).getPsf() 

1339 visit = warpExpRef.dataId["visit"] 

1340 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm 

1341 psfSizes[ccdVisits == visit] = psfSize 

1342 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd 

1343 # The selected PSFs are those that have a FWHM less than or equal to the smaller 

1344 # of the mean or median FWHM of the input exposures. 

1345 sizeThreshold = min(np.median(psfSizes), psfRefSize) 

1346 goodPsfs = psfSizes <= sizeThreshold 

1347 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), 

1348 self.config.coaddPsf.makeControl()) 

1349 return psf