Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23from math import ceil 

24import numpy as np 

25from scipy import ndimage 

26import lsst.geom as geom 

27import lsst.afw.image as afwImage 

28import lsst.afw.table as afwTable 

29import lsst.coadd.utils as coaddUtils 

30from lsst.daf.butler import DeferredDatasetHandle 

31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel 

32import lsst.meas.algorithms as measAlg 

33from lsst.meas.base import SingleFrameMeasurementTask 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.utils as utils 

37from lsst.skymap import BaseSkyMap 

38from lsst.utils.timer import timeMethod 

39from .assembleCoadd import (AssembleCoaddTask, 

40 CompareWarpAssembleCoaddConfig, 

41 CompareWarpAssembleCoaddTask) 

42from .coaddBase import makeSkyInfo 

43from .measurePsf import MeasurePsfTask 

44 

45__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"] 

46 

47 

48class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

49 dimensions=("tract", "patch", "band", "skymap"), 

50 defaultTemplates={"inputCoaddName": "deep", 

51 "outputCoaddName": "dcr", 

52 "warpType": "direct", 

53 "warpTypeSuffix": "", 

54 "fakesType": ""}): 

55 inputWarps = pipeBase.connectionTypes.Input( 

56 doc=("Input list of warps to be assembled i.e. stacked." 

57 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

58 name="{inputCoaddName}Coadd_{warpType}Warp", 

59 storageClass="ExposureF", 

60 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

61 deferLoad=True, 

62 multiple=True 

63 ) 

64 skyMap = pipeBase.connectionTypes.Input( 

65 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

66 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

67 storageClass="SkyMap", 

68 dimensions=("skymap", ), 

69 ) 

70 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

71 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

72 " BRIGHT_OBJECT."), 

73 name="brightObjectMask", 

74 storageClass="ObjectMaskCatalog", 

75 dimensions=("tract", "patch", "skymap", "band"), 

76 ) 

77 templateExposure = pipeBase.connectionTypes.Input( 

78 doc="Input coadded exposure, produced by previous call to AssembleCoadd", 

79 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}", 

80 storageClass="ExposureF", 

81 dimensions=("tract", "patch", "skymap", "band"), 

82 ) 

83 dcrCoadds = pipeBase.connectionTypes.Output( 

84 doc="Output coadded exposure, produced by stacking input warps", 

85 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

86 storageClass="ExposureF", 

87 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

88 multiple=True, 

89 ) 

90 dcrNImages = pipeBase.connectionTypes.Output( 

91 doc="Output image of number of input images per pixel", 

92 name="{outputCoaddName}Coadd_nImage", 

93 storageClass="ImageU", 

94 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

95 multiple=True, 

96 ) 

97 

98 def __init__(self, *, config=None): 

99 super().__init__(config=config) 

100 if not config.doWrite: 

101 self.outputs.remove("dcrCoadds") 

102 

103 

104class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, 

105 pipelineConnections=DcrAssembleCoaddConnections): 

106 dcrNumSubfilters = pexConfig.Field( 

107 dtype=int, 

108 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.", 

109 default=3, 

110 ) 

111 maxNumIter = pexConfig.Field( 

112 dtype=int, 

113 optional=True, 

114 doc="Maximum number of iterations of forward modeling.", 

115 default=None, 

116 ) 

117 minNumIter = pexConfig.Field( 

118 dtype=int, 

119 optional=True, 

120 doc="Minimum number of iterations of forward modeling.", 

121 default=None, 

122 ) 

123 convergenceThreshold = pexConfig.Field( 

124 dtype=float, 

125 doc="Target relative change in convergence between iterations of forward modeling.", 

126 default=0.001, 

127 ) 

128 useConvergence = pexConfig.Field( 

129 dtype=bool, 

130 doc="Use convergence test as a forward modeling end condition?" 

131 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations", 

132 default=True, 

133 ) 

134 baseGain = pexConfig.Field( 

135 dtype=float, 

136 optional=True, 

137 doc="Relative weight to give the new solution vs. the last solution when updating the model." 

138 "A value of 1.0 gives equal weight to both solutions." 

139 "Small values imply slower convergence of the solution, but can " 

140 "help prevent overshooting and failures in the fit." 

141 "If ``baseGain`` is None, a conservative gain " 

142 "will be calculated from the number of subfilters. ", 

143 default=None, 

144 ) 

145 useProgressiveGain = pexConfig.Field( 

146 dtype=bool, 

147 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 

148 "When calculating the next gain, we use up to 5 previous gains and convergence values." 

149 "Can be set to False to force the model to change at the rate of ``baseGain``. ", 

150 default=True, 

151 ) 

152 doAirmassWeight = pexConfig.Field( 

153 dtype=bool, 

154 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.", 

155 default=False, 

156 ) 

157 modelWeightsWidth = pexConfig.Field( 

158 dtype=float, 

159 doc="Width of the region around detected sources to include in the DcrModel.", 

160 default=3, 

161 ) 

162 useModelWeights = pexConfig.Field( 

163 dtype=bool, 

164 doc="Width of the region around detected sources to include in the DcrModel.", 

165 default=True, 

166 ) 

167 splitSubfilters = pexConfig.Field( 

168 dtype=bool, 

169 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter." 

170 "Instead of at the midpoint", 

171 default=True, 

172 ) 

173 splitThreshold = pexConfig.Field( 

174 dtype=float, 

175 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 

176 "Set to 0 to always split the subfilters.", 

177 default=0.1, 

178 ) 

179 regularizeModelIterations = pexConfig.Field( 

180 dtype=float, 

181 doc="Maximum relative change of the model allowed between iterations." 

182 "Set to zero to disable.", 

183 default=2., 

184 ) 

185 regularizeModelFrequency = pexConfig.Field( 

186 dtype=float, 

187 doc="Maximum relative change of the model allowed between subfilters." 

188 "Set to zero to disable.", 

189 default=4., 

190 ) 

191 convergenceMaskPlanes = pexConfig.ListField( 

192 dtype=str, 

193 default=["DETECTED"], 

194 doc="Mask planes to use to calculate convergence." 

195 ) 

196 regularizationWidth = pexConfig.Field( 

197 dtype=int, 

198 default=2, 

199 doc="Minimum radius of a region to include in regularization, in pixels." 

200 ) 

201 imageInterpOrder = pexConfig.Field( 

202 dtype=int, 

203 doc="The order of the spline interpolation used to shift the image plane.", 

204 default=3, 

205 ) 

206 accelerateModel = pexConfig.Field( 

207 dtype=float, 

208 doc="Factor to amplify the differences between model planes by to speed convergence.", 

209 default=3, 

210 ) 

211 doCalculatePsf = pexConfig.Field( 

212 dtype=bool, 

213 doc="Set to detect stars and recalculate the PSF from the final coadd." 

214 "Otherwise the PSF is estimated from a selection of the best input exposures", 

215 default=False, 

216 ) 

217 detectPsfSources = pexConfig.ConfigurableField( 

218 target=measAlg.SourceDetectionTask, 

219 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.", 

220 ) 

221 measurePsfSources = pexConfig.ConfigurableField( 

222 target=SingleFrameMeasurementTask, 

223 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 

224 ) 

225 measurePsf = pexConfig.ConfigurableField( 

226 target=MeasurePsfTask, 

227 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.", 

228 ) 

229 effectiveWavelength = pexConfig.Field( 

230 doc="Effective wavelength of the filter, in nm." 

231 "Required if transmission curves aren't used." 

232 "Support for using transmission curves is to be added in DM-13668.", 

233 dtype=float, 

234 ) 

235 bandwidth = pexConfig.Field( 

236 doc="Bandwidth of the physical filter, in nm." 

237 "Required if transmission curves aren't used." 

238 "Support for using transmission curves is to be added in DM-13668.", 

239 dtype=float, 

240 ) 

241 

242 def setDefaults(self): 

243 CompareWarpAssembleCoaddConfig.setDefaults(self) 

244 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask) 

245 self.doNImage = True 

246 self.assembleStaticSkyModel.warpType = self.warpType 

247 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time 

248 self.assembleStaticSkyModel.doNImage = False 

249 self.assembleStaticSkyModel.doWrite = False 

250 self.detectPsfSources.returnOriginalFootprints = False 

251 self.detectPsfSources.thresholdPolarity = "positive" 

252 # Only use bright sources for PSF measurement 

253 self.detectPsfSources.thresholdValue = 50 

254 self.detectPsfSources.nSigmaToGrow = 2 

255 # A valid star for PSF measurement should at least fill 5x5 pixels 

256 self.detectPsfSources.minPixels = 25 

257 # Use the variance plane to calculate signal to noise 

258 self.detectPsfSources.thresholdType = "pixel_stdev" 

259 # The signal to noise limit is good enough, while the flux limit is set 

260 # in dimensionless units and may not be appropriate for all data sets. 

261 self.measurePsf.starSelector["objectSize"].doFluxLimit = False 

262 

263 

264class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask): 

265 """Assemble DCR coadded images from a set of warps. 

266 

267 Attributes 

268 ---------- 

269 bufferSize : `int` 

270 The number of pixels to grow each subregion by to allow for DCR. 

271 

272 Notes 

273 ----- 

274 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 

275 Warps (also called coadded temporary exposures), including the effects of 

276 Differential Chromatic Refraction (DCR). 

277 For full details of the mathematics and algorithm, please see 

278 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 

279 

280 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 

281 each subfilter used in the iterative calculation. 

282 It begins by dividing the bandpass-defining filter into N equal bandwidth 

283 "subfilters", and divides the flux in each pixel from an initial coadd 

284 equally into each as a "dcrModel". Because the airmass and parallactic 

285 angle of each individual exposure is known, we can calculate the shift 

286 relative to the center of the band in each subfilter due to DCR. For each 

287 exposure we apply this shift as a linear transformation to the dcrModels 

288 and stack the results to produce a DCR-matched exposure. The matched 

289 exposures are subtracted from the input exposures to produce a set of 

290 residual images, and these residuals are reverse shifted for each 

291 exposures' subfilters and stacked. The shifted and stacked residuals are 

292 added to the dcrModels to produce a new estimate of the flux in each pixel 

293 within each subfilter. The dcrModels are solved for iteratively, which 

294 continues until the solution from a new iteration improves by less than 

295 a set percentage, or a maximum number of iterations is reached. 

296 Two forms of regularization are employed to reduce unphysical results. 

297 First, the new solution is averaged with the solution from the previous 

298 iteration, which mitigates oscillating solutions where the model 

299 overshoots with alternating very high and low values. 

300 Second, a common degeneracy when the data have a limited range of airmass or 

301 parallactic angle values is for one subfilter to be fit with very low or 

302 negative values, while another subfilter is fit with very high values. This 

303 typically appears in the form of holes next to sources in one subfilter, 

304 and corresponding extended wings in another. Because each subfilter has 

305 a narrow bandwidth we assume that physical sources that are above the noise 

306 level will not vary in flux by more than a factor of `frequencyClampFactor` 

307 between subfilters, and pixels that have flux deviations larger than that 

308 factor will have the excess flux distributed evenly among all subfilters. 

309 If `splitSubfilters` is set, then each subfilter will be further sub- 

310 divided during the forward modeling step (only). This approximates using 

311 a higher number of subfilters that may be necessary for high airmass 

312 observations, but does not increase the number of free parameters in the 

313 fit. This is needed when there are high airmass observations which would 

314 otherwise have significant DCR even within a subfilter. Because calculating 

315 the shifted images takes most of the time, splitting the subfilters is 

316 turned off by way of the `splitThreshold` option for low-airmass 

317 observations that do not suffer from DCR within a subfilter. 

318 """ 

319 

320 ConfigClass = DcrAssembleCoaddConfig 

321 _DefaultName = "dcrAssembleCoadd" 

322 

323 def __init__(self, *args, **kwargs): 

324 super().__init__(*args, **kwargs) 

325 if self.config.doCalculatePsf: 

326 self.schema = afwTable.SourceTable.makeMinimalSchema() 

327 self.makeSubtask("detectPsfSources", schema=self.schema) 

328 self.makeSubtask("measurePsfSources", schema=self.schema) 

329 self.makeSubtask("measurePsf", schema=self.schema) 

330 

331 @utils.inheritDoc(pipeBase.PipelineTask) 

332 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

333 # Docstring to be formatted with info from PipelineTask.runQuantum 

334 """ 

335 Notes 

336 ----- 

337 Assemble a coadd from a set of Warps. 

338 

339 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

340 Analogous to `runDataRef`, it prepares all the data products to be 

341 passed to `run`, and processes the results before returning a struct 

342 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

343 Therefore, its inputs are accessed subregion by subregion 

344 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

345 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

346 correspond to an update in `runDataRef` while both entry points 

347 are used. 

348 """ 

349 inputData = butlerQC.get(inputRefs) 

350 

351 # Construct skyInfo expected by run 

352 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

353 skyMap = inputData["skyMap"] 

354 outputDataId = butlerQC.quantum.dataId 

355 

356 inputData['skyInfo'] = makeSkyInfo(skyMap, 

357 tractId=outputDataId['tract'], 

358 patchId=outputDataId['patch']) 

359 

360 # Construct list of input Deferred Datasets 

361 # These quack a bit like like Gen2 DataRefs 

362 warpRefList = inputData['inputWarps'] 

363 # Perform same middle steps as `runDataRef` does 

364 inputs = self.prepareInputs(warpRefList) 

365 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

366 self.getTempExpDatasetName(self.warpType)) 

367 if len(inputs.tempExpRefList) == 0: 

368 self.log.warning("No coadd temporary exposures found") 

369 return 

370 

371 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

372 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

373 inputs.weightList, supplementaryData=supplementaryData) 

374 

375 inputData.setdefault('brightObjectMask', None) 

376 for subfilter in range(self.config.dcrNumSubfilters): 

377 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

378 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf()) 

379 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId) 

380 

381 if self.config.doWrite: 

382 butlerQC.put(retStruct, outputRefs) 

383 return retStruct 

384 

385 @timeMethod 

386 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

387 """Assemble a coadd from a set of warps. 

388 

389 Coadd a set of Warps. Compute weights to be applied to each Warp and 

390 find scalings to match the photometric zeropoint to a reference Warp. 

391 Assemble the Warps using run method. 

392 Forward model chromatic effects across multiple subfilters, 

393 and subtract from the input Warps to build sets of residuals. 

394 Use the residuals to construct a new ``DcrModel`` for each subfilter, 

395 and iterate until the model converges. 

396 Interpolate over NaNs and optionally write the coadd to disk. 

397 Return the coadded exposure. 

398 

399 Parameters 

400 ---------- 

401 dataRef : `lsst.daf.persistence.ButlerDataRef` 

402 Data reference defining the patch for coaddition and the 

403 reference Warp 

404 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 

405 List of data references to warps. Data to be coadded will be 

406 selected from this list based on overlap with the patch defined by 

407 the data reference. 

408 

409 Returns 

410 ------- 

411 results : `lsst.pipe.base.Struct` 

412 The Struct contains the following fields: 

413 

414 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

415 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

416 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

417 - ``dcrNImages``: `list` of exposure count images for each subfilter 

418 """ 

419 if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList): 

420 raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList") 

421 

422 skyInfo = self.getSkyInfo(dataRef) 

423 if warpRefList is None: 

424 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

425 if len(calExpRefList) == 0: 

426 self.log.warning("No exposures to coadd") 

427 return 

428 self.log.info("Coadding %d exposures", len(calExpRefList)) 

429 

430 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

431 

432 inputData = self.prepareInputs(warpRefList) 

433 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

434 self.getTempExpDatasetName(self.warpType)) 

435 if len(inputData.tempExpRefList) == 0: 

436 self.log.warning("No coadd temporary exposures found") 

437 return 

438 

439 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

440 

441 results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

442 inputData.weightList, supplementaryData=supplementaryData) 

443 if results is None: 

444 self.log.warning("Could not construct DcrModel for patch %s: no data to coadd.", 

445 skyInfo.patchInfo.getIndex()) 

446 return 

447 

448 if self.config.doCalculatePsf: 

449 self.measureCoaddPsf(results.coaddExposure) 

450 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

451 for subfilter in range(self.config.dcrNumSubfilters): 

452 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

453 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf()) 

454 self.processResults(results.dcrCoadds[subfilter], 

455 brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

456 if self.config.doWrite: 

457 self.log.info("Persisting dcrCoadd") 

458 dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter, 

459 numSubfilters=self.config.dcrNumSubfilters) 

460 if self.config.doNImage and results.dcrNImages is not None: 

461 dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter, 

462 numSubfilters=self.config.dcrNumSubfilters) 

463 

464 return results 

465 

466 @utils.inheritDoc(AssembleCoaddTask) 

467 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

468 """Load the previously-generated template coadd. 

469 

470 This can be removed entirely once we no longer support the Gen 2 butler. 

471 

472 Returns 

473 ------- 

474 templateCoadd : `lsst.pipe.base.Struct` 

475 Result struct with components: 

476 

477 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`) 

478 """ 

479 templateCoadd = butlerQC.get(inputRefs.templateExposure) 

480 

481 return pipeBase.Struct(templateCoadd=templateCoadd) 

482 

483 def measureCoaddPsf(self, coaddExposure): 

484 """Detect sources on the coadd exposure and measure the final PSF. 

485 

486 Parameters 

487 ---------- 

488 coaddExposure : `lsst.afw.image.Exposure` 

489 The final coadded exposure. 

490 """ 

491 table = afwTable.SourceTable.make(self.schema) 

492 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False) 

493 coaddSources = detResults.sources 

494 self.measurePsfSources.run( 

495 measCat=coaddSources, 

496 exposure=coaddExposure 

497 ) 

498 # Measure the PSF on the stacked subfilter coadds if possible. 

499 # We should already have a decent estimate of the coadd PSF, however, 

500 # so in case of any errors simply log them as a warning and use the 

501 # default PSF. 

502 try: 

503 psfResults = self.measurePsf.run(coaddExposure, coaddSources) 

504 except Exception as e: 

505 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e) 

506 else: 

507 coaddExposure.setPsf(psfResults.psf) 

508 

509 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList): 

510 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 

511 

512 Sets the property ``bufferSize``. 

513 

514 Parameters 

515 ---------- 

516 templateCoadd : `lsst.afw.image.ExposureF` 

517 The initial coadd exposure before accounting for DCR. 

518 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

519 `lsst.daf.persistence.ButlerDataRef` 

520 The data references to the input warped exposures. 

521 weightList : `list` of `float` 

522 The weight to give each input exposure in the coadd 

523 Will be modified in place if ``doAirmassWeight`` is set. 

524 

525 Returns 

526 ------- 

527 dcrModels : `lsst.pipe.tasks.DcrModel` 

528 Best fit model of the true sky after correcting chromatic effects. 

529 

530 Raises 

531 ------ 

532 NotImplementedError 

533 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 

534 """ 

535 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

536 filterLabel = templateCoadd.getFilterLabel() 

537 tempExpName = self.getTempExpDatasetName(self.warpType) 

538 dcrShifts = [] 

539 airmassDict = {} 

540 angleDict = {} 

541 psfSizeDict = {} 

542 for visitNum, warpExpRef in enumerate(warpRefList): 

543 if isinstance(warpExpRef, DeferredDatasetHandle): 

544 # Gen 3 API 

545 visitInfo = warpExpRef.get(component="visitInfo") 

546 psf = warpExpRef.get(component="psf") 

547 else: 

548 # Gen 2 API. Delete this when Gen 2 retired 

549 visitInfo = warpExpRef.get(tempExpName + "_visitInfo") 

550 psf = warpExpRef.get(tempExpName).getPsf() 

551 visit = warpExpRef.dataId["visit"] 

552 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm 

553 airmass = visitInfo.getBoresightAirmass() 

554 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees() 

555 airmassDict[visit] = airmass 

556 angleDict[visit] = parallacticAngle 

557 psfSizeDict[visit] = psfSize 

558 if self.config.doAirmassWeight: 

559 weightList[visitNum] *= airmass 

560 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(), 

561 self.config.effectiveWavelength, 

562 self.config.bandwidth, 

563 self.config.dcrNumSubfilters)))) 

564 self.log.info("Selected airmasses:\n%s", airmassDict) 

565 self.log.info("Selected parallactic angles:\n%s", angleDict) 

566 self.log.info("Selected PSF sizes:\n%s", psfSizeDict) 

567 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1) 

568 try: 

569 psf = self.selectCoaddPsf(templateCoadd, warpRefList) 

570 except Exception as e: 

571 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e) 

572 else: 

573 psf = templateCoadd.getPsf() 

574 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage, 

575 self.config.dcrNumSubfilters, 

576 effectiveWavelength=self.config.effectiveWavelength, 

577 bandwidth=self.config.bandwidth, 

578 filterLabel=filterLabel, 

579 psf=psf) 

580 return dcrModels 

581 

582 @timeMethod 

583 def run(self, skyInfo, warpRefList, imageScalerList, weightList, 

584 supplementaryData=None): 

585 """Assemble the coadd. 

586 

587 Requires additional inputs Struct ``supplementaryData`` to contain a 

588 ``templateCoadd`` that serves as the model of the static sky. 

589 

590 Find artifacts and apply them to the warps' masks creating a list of 

591 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 

592 Then pass these alternative masks to the base class's assemble method. 

593 

594 Divide the ``templateCoadd`` evenly between each subfilter of a 

595 ``DcrModel`` as the starting best estimate of the true wavelength- 

596 dependent sky. Forward model the ``DcrModel`` using the known 

597 chromatic effects in each subfilter and calculate a convergence metric 

598 based on how well the modeled template matches the input warps. If 

599 the convergence has not yet reached the desired threshold, then shift 

600 and stack the residual images to build a new ``DcrModel``. Apply 

601 conditioning to prevent oscillating solutions between iterations or 

602 between subfilters. 

603 

604 Once the ``DcrModel`` reaches convergence or the maximum number of 

605 iterations has been reached, fill the metadata for each subfilter 

606 image and make them proper ``coaddExposure``s. 

607 

608 Parameters 

609 ---------- 

610 skyInfo : `lsst.pipe.base.Struct` 

611 Patch geometry information, from getSkyInfo 

612 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

613 `lsst.daf.persistence.ButlerDataRef` 

614 The data references to the input warped exposures. 

615 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

616 The image scalars correct for the zero point of the exposures. 

617 weightList : `list` of `float` 

618 The weight to give each input exposure in the coadd 

619 supplementaryData : `lsst.pipe.base.Struct` 

620 Result struct returned by ``makeSupplementaryData`` with components: 

621 

622 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 

623 

624 Returns 

625 ------- 

626 result : `lsst.pipe.base.Struct` 

627 Result struct with components: 

628 

629 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

630 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

631 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

632 - ``dcrNImages``: `list` of exposure count images for each subfilter 

633 """ 

634 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters 

635 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3 

636 templateCoadd = supplementaryData.templateCoadd 

637 baseMask = templateCoadd.mask.clone() 

638 # The variance plane is for each subfilter 

639 # and should be proportionately lower than the full-band image 

640 baseVariance = templateCoadd.variance.clone() 

641 baseVariance /= self.config.dcrNumSubfilters 

642 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList) 

643 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask. 

644 templateCoadd.setMask(baseMask) 

645 badMaskPlanes = self.config.badMaskPlanes[:] 

646 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes`` 

647 # This is because pixels in observations that are significantly affect by DCR 

648 # are likely to have many pixels that are both "DETECTED" and "CLIPPED", 

649 # but those are necessary to constrain the DCR model. 

650 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes) 

651 

652 stats = self.prepareStats(mask=badPixelMask) 

653 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList) 

654 if self.config.doNImage: 

655 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList, 

656 spanSetMaskList, stats.ctrl) 

657 nImage = afwImage.ImageU(skyInfo.bbox) 

658 # Note that this nImage will be a factor of dcrNumSubfilters higher than 

659 # the nImage returned by assembleCoadd for most pixels. This is because each 

660 # subfilter may have a different nImage, and fractional values are not allowed. 

661 for dcrNImage in dcrNImages: 

662 nImage += dcrNImage 

663 else: 

664 dcrNImages = None 

665 

666 subregionSize = geom.Extent2I(*self.config.subregionSize) 

667 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) 

668 * ceil(skyInfo.bbox.getWidth()/subregionSize[0])) 

669 subIter = 0 

670 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

671 modelIter = 0 

672 subIter += 1 

673 self.log.info("Computing coadd over patch %s subregion %s of %s: %s", 

674 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox) 

675 dcrBBox = geom.Box2I(subBBox) 

676 dcrBBox.grow(self.bufferSize) 

677 dcrBBox.clip(dcrModels.bbox) 

678 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox) 

679 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList, 

680 imageScalerList, spanSetMaskList) 

681 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

682 warpRefList, weightList, stats.ctrl) 

683 self.log.info("Initial convergence : %s", convergenceMetric) 

684 convergenceList = [convergenceMetric] 

685 gainList = [] 

686 convergenceCheck = 1. 

687 refImage = templateCoadd.image 

688 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter): 

689 gain = self.calculateGain(convergenceList, gainList) 

690 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList, 

691 stats.ctrl, convergenceMetric, gain, 

692 modelWeights, refImage, dcrWeights) 

693 if self.config.useConvergence: 

694 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

695 warpRefList, weightList, stats.ctrl) 

696 if convergenceMetric == 0: 

697 self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is " 

698 "most likely due to there being no valid data in the region.", 

699 skyInfo.patchInfo.getIndex(), subIter) 

700 break 

701 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric 

702 if (convergenceCheck < 0) & (modelIter > minNumIter): 

703 self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum " 

704 "iterations or desired convergence improvement of %s." 

705 " Divergence: %s", 

706 skyInfo.patchInfo.getIndex(), subIter, 

707 self.config.convergenceThreshold, convergenceCheck) 

708 break 

709 convergenceList.append(convergenceMetric) 

710 if modelIter > maxNumIter: 

711 if self.config.useConvergence: 

712 self.log.warning("Coadd patch %s subregion %s reached maximum iterations " 

713 "before reaching desired convergence improvement of %s." 

714 " Final convergence improvement: %s", 

715 skyInfo.patchInfo.getIndex(), subIter, 

716 self.config.convergenceThreshold, convergenceCheck) 

717 break 

718 

719 if self.config.useConvergence: 

720 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)", 

721 modelIter, convergenceMetric, 100.*convergenceCheck, gain) 

722 modelIter += 1 

723 else: 

724 if self.config.useConvergence: 

725 self.log.info("Coadd patch %s subregion %s finished with " 

726 "convergence metric %s after %s iterations", 

727 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter) 

728 else: 

729 self.log.info("Coadd patch %s subregion %s finished after %s iterations", 

730 skyInfo.patchInfo.getIndex(), subIter, modelIter) 

731 if self.config.useConvergence and convergenceMetric > 0: 

732 self.log.info("Final convergence improvement was %.4f%% overall", 

733 100*(convergenceList[0] - convergenceMetric)/convergenceMetric) 

734 

735 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList, 

736 calibration=self.scaleZeroPoint.getPhotoCalib(), 

737 coaddInputs=templateCoadd.getInfo().getCoaddInputs(), 

738 mask=baseMask, 

739 variance=baseVariance) 

740 coaddExposure = self.stackCoadd(dcrCoadds) 

741 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

742 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages) 

743 

744 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl): 

745 """Calculate the number of exposures contributing to each subfilter. 

746 

747 Parameters 

748 ---------- 

749 dcrModels : `lsst.pipe.tasks.DcrModel` 

750 Best fit model of the true sky after correcting chromatic effects. 

751 bbox : `lsst.geom.box.Box2I` 

752 Bounding box of the patch to coadd. 

753 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

754 `lsst.daf.persistence.ButlerDataRef` 

755 The data references to the input warped exposures. 

756 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

757 Each element of the `dict` contains the new mask plane name 

758 (e.g. "CLIPPED and/or "NO_DATA") as the key, 

759 and the list of SpanSets to apply to the mask. 

760 statsCtrl : `lsst.afw.math.StatisticsControl` 

761 Statistics control object for coadd 

762 

763 Returns 

764 ------- 

765 dcrNImages : `list` of `lsst.afw.image.ImageU` 

766 List of exposure count images for each subfilter 

767 dcrWeights : `list` of `lsst.afw.image.ImageF` 

768 Per-pixel weights for each subfilter. 

769 Equal to 1/(number of unmasked images contributing to each pixel). 

770 """ 

771 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

772 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

773 tempExpName = self.getTempExpDatasetName(self.warpType) 

774 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList): 

775 if isinstance(warpExpRef, DeferredDatasetHandle): 

776 # Gen 3 API 

777 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

778 else: 

779 # Gen 2 API. Delete this when Gen 2 retired 

780 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

781 visitInfo = exposure.getInfo().getVisitInfo() 

782 wcs = exposure.getInfo().getWcs() 

783 mask = exposure.mask 

784 if altMaskSpans is not None: 

785 self.applyAltMaskPlanes(mask, altMaskSpans) 

786 weightImage = np.zeros_like(exposure.image.array) 

787 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1. 

788 # The weights must be shifted in exactly the same way as the residuals, 

789 # because they will be used as the denominator in the weighted average of residuals. 

790 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, 

791 dcrModels.effectiveWavelength, dcrModels.bandwidth) 

792 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights): 

793 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype) 

794 dcrWeight.array += shiftedWeights 

795 # Exclude any pixels that don't have at least one exposure contributing in all subfilters 

796 weightsThreshold = 1. 

797 goodPix = dcrWeights[0].array > weightsThreshold 

798 for weights in dcrWeights[1:]: 

799 goodPix = (weights.array > weightsThreshold) & goodPix 

800 for subfilter in range(self.config.dcrNumSubfilters): 

801 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix] 

802 dcrWeights[subfilter].array[~goodPix] = 0. 

803 dcrNImages[subfilter].array[~goodPix] = 0 

804 return (dcrNImages, dcrWeights) 

805 

806 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, 

807 statsCtrl, convergenceMetric, 

808 gain, modelWeights, refImage, dcrWeights): 

809 """Assemble the DCR coadd for a sub-region. 

810 

811 Build a DCR-matched template for each input exposure, then shift the 

812 residuals according to the DCR in each subfilter. 

813 Stack the shifted residuals and apply them as a correction to the 

814 solution from the previous iteration. 

815 Restrict the new model solutions from varying by more than a factor of 

816 `modelClampFactor` from the last solution, and additionally restrict the 

817 individual subfilter models from varying by more than a factor of 

818 `frequencyClampFactor` from their average. 

819 Finally, mitigate potentially oscillating solutions by averaging the new 

820 solution with the solution from the previous iteration, weighted by 

821 their convergence metric. 

822 

823 Parameters 

824 ---------- 

825 dcrModels : `lsst.pipe.tasks.DcrModel` 

826 Best fit model of the true sky after correcting chromatic effects. 

827 subExposures : `dict` of `lsst.afw.image.ExposureF` 

828 The pre-loaded exposures for the current subregion. 

829 bbox : `lsst.geom.box.Box2I` 

830 Bounding box of the subregion to coadd. 

831 dcrBBox : `lsst.geom.box.Box2I` 

832 Sub-region of the coadd which includes a buffer to allow for DCR. 

833 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

834 `lsst.daf.persistence.ButlerDataRef` 

835 The data references to the input warped exposures. 

836 statsCtrl : `lsst.afw.math.StatisticsControl` 

837 Statistics control object for coadd 

838 convergenceMetric : `float` 

839 Quality of fit metric for the matched templates of the input images. 

840 gain : `float`, optional 

841 Relative weight to give the new solution when updating the model. 

842 modelWeights : `numpy.ndarray` or `float` 

843 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

844 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

845 refImage : `lsst.afw.image.Image` 

846 A reference image used to supply the default pixel values. 

847 dcrWeights : `list` of `lsst.afw.image.Image` 

848 Per-pixel weights for each subfilter. 

849 Equal to 1/(number of unmasked images contributing to each pixel). 

850 """ 

851 residualGeneratorList = [] 

852 

853 for warpExpRef in warpRefList: 

854 visit = warpExpRef.dataId["visit"] 

855 exposure = subExposures[visit] 

856 visitInfo = exposure.getInfo().getVisitInfo() 

857 wcs = exposure.getInfo().getWcs() 

858 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

859 order=self.config.imageInterpOrder, 

860 splitSubfilters=self.config.splitSubfilters, 

861 splitThreshold=self.config.splitThreshold, 

862 amplifyModel=self.config.accelerateModel) 

863 residual = exposure.image.array - templateImage.array 

864 # Note that the variance plane here is used to store weights, not the actual variance 

865 residual *= exposure.variance.array 

866 # The residuals are stored as a list of generators. 

867 # This allows the residual for a given subfilter and exposure to be created 

868 # on the fly, instead of needing to store them all in memory. 

869 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, 

870 dcrModels.effectiveWavelength, 

871 dcrModels.bandwidth)) 

872 

873 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

874 gain=gain, 

875 modelWeights=modelWeights, 

876 refImage=refImage, 

877 dcrWeights=dcrWeights) 

878 dcrModels.assign(dcrSubModelOut, bbox) 

879 

880 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth): 

881 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 

882 

883 Parameters 

884 ---------- 

885 residual : `numpy.ndarray` 

886 The residual masked image for one exposure, 

887 after subtracting the matched template 

888 visitInfo : `lsst.afw.image.VisitInfo` 

889 Metadata for the exposure. 

890 wcs : `lsst.afw.geom.SkyWcs` 

891 Coordinate system definition (wcs) for the exposure. 

892 

893 Yields 

894 ------ 

895 residualImage : `numpy.ndarray` 

896 The residual image for the next subfilter, shifted for DCR. 

897 """ 

898 # Pre-calculate the spline-filtered residual image, so that step can be 

899 # skipped in the shift calculation in `applyDcr`. 

900 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder) 

901 # Note that `splitSubfilters` is always turned off in the reverse direction. 

902 # This option introduces additional blurring if applied to the residuals. 

903 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters, 

904 splitSubfilters=False) 

905 for dcr in dcrShift: 

906 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False, 

907 doPrefilter=False, order=self.config.imageInterpOrder) 

908 

909 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

910 gain, modelWeights, refImage, dcrWeights): 

911 """Calculate a new DcrModel from a set of image residuals. 

912 

913 Parameters 

914 ---------- 

915 dcrModels : `lsst.pipe.tasks.DcrModel` 

916 Current model of the true sky after correcting chromatic effects. 

917 residualGeneratorList : `generator` of `numpy.ndarray` 

918 The residual image for the next subfilter, shifted for DCR. 

919 dcrBBox : `lsst.geom.box.Box2I` 

920 Sub-region of the coadd which includes a buffer to allow for DCR. 

921 statsCtrl : `lsst.afw.math.StatisticsControl` 

922 Statistics control object for coadd 

923 gain : `float` 

924 Relative weight to give the new solution when updating the model. 

925 modelWeights : `numpy.ndarray` or `float` 

926 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

927 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

928 refImage : `lsst.afw.image.Image` 

929 A reference image used to supply the default pixel values. 

930 dcrWeights : `list` of `lsst.afw.image.Image` 

931 Per-pixel weights for each subfilter. 

932 Equal to 1/(number of unmasked images contributing to each pixel). 

933 

934 Returns 

935 ------- 

936 dcrModel : `lsst.pipe.tasks.DcrModel` 

937 New model of the true sky after correcting chromatic effects. 

938 """ 

939 newModelImages = [] 

940 for subfilter, model in enumerate(dcrModels): 

941 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList] 

942 residual = np.sum(residualsList, axis=0) 

943 residual *= dcrWeights[subfilter][dcrBBox].array 

944 # `MaskedImage`s only support in-place addition, so rename for readability 

945 newModel = model[dcrBBox].clone() 

946 newModel.array += residual 

947 # Catch any invalid values 

948 badPixels = ~np.isfinite(newModel.array) 

949 newModel.array[badPixels] = model[dcrBBox].array[badPixels] 

950 if self.config.regularizeModelIterations > 0: 

951 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox, 

952 self.config.regularizeModelIterations, 

953 self.config.regularizationWidth) 

954 newModelImages.append(newModel) 

955 if self.config.regularizeModelFrequency > 0: 

956 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl, 

957 self.config.regularizeModelFrequency, 

958 self.config.regularizationWidth) 

959 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain) 

960 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights) 

961 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength, 

962 dcrModels.bandwidth, dcrModels.psf, 

963 dcrModels.mask, dcrModels.variance) 

964 

965 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl): 

966 """Calculate a quality of fit metric for the matched templates. 

967 

968 Parameters 

969 ---------- 

970 dcrModels : `lsst.pipe.tasks.DcrModel` 

971 Best fit model of the true sky after correcting chromatic effects. 

972 subExposures : `dict` of `lsst.afw.image.ExposureF` 

973 The pre-loaded exposures for the current subregion. 

974 bbox : `lsst.geom.box.Box2I` 

975 Sub-region to coadd 

976 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

977 `lsst.daf.persistence.ButlerDataRef` 

978 The data references to the input warped exposures. 

979 weightList : `list` of `float` 

980 The weight to give each input exposure in the coadd 

981 statsCtrl : `lsst.afw.math.StatisticsControl` 

982 Statistics control object for coadd 

983 

984 Returns 

985 ------- 

986 convergenceMetric : `float` 

987 Quality of fit metric for all input exposures, within the sub-region 

988 """ 

989 significanceImage = np.abs(dcrModels.getReferenceImage(bbox)) 

990 nSigma = 3. 

991 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl, 

992 bufferSize=self.bufferSize) 

993 if np.max(significanceImage) == 0: 

994 significanceImage += 1. 

995 weight = 0 

996 metric = 0. 

997 metricList = {} 

998 for warpExpRef, expWeight in zip(warpRefList, weightList): 

999 visit = warpExpRef.dataId["visit"] 

1000 exposure = subExposures[visit][bbox] 

1001 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl) 

1002 metric += singleMetric 

1003 metricList[visit] = singleMetric 

1004 weight += 1. 

1005 self.log.info("Individual metrics:\n%s", metricList) 

1006 return 1.0 if weight == 0.0 else metric/weight 

1007 

1008 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl): 

1009 """Calculate a quality of fit metric for a single matched template. 

1010 

1011 Parameters 

1012 ---------- 

1013 dcrModels : `lsst.pipe.tasks.DcrModel` 

1014 Best fit model of the true sky after correcting chromatic effects. 

1015 exposure : `lsst.afw.image.ExposureF` 

1016 The input warped exposure to evaluate. 

1017 significanceImage : `numpy.ndarray` 

1018 Array of weights for each pixel corresponding to its significance 

1019 for the convergence calculation. 

1020 statsCtrl : `lsst.afw.math.StatisticsControl` 

1021 Statistics control object for coadd 

1022 

1023 Returns 

1024 ------- 

1025 convergenceMetric : `float` 

1026 Quality of fit metric for one exposure, within the sub-region. 

1027 """ 

1028 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1029 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

1030 order=self.config.imageInterpOrder, 

1031 splitSubfilters=self.config.splitSubfilters, 

1032 splitThreshold=self.config.splitThreshold, 

1033 amplifyModel=self.config.accelerateModel) 

1034 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage 

1035 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2. 

1036 

1037 finitePixels = np.isfinite(diffVals) 

1038 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0 

1039 convergeMaskPixels = exposure.mask.array & convergeMask > 0 

1040 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels 

1041 if np.sum(usePixels) == 0: 

1042 metric = 0. 

1043 else: 

1044 diffUse = diffVals[usePixels] 

1045 refUse = refVals[usePixels] 

1046 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse)) 

1047 return metric 

1048 

1049 def stackCoadd(self, dcrCoadds): 

1050 """Add a list of sub-band coadds together. 

1051 

1052 Parameters 

1053 ---------- 

1054 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1055 A list of coadd exposures, each exposure containing 

1056 the model for one subfilter. 

1057 

1058 Returns 

1059 ------- 

1060 coaddExposure : `lsst.afw.image.ExposureF` 

1061 A single coadd exposure that is the sum of the sub-bands. 

1062 """ 

1063 coaddExposure = dcrCoadds[0].clone() 

1064 for coadd in dcrCoadds[1:]: 

1065 coaddExposure.maskedImage += coadd.maskedImage 

1066 return coaddExposure 

1067 

1068 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, 

1069 mask=None, variance=None): 

1070 """Create a list of coadd exposures from a list of masked images. 

1071 

1072 Parameters 

1073 ---------- 

1074 dcrModels : `lsst.pipe.tasks.DcrModel` 

1075 Best fit model of the true sky after correcting chromatic effects. 

1076 skyInfo : `lsst.pipe.base.Struct` 

1077 Patch geometry information, from getSkyInfo 

1078 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1079 `lsst.daf.persistence.ButlerDataRef` 

1080 The data references to the input warped exposures. 

1081 weightList : `list` of `float` 

1082 The weight to give each input exposure in the coadd 

1083 calibration : `lsst.afw.Image.PhotoCalib`, optional 

1084 Scale factor to set the photometric calibration of an exposure. 

1085 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 

1086 A record of the observations that are included in the coadd. 

1087 mask : `lsst.afw.image.Mask`, optional 

1088 Optional mask to override the values in the final coadd. 

1089 variance : `lsst.afw.image.Image`, optional 

1090 Optional variance plane to override the values in the final coadd. 

1091 

1092 Returns 

1093 ------- 

1094 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1095 A list of coadd exposures, each exposure containing 

1096 the model for one subfilter. 

1097 """ 

1098 dcrCoadds = [] 

1099 refModel = dcrModels.getReferenceImage() 

1100 for model in dcrModels: 

1101 if self.config.accelerateModel > 1: 

1102 model.array = (model.array - refModel)*self.config.accelerateModel + refModel 

1103 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

1104 if calibration is not None: 

1105 coaddExposure.setPhotoCalib(calibration) 

1106 if coaddInputs is not None: 

1107 coaddExposure.getInfo().setCoaddInputs(coaddInputs) 

1108 # Set the metadata for the coadd, including PSF and aperture corrections. 

1109 self.assembleMetadata(coaddExposure, warpRefList, weightList) 

1110 # Overwrite the PSF 

1111 coaddExposure.setPsf(dcrModels.psf) 

1112 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox]) 

1113 maskedImage = afwImage.MaskedImageF(dcrModels.bbox) 

1114 maskedImage.image = model 

1115 maskedImage.mask = dcrModels.mask 

1116 maskedImage.variance = dcrModels.variance 

1117 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox]) 

1118 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

1119 if mask is not None: 

1120 coaddExposure.setMask(mask) 

1121 if variance is not None: 

1122 coaddExposure.setVariance(variance) 

1123 dcrCoadds.append(coaddExposure) 

1124 return dcrCoadds 

1125 

1126 def calculateGain(self, convergenceList, gainList): 

1127 """Calculate the gain to use for the current iteration. 

1128 

1129 After calculating a new DcrModel, each value is averaged with the 

1130 value in the corresponding pixel from the previous iteration. This 

1131 reduces oscillating solutions that iterative techniques are plagued by, 

1132 and speeds convergence. By far the biggest changes to the model 

1133 happen in the first couple iterations, so we can also use a more 

1134 aggressive gain later when the model is changing slowly. 

1135 

1136 Parameters 

1137 ---------- 

1138 convergenceList : `list` of `float` 

1139 The quality of fit metric from each previous iteration. 

1140 gainList : `list` of `float` 

1141 The gains used in each previous iteration: appended with the new 

1142 gain value. 

1143 Gains are numbers between ``self.config.baseGain`` and 1. 

1144 

1145 Returns 

1146 ------- 

1147 gain : `float` 

1148 Relative weight to give the new solution when updating the model. 

1149 A value of 1.0 gives equal weight to both solutions. 

1150 

1151 Raises 

1152 ------ 

1153 ValueError 

1154 If ``len(convergenceList) != len(gainList)+1``. 

1155 """ 

1156 nIter = len(convergenceList) 

1157 if nIter != len(gainList) + 1: 

1158 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)." 

1159 % (len(convergenceList), len(gainList))) 

1160 

1161 if self.config.baseGain is None: 

1162 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters 

1163 # The more subfilters being modeled, the lower the gain should be. 

1164 baseGain = 1./(self.config.dcrNumSubfilters - 1) 

1165 else: 

1166 baseGain = self.config.baseGain 

1167 

1168 if self.config.useProgressiveGain and nIter > 2: 

1169 # To calculate the best gain to use, compare the past gains that have been used 

1170 # with the resulting convergences to estimate the best gain to use. 

1171 # Algorithmically, this is a Kalman filter. 

1172 # If forward modeling proceeds perfectly, the convergence metric should 

1173 # asymptotically approach a final value. 

1174 # We can estimate that value from the measured changes in convergence 

1175 # weighted by the gains used in each previous iteration. 

1176 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i] 

1177 for i in range(nIter - 1)] 

1178 # The convergence metric is strictly positive, so if the estimated final convergence is 

1179 # less than zero, force it to zero. 

1180 estFinalConv = np.array(estFinalConv) 

1181 estFinalConv[estFinalConv < 0] = 0 

1182 # Because the estimate may slowly change over time, only use the most recent measurements. 

1183 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):]) 

1184 lastGain = gainList[-1] 

1185 lastConv = convergenceList[-2] 

1186 newConv = convergenceList[-1] 

1187 # The predicted convergence is the value we would get if the new model calculated 

1188 # in the previous iteration was perfect. Recall that the updated model that is 

1189 # actually used is the gain-weighted average of the new and old model, 

1190 # so the convergence would be similarly weighted. 

1191 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain) 

1192 # If the measured and predicted convergence are very close, that indicates 

1193 # that our forward model is accurate and we can use a more aggressive gain 

1194 # If the measured convergence is significantly worse (or better!) than predicted, 

1195 # that indicates that the model is not converging as expected and 

1196 # we should use a more conservative gain. 

1197 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain)) 

1198 newGain = 1 - abs(delta) 

1199 # Average the gains to prevent oscillating solutions. 

1200 newGain = (newGain + lastGain)/2. 

1201 gain = max(baseGain, newGain) 

1202 else: 

1203 gain = baseGain 

1204 gainList.append(gain) 

1205 return gain 

1206 

1207 def calculateModelWeights(self, dcrModels, dcrBBox): 

1208 """Build an array that smoothly tapers to 0 away from detected sources. 

1209 

1210 Parameters 

1211 ---------- 

1212 dcrModels : `lsst.pipe.tasks.DcrModel` 

1213 Best fit model of the true sky after correcting chromatic effects. 

1214 dcrBBox : `lsst.geom.box.Box2I` 

1215 Sub-region of the coadd which includes a buffer to allow for DCR. 

1216 

1217 Returns 

1218 ------- 

1219 weights : `numpy.ndarray` or `float` 

1220 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1221 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1222 

1223 Raises 

1224 ------ 

1225 ValueError 

1226 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 

1227 """ 

1228 if not self.config.useModelWeights: 

1229 return 1.0 

1230 if self.config.modelWeightsWidth < 0: 

1231 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set") 

1232 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1233 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0 

1234 weights = np.zeros_like(dcrModels[0][dcrBBox].array) 

1235 weights[convergeMaskPixels] = 1. 

1236 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth) 

1237 weights /= np.max(weights) 

1238 return weights 

1239 

1240 def applyModelWeights(self, modelImages, refImage, modelWeights): 

1241 """Smoothly replace model pixel values with those from a 

1242 reference at locations away from detected sources. 

1243 

1244 Parameters 

1245 ---------- 

1246 modelImages : `list` of `lsst.afw.image.Image` 

1247 The new DCR model images from the current iteration. 

1248 The values will be modified in place. 

1249 refImage : `lsst.afw.image.MaskedImage` 

1250 A reference image used to supply the default pixel values. 

1251 modelWeights : `numpy.ndarray` or `float` 

1252 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1253 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1254 """ 

1255 if self.config.useModelWeights: 

1256 for model in modelImages: 

1257 model.array *= modelWeights 

1258 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters 

1259 

1260 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList): 

1261 """Pre-load sub-regions of a list of exposures. 

1262 

1263 Parameters 

1264 ---------- 

1265 bbox : `lsst.geom.box.Box2I` 

1266 Sub-region to coadd 

1267 statsCtrl : `lsst.afw.math.StatisticsControl` 

1268 Statistics control object for coadd 

1269 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1270 `lsst.daf.persistence.ButlerDataRef` 

1271 The data references to the input warped exposures. 

1272 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

1273 The image scalars correct for the zero point of the exposures. 

1274 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

1275 Each element is dict with keys = mask plane name to add the spans to 

1276 

1277 Returns 

1278 ------- 

1279 subExposures : `dict` 

1280 The `dict` keys are the visit IDs, 

1281 and the values are `lsst.afw.image.ExposureF` 

1282 The pre-loaded exposures for the current subregion. 

1283 The variance plane contains weights, and not the variance 

1284 """ 

1285 tempExpName = self.getTempExpDatasetName(self.warpType) 

1286 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList) 

1287 subExposures = {} 

1288 for warpExpRef, imageScaler, altMaskSpans in zipIterables: 

1289 if isinstance(warpExpRef, DeferredDatasetHandle): 

1290 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

1291 else: 

1292 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

1293 visit = warpExpRef.dataId["visit"] 

1294 if altMaskSpans is not None: 

1295 self.applyAltMaskPlanes(exposure.mask, altMaskSpans) 

1296 imageScaler.scaleMaskedImage(exposure.maskedImage) 

1297 # Note that the variance plane here is used to store weights, not the actual variance 

1298 exposure.variance.array[:, :] = 0. 

1299 # Set the weight of unmasked pixels to 1. 

1300 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1. 

1301 # Set the image value of masked pixels to zero. 

1302 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual`` 

1303 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0. 

1304 subExposures[visit] = exposure 

1305 return subExposures 

1306 

1307 def selectCoaddPsf(self, templateCoadd, warpRefList): 

1308 """Compute the PSF of the coadd from the exposures with the best seeing. 

1309 

1310 Parameters 

1311 ---------- 

1312 templateCoadd : `lsst.afw.image.ExposureF` 

1313 The initial coadd exposure before accounting for DCR. 

1314 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1315 `lsst.daf.persistence.ButlerDataRef` 

1316 The data references to the input warped exposures. 

1317 

1318 Returns 

1319 ------- 

1320 psf : `lsst.meas.algorithms.CoaddPsf` 

1321 The average PSF of the input exposures with the best seeing. 

1322 """ 

1323 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

1324 tempExpName = self.getTempExpDatasetName(self.warpType) 

1325 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit 

1326 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef`` 

1327 ccds = templateCoadd.getInfo().getCoaddInputs().ccds 

1328 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm 

1329 psfSizes = np.zeros(len(ccds)) 

1330 ccdVisits = np.array(ccds["visit"]) 

1331 for warpExpRef in warpRefList: 

1332 if isinstance(warpExpRef, DeferredDatasetHandle): 

1333 # Gen 3 API 

1334 psf = warpExpRef.get(component="psf") 

1335 else: 

1336 # Gen 2 API. Delete this when Gen 2 retired 

1337 psf = warpExpRef.get(tempExpName).getPsf() 

1338 visit = warpExpRef.dataId["visit"] 

1339 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm 

1340 psfSizes[ccdVisits == visit] = psfSize 

1341 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd 

1342 # The selected PSFs are those that have a FWHM less than or equal to the smaller 

1343 # of the mean or median FWHM of the input exposures. 

1344 sizeThreshold = min(np.median(psfSizes), psfRefSize) 

1345 goodPsfs = psfSizes <= sizeThreshold 

1346 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), 

1347 self.config.coaddPsf.makeControl()) 

1348 return psf