Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23from math import ceil 

24import numpy as np 

25from scipy import ndimage 

26import lsst.geom as geom 

27import lsst.afw.image as afwImage 

28import lsst.afw.table as afwTable 

29import lsst.coadd.utils as coaddUtils 

30from lsst.daf.butler import DeferredDatasetHandle 

31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel 

32import lsst.meas.algorithms as measAlg 

33from lsst.meas.base import SingleFrameMeasurementTask 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.utils as utils 

37from .assembleCoadd import (AssembleCoaddTask, 

38 CompareWarpAssembleCoaddConfig, 

39 CompareWarpAssembleCoaddTask) 

40from .coaddBase import makeSkyInfo 

41from .measurePsf import MeasurePsfTask 

42 

43__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"] 

44 

45 

46class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("tract", "patch", "band", "skymap"), 

48 defaultTemplates={"inputCoaddName": "deep", 

49 "outputCoaddName": "dcr", 

50 "warpType": "direct", 

51 "warpTypeSuffix": "", 

52 "fakesType": ""}): 

53 inputWarps = pipeBase.connectionTypes.Input( 

54 doc=("Input list of warps to be assembled i.e. stacked." 

55 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

56 name="{inputCoaddName}Coadd_{warpType}Warp", 

57 storageClass="ExposureF", 

58 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

59 deferLoad=True, 

60 multiple=True 

61 ) 

62 skyMap = pipeBase.connectionTypes.Input( 

63 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

64 name="{inputCoaddName}Coadd_skyMap", 

65 storageClass="SkyMap", 

66 dimensions=("skymap", ), 

67 ) 

68 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

69 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

70 " BRIGHT_OBJECT."), 

71 name="brightObjectMask", 

72 storageClass="ObjectMaskCatalog", 

73 dimensions=("tract", "patch", "skymap", "band"), 

74 ) 

75 templateExposure = pipeBase.connectionTypes.Input( 

76 doc="Input coadded exposure, produced by previous call to AssembleCoadd", 

77 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}", 

78 storageClass="ExposureF", 

79 dimensions=("tract", "patch", "skymap", "band"), 

80 ) 

81 dcrCoadds = pipeBase.connectionTypes.Output( 

82 doc="Output coadded exposure, produced by stacking input warps", 

83 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

84 storageClass="ExposureF", 

85 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

86 multiple=True, 

87 ) 

88 dcrNImages = pipeBase.connectionTypes.Output( 

89 doc="Output image of number of input images per pixel", 

90 name="{outputCoaddName}Coadd_nImage", 

91 storageClass="ImageU", 

92 dimensions=("tract", "patch", "skymap", "band", "subfilter"), 

93 multiple=True, 

94 ) 

95 

96 def __init__(self, *, config=None): 

97 super().__init__(config=config) 

98 if not config.doWrite: 

99 self.outputs.remove("dcrCoadds") 

100 

101 

102class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, 

103 pipelineConnections=DcrAssembleCoaddConnections): 

104 dcrNumSubfilters = pexConfig.Field( 

105 dtype=int, 

106 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.", 

107 default=3, 

108 ) 

109 maxNumIter = pexConfig.Field( 

110 dtype=int, 

111 optional=True, 

112 doc="Maximum number of iterations of forward modeling.", 

113 default=None, 

114 ) 

115 minNumIter = pexConfig.Field( 

116 dtype=int, 

117 optional=True, 

118 doc="Minimum number of iterations of forward modeling.", 

119 default=None, 

120 ) 

121 convergenceThreshold = pexConfig.Field( 

122 dtype=float, 

123 doc="Target relative change in convergence between iterations of forward modeling.", 

124 default=0.001, 

125 ) 

126 useConvergence = pexConfig.Field( 

127 dtype=bool, 

128 doc="Use convergence test as a forward modeling end condition?" 

129 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations", 

130 default=True, 

131 ) 

132 baseGain = pexConfig.Field( 

133 dtype=float, 

134 optional=True, 

135 doc="Relative weight to give the new solution vs. the last solution when updating the model." 

136 "A value of 1.0 gives equal weight to both solutions." 

137 "Small values imply slower convergence of the solution, but can " 

138 "help prevent overshooting and failures in the fit." 

139 "If ``baseGain`` is None, a conservative gain " 

140 "will be calculated from the number of subfilters. ", 

141 default=None, 

142 ) 

143 useProgressiveGain = pexConfig.Field( 

144 dtype=bool, 

145 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 

146 "When calculating the next gain, we use up to 5 previous gains and convergence values." 

147 "Can be set to False to force the model to change at the rate of ``baseGain``. ", 

148 default=True, 

149 ) 

150 doAirmassWeight = pexConfig.Field( 

151 dtype=bool, 

152 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.", 

153 default=False, 

154 ) 

155 modelWeightsWidth = pexConfig.Field( 

156 dtype=float, 

157 doc="Width of the region around detected sources to include in the DcrModel.", 

158 default=3, 

159 ) 

160 useModelWeights = pexConfig.Field( 

161 dtype=bool, 

162 doc="Width of the region around detected sources to include in the DcrModel.", 

163 default=True, 

164 ) 

165 splitSubfilters = pexConfig.Field( 

166 dtype=bool, 

167 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter." 

168 "Instead of at the midpoint", 

169 default=True, 

170 ) 

171 splitThreshold = pexConfig.Field( 

172 dtype=float, 

173 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 

174 "Set to 0 to always split the subfilters.", 

175 default=0.1, 

176 ) 

177 regularizeModelIterations = pexConfig.Field( 

178 dtype=float, 

179 doc="Maximum relative change of the model allowed between iterations." 

180 "Set to zero to disable.", 

181 default=2., 

182 ) 

183 regularizeModelFrequency = pexConfig.Field( 

184 dtype=float, 

185 doc="Maximum relative change of the model allowed between subfilters." 

186 "Set to zero to disable.", 

187 default=4., 

188 ) 

189 convergenceMaskPlanes = pexConfig.ListField( 

190 dtype=str, 

191 default=["DETECTED"], 

192 doc="Mask planes to use to calculate convergence." 

193 ) 

194 regularizationWidth = pexConfig.Field( 

195 dtype=int, 

196 default=2, 

197 doc="Minimum radius of a region to include in regularization, in pixels." 

198 ) 

199 imageInterpOrder = pexConfig.Field( 

200 dtype=int, 

201 doc="The order of the spline interpolation used to shift the image plane.", 

202 default=3, 

203 ) 

204 accelerateModel = pexConfig.Field( 

205 dtype=float, 

206 doc="Factor to amplify the differences between model planes by to speed convergence.", 

207 default=3, 

208 ) 

209 doCalculatePsf = pexConfig.Field( 

210 dtype=bool, 

211 doc="Set to detect stars and recalculate the PSF from the final coadd." 

212 "Otherwise the PSF is estimated from a selection of the best input exposures", 

213 default=False, 

214 ) 

215 detectPsfSources = pexConfig.ConfigurableField( 

216 target=measAlg.SourceDetectionTask, 

217 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.", 

218 ) 

219 measurePsfSources = pexConfig.ConfigurableField( 

220 target=SingleFrameMeasurementTask, 

221 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 

222 ) 

223 measurePsf = pexConfig.ConfigurableField( 

224 target=MeasurePsfTask, 

225 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.", 

226 ) 

227 

228 def setDefaults(self): 

229 CompareWarpAssembleCoaddConfig.setDefaults(self) 

230 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask) 

231 self.doNImage = True 

232 self.assembleStaticSkyModel.warpType = self.warpType 

233 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time 

234 self.assembleStaticSkyModel.doNImage = False 

235 self.assembleStaticSkyModel.doWrite = False 

236 self.detectPsfSources.returnOriginalFootprints = False 

237 self.detectPsfSources.thresholdPolarity = "positive" 

238 # Only use bright sources for PSF measurement 

239 self.detectPsfSources.thresholdValue = 50 

240 self.detectPsfSources.nSigmaToGrow = 2 

241 # A valid star for PSF measurement should at least fill 5x5 pixels 

242 self.detectPsfSources.minPixels = 25 

243 # Use the variance plane to calculate signal to noise 

244 self.detectPsfSources.thresholdType = "pixel_stdev" 

245 # The signal to noise limit is good enough, while the flux limit is set 

246 # in dimensionless units and may not be appropriate for all data sets. 

247 self.measurePsf.starSelector["objectSize"].doFluxLimit = False 

248 

249 

250class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask): 

251 """Assemble DCR coadded images from a set of warps. 

252 

253 Attributes 

254 ---------- 

255 bufferSize : `int` 

256 The number of pixels to grow each subregion by to allow for DCR. 

257 

258 Notes 

259 ----- 

260 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 

261 Warps (also called coadded temporary exposures), including the effects of 

262 Differential Chromatic Refraction (DCR). 

263 For full details of the mathematics and algorithm, please see 

264 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 

265 

266 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 

267 each subfilter used in the iterative calculation. 

268 It begins by dividing the bandpass-defining filter into N equal bandwidth 

269 "subfilters", and divides the flux in each pixel from an initial coadd 

270 equally into each as a "dcrModel". Because the airmass and parallactic 

271 angle of each individual exposure is known, we can calculate the shift 

272 relative to the center of the band in each subfilter due to DCR. For each 

273 exposure we apply this shift as a linear transformation to the dcrModels 

274 and stack the results to produce a DCR-matched exposure. The matched 

275 exposures are subtracted from the input exposures to produce a set of 

276 residual images, and these residuals are reverse shifted for each 

277 exposures' subfilters and stacked. The shifted and stacked residuals are 

278 added to the dcrModels to produce a new estimate of the flux in each pixel 

279 within each subfilter. The dcrModels are solved for iteratively, which 

280 continues until the solution from a new iteration improves by less than 

281 a set percentage, or a maximum number of iterations is reached. 

282 Two forms of regularization are employed to reduce unphysical results. 

283 First, the new solution is averaged with the solution from the previous 

284 iteration, which mitigates oscillating solutions where the model 

285 overshoots with alternating very high and low values. 

286 Second, a common degeneracy when the data have a limited range of airmass or 

287 parallactic angle values is for one subfilter to be fit with very low or 

288 negative values, while another subfilter is fit with very high values. This 

289 typically appears in the form of holes next to sources in one subfilter, 

290 and corresponding extended wings in another. Because each subfilter has 

291 a narrow bandwidth we assume that physical sources that are above the noise 

292 level will not vary in flux by more than a factor of `frequencyClampFactor` 

293 between subfilters, and pixels that have flux deviations larger than that 

294 factor will have the excess flux distributed evenly among all subfilters. 

295 If `splitSubfilters` is set, then each subfilter will be further sub- 

296 divided during the forward modeling step (only). This approximates using 

297 a higher number of subfilters that may be necessary for high airmass 

298 observations, but does not increase the number of free parameters in the 

299 fit. This is needed when there are high airmass observations which would 

300 otherwise have significant DCR even within a subfilter. Because calculating 

301 the shifted images takes most of the time, splitting the subfilters is 

302 turned off by way of the `splitThreshold` option for low-airmass 

303 observations that do not suffer from DCR within a subfilter. 

304 """ 

305 

306 ConfigClass = DcrAssembleCoaddConfig 

307 _DefaultName = "dcrAssembleCoadd" 

308 

309 def __init__(self, *args, **kwargs): 

310 super().__init__(*args, **kwargs) 

311 if self.config.doCalculatePsf: 311 ↛ 312line 311 didn't jump to line 312, because the condition on line 311 was never true

312 self.schema = afwTable.SourceTable.makeMinimalSchema() 

313 self.makeSubtask("detectPsfSources", schema=self.schema) 

314 self.makeSubtask("measurePsfSources", schema=self.schema) 

315 self.makeSubtask("measurePsf", schema=self.schema) 

316 

317 @utils.inheritDoc(pipeBase.PipelineTask) 

318 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

319 # Docstring to be formatted with info from PipelineTask.runQuantum 

320 """ 

321 Notes 

322 ----- 

323 Assemble a coadd from a set of Warps. 

324 

325 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

326 Analogous to `runDataRef`, it prepares all the data products to be 

327 passed to `run`, and processes the results before returning a struct 

328 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

329 Therefore, its inputs are accessed subregion by subregion 

330 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

331 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

332 correspond to an update in `runDataRef` while both entry points 

333 are used. 

334 """ 

335 inputData = butlerQC.get(inputRefs) 

336 

337 # Construct skyInfo expected by run 

338 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

339 skyMap = inputData["skyMap"] 

340 outputDataId = butlerQC.quantum.dataId 

341 

342 inputData['skyInfo'] = makeSkyInfo(skyMap, 

343 tractId=outputDataId['tract'], 

344 patchId=outputDataId['patch']) 

345 

346 # Construct list of input Deferred Datasets 

347 # These quack a bit like like Gen2 DataRefs 

348 warpRefList = inputData['inputWarps'] 

349 # Perform same middle steps as `runDataRef` does 

350 inputs = self.prepareInputs(warpRefList) 

351 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

352 self.getTempExpDatasetName(self.warpType)) 

353 if len(inputs.tempExpRefList) == 0: 

354 self.log.warn("No coadd temporary exposures found") 

355 return 

356 

357 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

358 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

359 inputs.weightList, supplementaryData=supplementaryData) 

360 

361 inputData.setdefault('brightObjectMask', None) 

362 for subfilter in range(self.config.dcrNumSubfilters): 

363 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

364 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf()) 

365 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId) 

366 

367 if self.config.doWrite: 

368 butlerQC.put(retStruct, outputRefs) 

369 return retStruct 

370 

371 @pipeBase.timeMethod 

372 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

373 """Assemble a coadd from a set of warps. 

374 

375 Coadd a set of Warps. Compute weights to be applied to each Warp and 

376 find scalings to match the photometric zeropoint to a reference Warp. 

377 Assemble the Warps using run method. 

378 Forward model chromatic effects across multiple subfilters, 

379 and subtract from the input Warps to build sets of residuals. 

380 Use the residuals to construct a new ``DcrModel`` for each subfilter, 

381 and iterate until the model converges. 

382 Interpolate over NaNs and optionally write the coadd to disk. 

383 Return the coadded exposure. 

384 

385 Parameters 

386 ---------- 

387 dataRef : `lsst.daf.persistence.ButlerDataRef` 

388 Data reference defining the patch for coaddition and the 

389 reference Warp 

390 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 

391 List of data references to warps. Data to be coadded will be 

392 selected from this list based on overlap with the patch defined by 

393 the data reference. 

394 

395 Returns 

396 ------- 

397 results : `lsst.pipe.base.Struct` 

398 The Struct contains the following fields: 

399 

400 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

401 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

402 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

403 - ``dcrNImages``: `list` of exposure count images for each subfilter 

404 """ 

405 if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList): 

406 raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList") 

407 

408 skyInfo = self.getSkyInfo(dataRef) 

409 if warpRefList is None: 

410 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

411 if len(calExpRefList) == 0: 

412 self.log.warn("No exposures to coadd") 

413 return 

414 self.log.info("Coadding %d exposures", len(calExpRefList)) 

415 

416 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

417 

418 inputData = self.prepareInputs(warpRefList) 

419 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

420 self.getTempExpDatasetName(self.warpType)) 

421 if len(inputData.tempExpRefList) == 0: 

422 self.log.warn("No coadd temporary exposures found") 

423 return 

424 

425 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

426 

427 results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

428 inputData.weightList, supplementaryData=supplementaryData) 

429 if results is None: 

430 self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.", 

431 skyInfo.patchInfo.getIndex()) 

432 return 

433 

434 if self.config.doCalculatePsf: 

435 self.measureCoaddPsf(results.coaddExposure) 

436 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

437 for subfilter in range(self.config.dcrNumSubfilters): 

438 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter 

439 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf()) 

440 self.processResults(results.dcrCoadds[subfilter], 

441 brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

442 if self.config.doWrite: 

443 self.log.info("Persisting dcrCoadd") 

444 dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter, 

445 numSubfilters=self.config.dcrNumSubfilters) 

446 if self.config.doNImage and results.dcrNImages is not None: 

447 dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter, 

448 numSubfilters=self.config.dcrNumSubfilters) 

449 

450 return results 

451 

452 @utils.inheritDoc(AssembleCoaddTask) 

453 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

454 """Load the previously-generated template coadd. 

455 

456 This can be removed entirely once we no longer support the Gen 2 butler. 

457 

458 Returns 

459 ------- 

460 templateCoadd : `lsst.pipe.base.Struct` 

461 Result struct with components: 

462 

463 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`) 

464 """ 

465 templateCoadd = butlerQC.get(inputRefs.templateExposure) 

466 

467 return pipeBase.Struct(templateCoadd=templateCoadd) 

468 

469 def measureCoaddPsf(self, coaddExposure): 

470 """Detect sources on the coadd exposure and measure the final PSF. 

471 

472 Parameters 

473 ---------- 

474 coaddExposure : `lsst.afw.image.Exposure` 

475 The final coadded exposure. 

476 """ 

477 table = afwTable.SourceTable.make(self.schema) 

478 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False) 

479 coaddSources = detResults.sources 

480 self.measurePsfSources.run( 

481 measCat=coaddSources, 

482 exposure=coaddExposure 

483 ) 

484 # Measure the PSF on the stacked subfilter coadds if possible. 

485 # We should already have a decent estimate of the coadd PSF, however, 

486 # so in case of any errors simply log them as a warning and use the 

487 # default PSF. 

488 try: 

489 psfResults = self.measurePsf.run(coaddExposure, coaddSources) 

490 except Exception as e: 

491 self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e) 

492 else: 

493 coaddExposure.setPsf(psfResults.psf) 

494 

495 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList): 

496 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 

497 

498 Sets the property ``bufferSize``. 

499 

500 Parameters 

501 ---------- 

502 templateCoadd : `lsst.afw.image.ExposureF` 

503 The initial coadd exposure before accounting for DCR. 

504 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

505 `lsst.daf.persistence.ButlerDataRef` 

506 The data references to the input warped exposures. 

507 weightList : `list` of `float` 

508 The weight to give each input exposure in the coadd 

509 Will be modified in place if ``doAirmassWeight`` is set. 

510 

511 Returns 

512 ------- 

513 dcrModels : `lsst.pipe.tasks.DcrModel` 

514 Best fit model of the true sky after correcting chromatic effects. 

515 

516 Raises 

517 ------ 

518 NotImplementedError 

519 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 

520 """ 

521 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

522 filterInfo = templateCoadd.getFilter() 

523 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()): 523 ↛ 524line 523 didn't jump to line 524, because the condition on line 523 was never true

524 raise NotImplementedError("No minimum/maximum wavelength information found" 

525 " in the filter definition! Please add lambdaMin and lambdaMax" 

526 " to the Mapper class in your obs package.") 

527 tempExpName = self.getTempExpDatasetName(self.warpType) 

528 dcrShifts = [] 

529 airmassDict = {} 

530 angleDict = {} 

531 psfSizeDict = {} 

532 for visitNum, warpExpRef in enumerate(warpRefList): 

533 if isinstance(warpExpRef, DeferredDatasetHandle): 

534 # Gen 3 API 

535 visitInfo = warpExpRef.get(component="visitInfo") 

536 psf = warpExpRef.get(component="psf") 

537 else: 

538 # Gen 2 API. Delete this when Gen 2 retired 

539 visitInfo = warpExpRef.get(tempExpName + "_visitInfo") 

540 psf = warpExpRef.get(tempExpName).getPsf() 

541 visit = warpExpRef.dataId["visit"] 

542 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm 

543 airmass = visitInfo.getBoresightAirmass() 

544 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees() 

545 airmassDict[visit] = airmass 

546 angleDict[visit] = parallacticAngle 

547 psfSizeDict[visit] = psfSize 

548 if self.config.doAirmassWeight: 548 ↛ 549line 548 didn't jump to line 549, because the condition on line 548 was never true

549 weightList[visitNum] *= airmass 

550 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(), 

551 filterInfo, self.config.dcrNumSubfilters)))) 

552 self.log.info("Selected airmasses:\n%s", airmassDict) 

553 self.log.info("Selected parallactic angles:\n%s", angleDict) 

554 self.log.info("Selected PSF sizes:\n%s", psfSizeDict) 

555 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1) 

556 try: 

557 psf = self.selectCoaddPsf(templateCoadd, warpRefList) 

558 except Exception as e: 

559 self.log.warn("Unable to calculate restricted PSF, using default coadd PSF: %s" % e) 

560 else: 

561 psf = templateCoadd.getPsf() 

562 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage, 

563 self.config.dcrNumSubfilters, 

564 filterInfo=filterInfo, 

565 psf=psf) 

566 return dcrModels 

567 

568 @pipeBase.timeMethod 

569 def run(self, skyInfo, warpRefList, imageScalerList, weightList, 

570 supplementaryData=None): 

571 """Assemble the coadd. 

572 

573 Requires additional inputs Struct ``supplementaryData`` to contain a 

574 ``templateCoadd`` that serves as the model of the static sky. 

575 

576 Find artifacts and apply them to the warps' masks creating a list of 

577 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 

578 Then pass these alternative masks to the base class's assemble method. 

579 

580 Divide the ``templateCoadd`` evenly between each subfilter of a 

581 ``DcrModel`` as the starting best estimate of the true wavelength- 

582 dependent sky. Forward model the ``DcrModel`` using the known 

583 chromatic effects in each subfilter and calculate a convergence metric 

584 based on how well the modeled template matches the input warps. If 

585 the convergence has not yet reached the desired threshold, then shift 

586 and stack the residual images to build a new ``DcrModel``. Apply 

587 conditioning to prevent oscillating solutions between iterations or 

588 between subfilters. 

589 

590 Once the ``DcrModel`` reaches convergence or the maximum number of 

591 iterations has been reached, fill the metadata for each subfilter 

592 image and make them proper ``coaddExposure``s. 

593 

594 Parameters 

595 ---------- 

596 skyInfo : `lsst.pipe.base.Struct` 

597 Patch geometry information, from getSkyInfo 

598 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

599 `lsst.daf.persistence.ButlerDataRef` 

600 The data references to the input warped exposures. 

601 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

602 The image scalars correct for the zero point of the exposures. 

603 weightList : `list` of `float` 

604 The weight to give each input exposure in the coadd 

605 supplementaryData : `lsst.pipe.base.Struct` 

606 Result struct returned by ``makeSupplementaryData`` with components: 

607 

608 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 

609 

610 Returns 

611 ------- 

612 result : `lsst.pipe.base.Struct` 

613 Result struct with components: 

614 

615 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 

616 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 

617 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 

618 - ``dcrNImages``: `list` of exposure count images for each subfilter 

619 """ 

620 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters 

621 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3 

622 templateCoadd = supplementaryData.templateCoadd 

623 baseMask = templateCoadd.mask.clone() 

624 # The variance plane is for each subfilter 

625 # and should be proportionately lower than the full-band image 

626 baseVariance = templateCoadd.variance.clone() 

627 baseVariance /= self.config.dcrNumSubfilters 

628 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList) 

629 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask. 

630 templateCoadd.setMask(baseMask) 

631 badMaskPlanes = self.config.badMaskPlanes[:] 

632 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes`` 

633 # This is because pixels in observations that are significantly affect by DCR 

634 # are likely to have many pixels that are both "DETECTED" and "CLIPPED", 

635 # but those are necessary to constrain the DCR model. 

636 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes) 

637 

638 stats = self.prepareStats(mask=badPixelMask) 

639 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList) 

640 if self.config.doNImage: 640 ↛ 650line 640 didn't jump to line 650, because the condition on line 640 was never false

641 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList, 

642 spanSetMaskList, stats.ctrl) 

643 nImage = afwImage.ImageU(skyInfo.bbox) 

644 # Note that this nImage will be a factor of dcrNumSubfilters higher than 

645 # the nImage returned by assembleCoadd for most pixels. This is because each 

646 # subfilter may have a different nImage, and fractional values are not allowed. 

647 for dcrNImage in dcrNImages: 

648 nImage += dcrNImage 

649 else: 

650 dcrNImages = None 

651 

652 subregionSize = geom.Extent2I(*self.config.subregionSize) 

653 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) 

654 * ceil(skyInfo.bbox.getWidth()/subregionSize[0])) 

655 subIter = 0 

656 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

657 modelIter = 0 

658 subIter += 1 

659 self.log.info("Computing coadd over patch %s subregion %s of %s: %s", 

660 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox) 

661 dcrBBox = geom.Box2I(subBBox) 

662 dcrBBox.grow(self.bufferSize) 

663 dcrBBox.clip(dcrModels.bbox) 

664 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox) 

665 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList, 

666 imageScalerList, spanSetMaskList) 

667 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

668 warpRefList, weightList, stats.ctrl) 

669 self.log.info("Initial convergence : %s", convergenceMetric) 

670 convergenceList = [convergenceMetric] 

671 gainList = [] 

672 convergenceCheck = 1. 

673 refImage = templateCoadd.image 

674 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter): 

675 gain = self.calculateGain(convergenceList, gainList) 

676 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList, 

677 stats.ctrl, convergenceMetric, gain, 

678 modelWeights, refImage, dcrWeights) 

679 if self.config.useConvergence: 679 ↛ 696line 679 didn't jump to line 696, because the condition on line 679 was never false

680 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox, 

681 warpRefList, weightList, stats.ctrl) 

682 if convergenceMetric == 0: 682 ↛ 683line 682 didn't jump to line 683, because the condition on line 682 was never true

683 self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is " 

684 "most likely due to there being no valid data in the region.", 

685 skyInfo.patchInfo.getIndex(), subIter) 

686 break 

687 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric 

688 if (convergenceCheck < 0) & (modelIter > minNumIter): 688 ↛ 689line 688 didn't jump to line 689, because the condition on line 688 was never true

689 self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum " 

690 "iterations or desired convergence improvement of %s." 

691 " Divergence: %s", 

692 skyInfo.patchInfo.getIndex(), subIter, 

693 self.config.convergenceThreshold, convergenceCheck) 

694 break 

695 convergenceList.append(convergenceMetric) 

696 if modelIter > maxNumIter: 696 ↛ 697line 696 didn't jump to line 697, because the condition on line 696 was never true

697 if self.config.useConvergence: 

698 self.log.warn("Coadd patch %s subregion %s reached maximum iterations " 

699 "before reaching desired convergence improvement of %s." 

700 " Final convergence improvement: %s", 

701 skyInfo.patchInfo.getIndex(), subIter, 

702 self.config.convergenceThreshold, convergenceCheck) 

703 break 

704 

705 if self.config.useConvergence: 705 ↛ 708line 705 didn't jump to line 708, because the condition on line 705 was never false

706 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)", 

707 modelIter, convergenceMetric, 100.*convergenceCheck, gain) 

708 modelIter += 1 

709 else: 

710 if self.config.useConvergence: 710 ↛ 715line 710 didn't jump to line 715, because the condition on line 710 was never false

711 self.log.info("Coadd patch %s subregion %s finished with " 

712 "convergence metric %s after %s iterations", 

713 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter) 

714 else: 

715 self.log.info("Coadd patch %s subregion %s finished after %s iterations", 

716 skyInfo.patchInfo.getIndex(), subIter, modelIter) 

717 if self.config.useConvergence and convergenceMetric > 0: 717 ↛ 656line 717 didn't jump to line 656, because the condition on line 717 was never false

718 self.log.info("Final convergence improvement was %.4f%% overall", 

719 100*(convergenceList[0] - convergenceMetric)/convergenceMetric) 

720 

721 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList, 

722 calibration=self.scaleZeroPoint.getPhotoCalib(), 

723 coaddInputs=templateCoadd.getInfo().getCoaddInputs(), 

724 mask=baseMask, 

725 variance=baseVariance) 

726 coaddExposure = self.stackCoadd(dcrCoadds) 

727 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

728 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages) 

729 

730 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl): 

731 """Calculate the number of exposures contributing to each subfilter. 

732 

733 Parameters 

734 ---------- 

735 dcrModels : `lsst.pipe.tasks.DcrModel` 

736 Best fit model of the true sky after correcting chromatic effects. 

737 bbox : `lsst.geom.box.Box2I` 

738 Bounding box of the patch to coadd. 

739 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

740 `lsst.daf.persistence.ButlerDataRef` 

741 The data references to the input warped exposures. 

742 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

743 Each element of the `dict` contains the new mask plane name 

744 (e.g. "CLIPPED and/or "NO_DATA") as the key, 

745 and the list of SpanSets to apply to the mask. 

746 statsCtrl : `lsst.afw.math.StatisticsControl` 

747 Statistics control object for coadd 

748 

749 Returns 

750 ------- 

751 dcrNImages : `list` of `lsst.afw.image.ImageU` 

752 List of exposure count images for each subfilter 

753 dcrWeights : `list` of `lsst.afw.image.ImageF` 

754 Per-pixel weights for each subfilter. 

755 Equal to 1/(number of unmasked images contributing to each pixel). 

756 """ 

757 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

758 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)] 

759 tempExpName = self.getTempExpDatasetName(self.warpType) 

760 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList): 

761 if isinstance(warpExpRef, DeferredDatasetHandle): 

762 # Gen 3 API 

763 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

764 else: 

765 # Gen 2 API. Delete this when Gen 2 retired 

766 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

767 visitInfo = exposure.getInfo().getVisitInfo() 

768 wcs = exposure.getInfo().getWcs() 

769 mask = exposure.mask 

770 if altMaskSpans is not None: 770 ↛ 772line 770 didn't jump to line 772, because the condition on line 770 was never false

771 self.applyAltMaskPlanes(mask, altMaskSpans) 

772 weightImage = np.zeros_like(exposure.image.array) 

773 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1. 

774 # The weights must be shifted in exactly the same way as the residuals, 

775 # because they will be used as the denominator in the weighted average of residuals. 

776 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter) 

777 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights): 

778 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype) 

779 dcrWeight.array += shiftedWeights 

780 # Exclude any pixels that don't have at least one exposure contributing in all subfilters 

781 weightsThreshold = 1. 

782 goodPix = dcrWeights[0].array > weightsThreshold 

783 for weights in dcrWeights[1:]: 

784 goodPix = (weights.array > weightsThreshold) & goodPix 

785 for subfilter in range(self.config.dcrNumSubfilters): 

786 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix] 

787 dcrWeights[subfilter].array[~goodPix] = 0. 

788 dcrNImages[subfilter].array[~goodPix] = 0 

789 return (dcrNImages, dcrWeights) 

790 

791 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, 

792 statsCtrl, convergenceMetric, 

793 gain, modelWeights, refImage, dcrWeights): 

794 """Assemble the DCR coadd for a sub-region. 

795 

796 Build a DCR-matched template for each input exposure, then shift the 

797 residuals according to the DCR in each subfilter. 

798 Stack the shifted residuals and apply them as a correction to the 

799 solution from the previous iteration. 

800 Restrict the new model solutions from varying by more than a factor of 

801 `modelClampFactor` from the last solution, and additionally restrict the 

802 individual subfilter models from varying by more than a factor of 

803 `frequencyClampFactor` from their average. 

804 Finally, mitigate potentially oscillating solutions by averaging the new 

805 solution with the solution from the previous iteration, weighted by 

806 their convergence metric. 

807 

808 Parameters 

809 ---------- 

810 dcrModels : `lsst.pipe.tasks.DcrModel` 

811 Best fit model of the true sky after correcting chromatic effects. 

812 subExposures : `dict` of `lsst.afw.image.ExposureF` 

813 The pre-loaded exposures for the current subregion. 

814 bbox : `lsst.geom.box.Box2I` 

815 Bounding box of the subregion to coadd. 

816 dcrBBox : `lsst.geom.box.Box2I` 

817 Sub-region of the coadd which includes a buffer to allow for DCR. 

818 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

819 `lsst.daf.persistence.ButlerDataRef` 

820 The data references to the input warped exposures. 

821 statsCtrl : `lsst.afw.math.StatisticsControl` 

822 Statistics control object for coadd 

823 convergenceMetric : `float` 

824 Quality of fit metric for the matched templates of the input images. 

825 gain : `float`, optional 

826 Relative weight to give the new solution when updating the model. 

827 modelWeights : `numpy.ndarray` or `float` 

828 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

829 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

830 refImage : `lsst.afw.image.Image` 

831 A reference image used to supply the default pixel values. 

832 dcrWeights : `list` of `lsst.afw.image.Image` 

833 Per-pixel weights for each subfilter. 

834 Equal to 1/(number of unmasked images contributing to each pixel). 

835 """ 

836 residualGeneratorList = [] 

837 

838 for warpExpRef in warpRefList: 

839 visit = warpExpRef.dataId["visit"] 

840 exposure = subExposures[visit] 

841 visitInfo = exposure.getInfo().getVisitInfo() 

842 wcs = exposure.getInfo().getWcs() 

843 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

844 order=self.config.imageInterpOrder, 

845 splitSubfilters=self.config.splitSubfilters, 

846 splitThreshold=self.config.splitThreshold, 

847 amplifyModel=self.config.accelerateModel) 

848 residual = exposure.image.array - templateImage.array 

849 # Note that the variance plane here is used to store weights, not the actual variance 

850 residual *= exposure.variance.array 

851 # The residuals are stored as a list of generators. 

852 # This allows the residual for a given subfilter and exposure to be created 

853 # on the fly, instead of needing to store them all in memory. 

854 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter)) 

855 

856 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

857 gain=gain, 

858 modelWeights=modelWeights, 

859 refImage=refImage, 

860 dcrWeights=dcrWeights) 

861 dcrModels.assign(dcrSubModelOut, bbox) 

862 

863 def dcrResiduals(self, residual, visitInfo, wcs, filterInfo): 

864 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 

865 

866 Parameters 

867 ---------- 

868 residual : `numpy.ndarray` 

869 The residual masked image for one exposure, 

870 after subtracting the matched template 

871 visitInfo : `lsst.afw.image.VisitInfo` 

872 Metadata for the exposure. 

873 wcs : `lsst.afw.geom.SkyWcs` 

874 Coordinate system definition (wcs) for the exposure. 

875 filterInfo : `lsst.afw.image.Filter` 

876 The filter definition, set in the current instruments' obs package. 

877 Required for any calculation of DCR, including making matched templates. 

878 

879 Yields 

880 ------ 

881 residualImage : `numpy.ndarray` 

882 The residual image for the next subfilter, shifted for DCR. 

883 """ 

884 # Pre-calculate the spline-filtered residual image, so that step can be 

885 # skipped in the shift calculation in `applyDcr`. 

886 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder) 

887 # Note that `splitSubfilters` is always turned off in the reverse direction. 

888 # This option introduces additional blurring if applied to the residuals. 

889 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters, 

890 splitSubfilters=False) 

891 for dcr in dcrShift: 

892 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False, 

893 doPrefilter=False, order=self.config.imageInterpOrder) 

894 

895 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, 

896 gain, modelWeights, refImage, dcrWeights): 

897 """Calculate a new DcrModel from a set of image residuals. 

898 

899 Parameters 

900 ---------- 

901 dcrModels : `lsst.pipe.tasks.DcrModel` 

902 Current model of the true sky after correcting chromatic effects. 

903 residualGeneratorList : `generator` of `numpy.ndarray` 

904 The residual image for the next subfilter, shifted for DCR. 

905 dcrBBox : `lsst.geom.box.Box2I` 

906 Sub-region of the coadd which includes a buffer to allow for DCR. 

907 statsCtrl : `lsst.afw.math.StatisticsControl` 

908 Statistics control object for coadd 

909 gain : `float` 

910 Relative weight to give the new solution when updating the model. 

911 modelWeights : `numpy.ndarray` or `float` 

912 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

913 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

914 refImage : `lsst.afw.image.Image` 

915 A reference image used to supply the default pixel values. 

916 dcrWeights : `list` of `lsst.afw.image.Image` 

917 Per-pixel weights for each subfilter. 

918 Equal to 1/(number of unmasked images contributing to each pixel). 

919 

920 Returns 

921 ------- 

922 dcrModel : `lsst.pipe.tasks.DcrModel` 

923 New model of the true sky after correcting chromatic effects. 

924 """ 

925 newModelImages = [] 

926 for subfilter, model in enumerate(dcrModels): 

927 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList] 

928 residual = np.sum(residualsList, axis=0) 

929 residual *= dcrWeights[subfilter][dcrBBox].array 

930 # `MaskedImage`s only support in-place addition, so rename for readability 

931 newModel = model[dcrBBox].clone() 

932 newModel.array += residual 

933 # Catch any invalid values 

934 badPixels = ~np.isfinite(newModel.array) 

935 newModel.array[badPixels] = model[dcrBBox].array[badPixels] 

936 if self.config.regularizeModelIterations > 0: 936 ↛ 940line 936 didn't jump to line 940, because the condition on line 936 was never false

937 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox, 

938 self.config.regularizeModelIterations, 

939 self.config.regularizationWidth) 

940 newModelImages.append(newModel) 

941 if self.config.regularizeModelFrequency > 0: 941 ↛ 945line 941 didn't jump to line 945, because the condition on line 941 was never false

942 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl, 

943 self.config.regularizeModelFrequency, 

944 self.config.regularizationWidth) 

945 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain) 

946 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights) 

947 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf, 

948 dcrModels.mask, dcrModels.variance) 

949 

950 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl): 

951 """Calculate a quality of fit metric for the matched templates. 

952 

953 Parameters 

954 ---------- 

955 dcrModels : `lsst.pipe.tasks.DcrModel` 

956 Best fit model of the true sky after correcting chromatic effects. 

957 subExposures : `dict` of `lsst.afw.image.ExposureF` 

958 The pre-loaded exposures for the current subregion. 

959 bbox : `lsst.geom.box.Box2I` 

960 Sub-region to coadd 

961 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

962 `lsst.daf.persistence.ButlerDataRef` 

963 The data references to the input warped exposures. 

964 weightList : `list` of `float` 

965 The weight to give each input exposure in the coadd 

966 statsCtrl : `lsst.afw.math.StatisticsControl` 

967 Statistics control object for coadd 

968 

969 Returns 

970 ------- 

971 convergenceMetric : `float` 

972 Quality of fit metric for all input exposures, within the sub-region 

973 """ 

974 significanceImage = np.abs(dcrModels.getReferenceImage(bbox)) 

975 nSigma = 3. 

976 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl, 

977 bufferSize=self.bufferSize) 

978 if np.max(significanceImage) == 0: 978 ↛ 979line 978 didn't jump to line 979, because the condition on line 978 was never true

979 significanceImage += 1. 

980 weight = 0 

981 metric = 0. 

982 metricList = {} 

983 for warpExpRef, expWeight in zip(warpRefList, weightList): 

984 visit = warpExpRef.dataId["visit"] 

985 exposure = subExposures[visit][bbox] 

986 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl) 

987 metric += singleMetric 

988 metricList[visit] = singleMetric 

989 weight += 1. 

990 self.log.info("Individual metrics:\n%s", metricList) 

991 return 1.0 if weight == 0.0 else metric/weight 

992 

993 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl): 

994 """Calculate a quality of fit metric for a single matched template. 

995 

996 Parameters 

997 ---------- 

998 dcrModels : `lsst.pipe.tasks.DcrModel` 

999 Best fit model of the true sky after correcting chromatic effects. 

1000 exposure : `lsst.afw.image.ExposureF` 

1001 The input warped exposure to evaluate. 

1002 significanceImage : `numpy.ndarray` 

1003 Array of weights for each pixel corresponding to its significance 

1004 for the convergence calculation. 

1005 statsCtrl : `lsst.afw.math.StatisticsControl` 

1006 Statistics control object for coadd 

1007 

1008 Returns 

1009 ------- 

1010 convergenceMetric : `float` 

1011 Quality of fit metric for one exposure, within the sub-region. 

1012 """ 

1013 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1014 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, 

1015 order=self.config.imageInterpOrder, 

1016 splitSubfilters=self.config.splitSubfilters, 

1017 splitThreshold=self.config.splitThreshold, 

1018 amplifyModel=self.config.accelerateModel) 

1019 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage 

1020 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2. 

1021 

1022 finitePixels = np.isfinite(diffVals) 

1023 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0 

1024 convergeMaskPixels = exposure.mask.array & convergeMask > 0 

1025 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels 

1026 if np.sum(usePixels) == 0: 1026 ↛ 1027line 1026 didn't jump to line 1027, because the condition on line 1026 was never true

1027 metric = 0. 

1028 else: 

1029 diffUse = diffVals[usePixels] 

1030 refUse = refVals[usePixels] 

1031 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse)) 

1032 return metric 

1033 

1034 def stackCoadd(self, dcrCoadds): 

1035 """Add a list of sub-band coadds together. 

1036 

1037 Parameters 

1038 ---------- 

1039 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1040 A list of coadd exposures, each exposure containing 

1041 the model for one subfilter. 

1042 

1043 Returns 

1044 ------- 

1045 coaddExposure : `lsst.afw.image.ExposureF` 

1046 A single coadd exposure that is the sum of the sub-bands. 

1047 """ 

1048 coaddExposure = dcrCoadds[0].clone() 

1049 for coadd in dcrCoadds[1:]: 

1050 coaddExposure.maskedImage += coadd.maskedImage 

1051 return coaddExposure 

1052 

1053 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, 

1054 mask=None, variance=None): 

1055 """Create a list of coadd exposures from a list of masked images. 

1056 

1057 Parameters 

1058 ---------- 

1059 dcrModels : `lsst.pipe.tasks.DcrModel` 

1060 Best fit model of the true sky after correcting chromatic effects. 

1061 skyInfo : `lsst.pipe.base.Struct` 

1062 Patch geometry information, from getSkyInfo 

1063 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1064 `lsst.daf.persistence.ButlerDataRef` 

1065 The data references to the input warped exposures. 

1066 weightList : `list` of `float` 

1067 The weight to give each input exposure in the coadd 

1068 calibration : `lsst.afw.Image.PhotoCalib`, optional 

1069 Scale factor to set the photometric calibration of an exposure. 

1070 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 

1071 A record of the observations that are included in the coadd. 

1072 mask : `lsst.afw.image.Mask`, optional 

1073 Optional mask to override the values in the final coadd. 

1074 variance : `lsst.afw.image.Image`, optional 

1075 Optional variance plane to override the values in the final coadd. 

1076 

1077 Returns 

1078 ------- 

1079 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 

1080 A list of coadd exposures, each exposure containing 

1081 the model for one subfilter. 

1082 """ 

1083 dcrCoadds = [] 

1084 refModel = dcrModels.getReferenceImage() 

1085 for model in dcrModels: 

1086 if self.config.accelerateModel > 1: 1086 ↛ 1088line 1086 didn't jump to line 1088, because the condition on line 1086 was never false

1087 model.array = (model.array - refModel)*self.config.accelerateModel + refModel 

1088 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

1089 if calibration is not None: 1089 ↛ 1091line 1089 didn't jump to line 1091, because the condition on line 1089 was never false

1090 coaddExposure.setPhotoCalib(calibration) 

1091 if coaddInputs is not None: 1091 ↛ 1094line 1091 didn't jump to line 1094, because the condition on line 1091 was never false

1092 coaddExposure.getInfo().setCoaddInputs(coaddInputs) 

1093 # Set the metadata for the coadd, including PSF and aperture corrections. 

1094 self.assembleMetadata(coaddExposure, warpRefList, weightList) 

1095 # Overwrite the PSF 

1096 coaddExposure.setPsf(dcrModels.psf) 

1097 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox]) 

1098 maskedImage = afwImage.MaskedImageF(dcrModels.bbox) 

1099 maskedImage.image = model 

1100 maskedImage.mask = dcrModels.mask 

1101 maskedImage.variance = dcrModels.variance 

1102 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox]) 

1103 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

1104 if mask is not None: 1104 ↛ 1106line 1104 didn't jump to line 1106, because the condition on line 1104 was never false

1105 coaddExposure.setMask(mask) 

1106 if variance is not None: 1106 ↛ 1108line 1106 didn't jump to line 1108, because the condition on line 1106 was never false

1107 coaddExposure.setVariance(variance) 

1108 dcrCoadds.append(coaddExposure) 

1109 return dcrCoadds 

1110 

1111 def calculateGain(self, convergenceList, gainList): 

1112 """Calculate the gain to use for the current iteration. 

1113 

1114 After calculating a new DcrModel, each value is averaged with the 

1115 value in the corresponding pixel from the previous iteration. This 

1116 reduces oscillating solutions that iterative techniques are plagued by, 

1117 and speeds convergence. By far the biggest changes to the model 

1118 happen in the first couple iterations, so we can also use a more 

1119 aggressive gain later when the model is changing slowly. 

1120 

1121 Parameters 

1122 ---------- 

1123 convergenceList : `list` of `float` 

1124 The quality of fit metric from each previous iteration. 

1125 gainList : `list` of `float` 

1126 The gains used in each previous iteration: appended with the new 

1127 gain value. 

1128 Gains are numbers between ``self.config.baseGain`` and 1. 

1129 

1130 Returns 

1131 ------- 

1132 gain : `float` 

1133 Relative weight to give the new solution when updating the model. 

1134 A value of 1.0 gives equal weight to both solutions. 

1135 

1136 Raises 

1137 ------ 

1138 ValueError 

1139 If ``len(convergenceList) != len(gainList)+1``. 

1140 """ 

1141 nIter = len(convergenceList) 

1142 if nIter != len(gainList) + 1: 

1143 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)." 

1144 % (len(convergenceList), len(gainList))) 

1145 

1146 if self.config.baseGain is None: 

1147 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters 

1148 # The more subfilters being modeled, the lower the gain should be. 

1149 baseGain = 1./(self.config.dcrNumSubfilters - 1) 

1150 else: 

1151 baseGain = self.config.baseGain 

1152 

1153 if self.config.useProgressiveGain and nIter > 2: 

1154 # To calculate the best gain to use, compare the past gains that have been used 

1155 # with the resulting convergences to estimate the best gain to use. 

1156 # Algorithmically, this is a Kalman filter. 

1157 # If forward modeling proceeds perfectly, the convergence metric should 

1158 # asymptotically approach a final value. 

1159 # We can estimate that value from the measured changes in convergence 

1160 # weighted by the gains used in each previous iteration. 

1161 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i] 

1162 for i in range(nIter - 1)] 

1163 # The convergence metric is strictly positive, so if the estimated final convergence is 

1164 # less than zero, force it to zero. 

1165 estFinalConv = np.array(estFinalConv) 

1166 estFinalConv[estFinalConv < 0] = 0 

1167 # Because the estimate may slowly change over time, only use the most recent measurements. 

1168 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):]) 

1169 lastGain = gainList[-1] 

1170 lastConv = convergenceList[-2] 

1171 newConv = convergenceList[-1] 

1172 # The predicted convergence is the value we would get if the new model calculated 

1173 # in the previous iteration was perfect. Recall that the updated model that is 

1174 # actually used is the gain-weighted average of the new and old model, 

1175 # so the convergence would be similarly weighted. 

1176 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain) 

1177 # If the measured and predicted convergence are very close, that indicates 

1178 # that our forward model is accurate and we can use a more aggressive gain 

1179 # If the measured convergence is significantly worse (or better!) than predicted, 

1180 # that indicates that the model is not converging as expected and 

1181 # we should use a more conservative gain. 

1182 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain)) 

1183 newGain = 1 - abs(delta) 

1184 # Average the gains to prevent oscillating solutions. 

1185 newGain = (newGain + lastGain)/2. 

1186 gain = max(baseGain, newGain) 

1187 else: 

1188 gain = baseGain 

1189 gainList.append(gain) 

1190 return gain 

1191 

1192 def calculateModelWeights(self, dcrModels, dcrBBox): 

1193 """Build an array that smoothly tapers to 0 away from detected sources. 

1194 

1195 Parameters 

1196 ---------- 

1197 dcrModels : `lsst.pipe.tasks.DcrModel` 

1198 Best fit model of the true sky after correcting chromatic effects. 

1199 dcrBBox : `lsst.geom.box.Box2I` 

1200 Sub-region of the coadd which includes a buffer to allow for DCR. 

1201 

1202 Returns 

1203 ------- 

1204 weights : `numpy.ndarray` or `float` 

1205 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1206 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1207 

1208 Raises 

1209 ------ 

1210 ValueError 

1211 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 

1212 """ 

1213 if not self.config.useModelWeights: 1213 ↛ 1214line 1213 didn't jump to line 1214, because the condition on line 1213 was never true

1214 return 1.0 

1215 if self.config.modelWeightsWidth < 0: 1215 ↛ 1216line 1215 didn't jump to line 1216, because the condition on line 1215 was never true

1216 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set") 

1217 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes) 

1218 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0 

1219 weights = np.zeros_like(dcrModels[0][dcrBBox].array) 

1220 weights[convergeMaskPixels] = 1. 

1221 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth) 

1222 weights /= np.max(weights) 

1223 return weights 

1224 

1225 def applyModelWeights(self, modelImages, refImage, modelWeights): 

1226 """Smoothly replace model pixel values with those from a 

1227 reference at locations away from detected sources. 

1228 

1229 Parameters 

1230 ---------- 

1231 modelImages : `list` of `lsst.afw.image.Image` 

1232 The new DCR model images from the current iteration. 

1233 The values will be modified in place. 

1234 refImage : `lsst.afw.image.MaskedImage` 

1235 A reference image used to supply the default pixel values. 

1236 modelWeights : `numpy.ndarray` or `float` 

1237 A 2D array of weight values that tapers smoothly to zero away from detected sources. 

1238 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 

1239 """ 

1240 if self.config.useModelWeights: 1240 ↛ exitline 1240 didn't return from function 'applyModelWeights', because the condition on line 1240 was never false

1241 for model in modelImages: 

1242 model.array *= modelWeights 

1243 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters 

1244 

1245 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList): 

1246 """Pre-load sub-regions of a list of exposures. 

1247 

1248 Parameters 

1249 ---------- 

1250 bbox : `lsst.geom.box.Box2I` 

1251 Sub-region to coadd 

1252 statsCtrl : `lsst.afw.math.StatisticsControl` 

1253 Statistics control object for coadd 

1254 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1255 `lsst.daf.persistence.ButlerDataRef` 

1256 The data references to the input warped exposures. 

1257 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 

1258 The image scalars correct for the zero point of the exposures. 

1259 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 

1260 Each element is dict with keys = mask plane name to add the spans to 

1261 

1262 Returns 

1263 ------- 

1264 subExposures : `dict` 

1265 The `dict` keys are the visit IDs, 

1266 and the values are `lsst.afw.image.ExposureF` 

1267 The pre-loaded exposures for the current subregion. 

1268 The variance plane contains weights, and not the variance 

1269 """ 

1270 tempExpName = self.getTempExpDatasetName(self.warpType) 

1271 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList) 

1272 subExposures = {} 

1273 for warpExpRef, imageScaler, altMaskSpans in zipIterables: 

1274 if isinstance(warpExpRef, DeferredDatasetHandle): 

1275 exposure = warpExpRef.get(parameters={'bbox': bbox}) 

1276 else: 

1277 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox) 

1278 visit = warpExpRef.dataId["visit"] 

1279 if altMaskSpans is not None: 1279 ↛ 1281line 1279 didn't jump to line 1281, because the condition on line 1279 was never false

1280 self.applyAltMaskPlanes(exposure.mask, altMaskSpans) 

1281 imageScaler.scaleMaskedImage(exposure.maskedImage) 

1282 # Note that the variance plane here is used to store weights, not the actual variance 

1283 exposure.variance.array[:, :] = 0. 

1284 # Set the weight of unmasked pixels to 1. 

1285 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1. 

1286 # Set the image value of masked pixels to zero. 

1287 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual`` 

1288 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0. 

1289 subExposures[visit] = exposure 

1290 return subExposures 

1291 

1292 def selectCoaddPsf(self, templateCoadd, warpRefList): 

1293 """Compute the PSF of the coadd from the exposures with the best seeing. 

1294 

1295 Parameters 

1296 ---------- 

1297 templateCoadd : `lsst.afw.image.ExposureF` 

1298 The initial coadd exposure before accounting for DCR. 

1299 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or 

1300 `lsst.daf.persistence.ButlerDataRef` 

1301 The data references to the input warped exposures. 

1302 

1303 Returns 

1304 ------- 

1305 psf : `lsst.meas.algorithms.CoaddPsf` 

1306 The average PSF of the input exposures with the best seeing. 

1307 """ 

1308 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.)) 

1309 tempExpName = self.getTempExpDatasetName(self.warpType) 

1310 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit 

1311 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef`` 

1312 ccds = templateCoadd.getInfo().getCoaddInputs().ccds 

1313 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm 

1314 psfSizes = np.zeros(len(ccds)) 

1315 ccdVisits = np.array(ccds["visit"]) 

1316 for warpExpRef in warpRefList: 

1317 if isinstance(warpExpRef, DeferredDatasetHandle): 

1318 # Gen 3 API 

1319 psf = warpExpRef.get(component="psf") 

1320 else: 

1321 # Gen 2 API. Delete this when Gen 2 retired 

1322 psf = warpExpRef.get(tempExpName).getPsf() 

1323 visit = warpExpRef.dataId["visit"] 

1324 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm 

1325 psfSizes[ccdVisits == visit] = psfSize 

1326 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd 

1327 # The selected PSFs are those that have a FWHM less than or equal to the smaller 

1328 # of the mean or median FWHM of the input exposures. 

1329 sizeThreshold = min(np.median(psfSizes), psfRefSize) 

1330 goodPsfs = psfSizes <= sizeThreshold 

1331 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), 

1332 self.config.coaddPsf.makeControl()) 

1333 return psf