Coverage for python/lsst/pipe/tasks/makeWarp.py: 15%

280 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-29 10:41 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["MakeWarpTask", "MakeWarpConfig"] 

23 

24from deprecated.sphinx import deprecated 

25import logging 

26import numpy 

27 

28import lsst.pex.config as pexConfig 

29import lsst.afw.image as afwImage 

30import lsst.coadd.utils as coaddUtils 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as connectionTypes 

33import lsst.utils as utils 

34import lsst.geom 

35from lsst.daf.butler import DeferredDatasetHandle 

36from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

37from lsst.skymap import BaseSkyMap 

38from lsst.utils.timer import timeMethod 

39from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

40from .warpAndPsfMatch import WarpAndPsfMatchTask 

41from collections.abc import Iterable 

42 

43log = logging.getLogger(__name__) 

44 

45 

46class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

48 defaultTemplates={"coaddName": "deep", 

49 "skyWcsName": "gbdesAstrometricFit", 

50 "photoCalibName": "fgcm", 

51 "calexpType": ""}): 

52 calExpList = connectionTypes.Input( 

53 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

54 name="{calexpType}calexp", 

55 storageClass="ExposureF", 

56 dimensions=("instrument", "visit", "detector"), 

57 multiple=True, 

58 deferLoad=True, 

59 ) 

60 backgroundList = connectionTypes.Input( 

61 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

62 name="calexpBackground", 

63 storageClass="Background", 

64 dimensions=("instrument", "visit", "detector"), 

65 multiple=True, 

66 ) 

67 skyCorrList = connectionTypes.Input( 

68 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

69 name="skyCorr", 

70 storageClass="Background", 

71 dimensions=("instrument", "visit", "detector"), 

72 multiple=True, 

73 ) 

74 skyMap = connectionTypes.Input( 

75 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

76 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

77 storageClass="SkyMap", 

78 dimensions=("skymap",), 

79 ) 

80 externalSkyWcsTractCatalog = connectionTypes.Input( 

81 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

82 "id for the catalog id, sorted on id for fast lookup."), 

83 name="{skyWcsName}SkyWcsCatalog", 

84 storageClass="ExposureCatalog", 

85 dimensions=("instrument", "visit", "tract"), 

86 ) 

87 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

88 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

89 "These catalogs use the detector id for the catalog id, sorted on id for " 

90 "fast lookup."), 

91 name="finalVisitSummary", 

92 storageClass="ExposureCatalog", 

93 dimensions=("instrument", "visit"), 

94 ) 

95 externalPhotoCalibTractCatalog = connectionTypes.Input( 

96 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

97 "detector id for the catalog id, sorted on id for fast lookup."), 

98 name="{photoCalibName}PhotoCalibCatalog", 

99 storageClass="ExposureCatalog", 

100 dimensions=("instrument", "visit", "tract"), 

101 ) 

102 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

103 doc=("Per-visit photometric calibrations computed globally (with no tract " 

104 "information). These catalogs use the detector id for the catalog id, " 

105 "sorted on id for fast lookup."), 

106 name="finalVisitSummary", 

107 storageClass="ExposureCatalog", 

108 dimensions=("instrument", "visit"), 

109 ) 

110 finalizedPsfApCorrCatalog = connectionTypes.Input( 

111 doc=("Per-visit finalized psf models and aperture correction maps. " 

112 "These catalogs use the detector id for the catalog id, " 

113 "sorted on id for fast lookup."), 

114 name="finalVisitSummary", 

115 storageClass="ExposureCatalog", 

116 dimensions=("instrument", "visit"), 

117 ) 

118 direct = connectionTypes.Output( 

119 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

120 "calexps onto the skyMap patch geometry."), 

121 name="{coaddName}Coadd_directWarp", 

122 storageClass="ExposureF", 

123 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

124 ) 

125 psfMatched = connectionTypes.Output( 

126 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

127 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

128 name="{coaddName}Coadd_psfMatchedWarp", 

129 storageClass="ExposureF", 

130 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

131 ) 

132 # TODO DM-28769, have selectImages subtask indicate which connections they 

133 # need: 

134 wcsList = connectionTypes.Input( 

135 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

136 name="{calexpType}calexp.wcs", 

137 storageClass="Wcs", 

138 dimensions=("instrument", "visit", "detector"), 

139 multiple=True, 

140 ) 

141 bboxList = connectionTypes.Input( 

142 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

143 name="{calexpType}calexp.bbox", 

144 storageClass="Box2I", 

145 dimensions=("instrument", "visit", "detector"), 

146 multiple=True, 

147 ) 

148 visitSummary = connectionTypes.Input( 

149 doc="Consolidated exposure metadata", 

150 name="finalVisitSummary", 

151 storageClass="ExposureCatalog", 

152 dimensions=("instrument", "visit",), 

153 ) 

154 

155 def __init__(self, *, config=None): 

156 super().__init__(config=config) 

157 if config.bgSubtracted: 

158 self.inputs.remove("backgroundList") 

159 if not config.doApplySkyCorr: 

160 self.inputs.remove("skyCorrList") 

161 if config.doApplyExternalSkyWcs: 

162 if config.useGlobalExternalSkyWcs: 

163 self.inputs.remove("externalSkyWcsTractCatalog") 

164 else: 

165 self.inputs.remove("externalSkyWcsGlobalCatalog") 

166 else: 

167 self.inputs.remove("externalSkyWcsTractCatalog") 

168 self.inputs.remove("externalSkyWcsGlobalCatalog") 

169 if config.doApplyExternalPhotoCalib: 

170 if config.useGlobalExternalPhotoCalib: 

171 self.inputs.remove("externalPhotoCalibTractCatalog") 

172 else: 

173 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

174 else: 

175 self.inputs.remove("externalPhotoCalibTractCatalog") 

176 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

177 if not config.doApplyFinalizedPsf: 

178 self.inputs.remove("finalizedPsfApCorrCatalog") 

179 if not config.makeDirect: 

180 self.outputs.remove("direct") 

181 if not config.makePsfMatched: 

182 self.outputs.remove("psfMatched") 

183 # TODO DM-28769: add connection per selectImages connections 

184 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

185 self.inputs.remove("visitSummary") 

186 

187 

188class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, 

189 pipelineConnections=MakeWarpConnections): 

190 """Config for MakeWarpTask.""" 

191 

192 warpAndPsfMatch = pexConfig.ConfigurableField( 

193 target=WarpAndPsfMatchTask, 

194 doc="Task to warp and PSF-match calexp", 

195 ) 

196 doWrite = pexConfig.Field( 

197 doc="persist <coaddName>Coadd_<warpType>Warp", 

198 dtype=bool, 

199 default=True, 

200 ) 

201 bgSubtracted = pexConfig.Field( 

202 doc="Work with a background subtracted calexp?", 

203 dtype=bool, 

204 default=True, 

205 ) 

206 coaddPsf = pexConfig.ConfigField( 

207 doc="Configuration for CoaddPsf", 

208 dtype=CoaddPsfConfig, 

209 ) 

210 makeDirect = pexConfig.Field( 

211 doc="Make direct Warp/Coadds", 

212 dtype=bool, 

213 default=True, 

214 ) 

215 makePsfMatched = pexConfig.Field( 

216 doc="Make Psf-Matched Warp/Coadd?", 

217 dtype=bool, 

218 default=False, 

219 ) 

220 doWriteEmptyWarps = pexConfig.Field( 

221 dtype=bool, 

222 default=False, 

223 doc="Write out warps even if they are empty" 

224 ) 

225 hasFakes = pexConfig.Field( 

226 doc="Should be set to True if fake sources have been inserted into the input data.", 

227 dtype=bool, 

228 default=False, 

229 ) 

230 doApplySkyCorr = pexConfig.Field( 

231 dtype=bool, 

232 default=False, 

233 doc="Apply sky correction?", 

234 ) 

235 doApplyFinalizedPsf = pexConfig.Field( 

236 doc="Whether to apply finalized psf models and aperture correction map.", 

237 dtype=bool, 

238 default=True, 

239 ) 

240 

241 def validate(self): 

242 CoaddBaseTask.ConfigClass.validate(self) 

243 

244 if not self.makePsfMatched and not self.makeDirect: 

245 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

246 if self.doPsfMatch: 

247 # Backwards compatibility. 

248 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

249 self.makePsfMatched = True 

250 self.makeDirect = False 

251 

252 def setDefaults(self): 

253 CoaddBaseTask.ConfigClass.setDefaults(self) 

254 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

255 

256 

257class MakeWarpTask(CoaddBaseTask): 

258 """Warp and optionally PSF-Match calexps onto an a common projection. 

259 

260 Warp and optionally PSF-Match calexps onto a common projection, by 

261 performing the following operations: 

262 - Group calexps by visit/run 

263 - For each visit, generate a Warp by calling method @ref run. 

264 `run` loops over the visit's calexps calling 

265 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit 

266 

267 Notes 

268 ----- 

269 WarpType identifies the types of convolutions applied to Warps 

270 (previously CoaddTempExps). Only two types are available: direct 

271 (for regular Warps/Coadds) and psfMatched(for Warps/Coadds with 

272 homogenized PSFs). We expect to add a third type, likelihood, for 

273 generating likelihood Coadds with Warps that have been correlated with 

274 their own PSF. 

275 

276 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

277 `~lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask` 

278 is responsible for the PSF-Matching, and its config is accessed via 

279 `config.warpAndPsfMatch.psfMatch`. 

280 

281 The optimal configuration depends on aspects of dataset: the pixel scale, 

282 average PSF FWHM and dimensions of the PSF kernel. These configs include 

283 the requested model PSF, the matching kernel size, padding of the science 

284 PSF thumbnail and spatial sampling frequency of the PSF. 

285 

286 *Config Guidelines*: The user must specify the size of the model PSF to 

287 which to match by setting `config.modelPsf.defaultFwhm` in units of pixels. 

288 The appropriate values depends on science case. In general, for a set of 

289 input images, this config should equal the FWHM of the visit with the worst 

290 seeing. The smallest it should be set to is the median FWHM. The defaults 

291 of the other config options offer a reasonable starting point. 

292 

293 The following list presents the most common problems that arise from a 

294 misconfigured 

295 @link ip::diffim::modelPsfMatch::ModelPsfMatchTask ModelPsfMatchTask 

296 @endlink 

297 and corresponding solutions. All assume the default Alard-Lupton kernel, 

298 with configs accessed via 

299 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list 

300 is formatted as: 

301 Problem: Explanation. *Solution* 

302 

303 *Troublshooting PSF-Matching Configuration:* 

304 - Matched PSFs look boxy: The matching kernel is too small. 

305 _Increase the matching kernel size. 

306 For example:_ 

307 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

308 # default 21 

309 Note that increasing the kernel size also increases runtime. 

310 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find 

311 good solution for matching kernel. 

312 _Provide the matcher with more data by either increasing 

313 the spatial sampling by decreasing the spatial cell size,_ 

314 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 

315 # default 128 

316 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 

317 # default 128 

318 _or increasing the padding around the Science PSF, for example:_ 

319 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

320 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF 

321 dimensions to the matching kernel dimensions, thus increasing the 

322 number of pixels available to fit after convolving the PSF with the 

323 matching kernel. Optionally, for debugging the effects of padding, the 

324 level of padding may be manually controlled by setting turning off the 

325 automatic padding and setting the number of pixels by which to pad the 

326 PSF: 

327 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False 

328 # default True 

329 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 

330 # pixels. default 0 

331 - Deconvolution: Matching a large PSF to a smaller PSF produces 

332 a telltale noise pattern which looks like ripples or a brain. 

333 _Increase the size of the requested model PSF. For example:_ 

334 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of 

335 pixels. 

336 - High frequency (sometimes checkered) noise: The matching basis functions 

337 are too small. 

338 _Increase the width of the Gaussian basis functions. For example:_ 

339 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss= 

340 [1.5, 3.0, 6.0] # from default [0.7, 1.5, 3.0] 

341 """ 

342 ConfigClass = MakeWarpConfig 

343 _DefaultName = "makeWarp" 

344 

345 def __init__(self, **kwargs): 

346 CoaddBaseTask.__init__(self, **kwargs) 

347 self.makeSubtask("warpAndPsfMatch") 

348 if self.config.hasFakes: 

349 self.calexpType = "fakes_calexp" 

350 else: 

351 self.calexpType = "calexp" 

352 

353 @utils.inheritDoc(pipeBase.PipelineTask) 

354 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

355 # Obtain the list of input detectors from calExpList. Sort them by 

356 # detector order (to ensure reproducibility). Then ensure all input 

357 # lists are in the same sorted detector order. 

358 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

359 detectorOrder.sort() 

360 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

361 

362 # Read in all inputs. 

363 inputs = butlerQC.get(inputRefs) 

364 

365 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

366 # from the dictionary so we can pass it as kwargs later. 

367 skyMap = inputs.pop("skyMap") 

368 quantumDataId = butlerQC.quantum.dataId 

369 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

370 

371 # Construct list of input DataIds expected by `run`. 

372 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

373 # Construct list of packed integer IDs expected by `run`. 

374 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

375 

376 if self.config.doApplyExternalSkyWcs: 

377 if self.config.useGlobalExternalSkyWcs: 

378 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

379 else: 

380 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

381 else: 

382 externalSkyWcsCatalog = None 

383 

384 if self.config.doApplyExternalPhotoCalib: 

385 if self.config.useGlobalExternalPhotoCalib: 

386 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

387 else: 

388 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

389 else: 

390 externalPhotoCalibCatalog = None 

391 

392 if self.config.doApplyFinalizedPsf: 

393 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog") 

394 else: 

395 finalizedPsfApCorrCatalog = None 

396 

397 # Do an initial selection on inputs with complete wcs/photoCalib info. 

398 # Qualifying calexps will be read in the following call. 

399 completeIndices = self._prepareCalibratedExposures( 

400 **inputs, 

401 externalSkyWcsCatalog=externalSkyWcsCatalog, 

402 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

403 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

404 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

405 

406 # Do another selection based on the configured selection task 

407 # (using updated WCSs to determine patch overlap if an external 

408 # calibration was applied). 

409 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

410 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

411 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

412 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

413 

414 # Extract integer visitId requested by `run`. 

415 visitId = dataIdList[0]["visit"] 

416 

417 results = self.run(**inputs, visitId=visitId, 

418 ccdIdList=[ccdIdList[i] for i in goodIndices], 

419 dataIdList=[dataIdList[i] for i in goodIndices], 

420 skyInfo=skyInfo) 

421 if self.config.makeDirect and results.exposures["direct"] is not None: 

422 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

423 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

424 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

425 

426 @timeMethod 

427 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

428 """Create a Warp from inputs. 

429 

430 We iterate over the multiple calexps in a single exposure to construct 

431 the warp (previously called a coaddTempExp) of that exposure to the 

432 supplied tract/patch. 

433 

434 Pixels that receive no pixels are set to NAN; this is not correct 

435 (violates LSST algorithms group policy), but will be fixed up by 

436 interpolating after the coaddition. 

437 

438 calexpRefList : `list` 

439 List of data references for calexps that (may) 

440 overlap the patch of interest. 

441 skyInfo : `lsst.pipe.base.Struct` 

442 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

443 geometric information about the patch. 

444 visitId : `int` 

445 Integer identifier for visit, for the table that will 

446 produce the CoaddPsf. 

447 

448 Returns 

449 ------- 

450 result : `lsst.pipe.base.Struct` 

451 Results as a struct with attributes: 

452 

453 ``exposures`` 

454 A dictionary containing the warps requested: 

455 "direct": direct warp if ``config.makeDirect`` 

456 "psfMatched": PSF-matched warp if ``config.makePsfMatched`` 

457 (`dict`). 

458 """ 

459 warpTypeList = self.getWarpTypeList() 

460 

461 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

462 didSetMetadata = {warpType: False for warpType in warpTypeList} 

463 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

464 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

465 for warpType in warpTypeList} 

466 

467 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

468 if dataIdList is None: 

469 dataIdList = ccdIdList 

470 

471 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

472 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

473 calExpInd+1, len(calExpList), dataId) 

474 # TODO: The following conditional is only required for backwards 

475 # compatibility with the deprecated prepareCalibratedExposures() 

476 # method. Can remove with its removal after the deprecation 

477 # period. 

478 if isinstance(calExp, DeferredDatasetHandle): 

479 calExp = calExp.get() 

480 try: 

481 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

482 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

483 makeDirect=self.config.makeDirect, 

484 makePsfMatched=self.config.makePsfMatched) 

485 except Exception as e: 

486 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

487 continue 

488 try: 

489 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

490 for warpType in warpTypeList: 

491 exposure = warpedAndMatched.getDict()[warpType] 

492 if exposure is None: 

493 continue 

494 warp = warps[warpType] 

495 if didSetMetadata[warpType]: 

496 mimg = exposure.getMaskedImage() 

497 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

498 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

499 del mimg 

500 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

501 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

502 totGoodPix[warpType] += numGoodPix[warpType] 

503 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

504 dataId, numGoodPix[warpType], 

505 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

506 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

507 warp.info.id = exposure.info.id 

508 warp.setPhotoCalib(exposure.getPhotoCalib()) 

509 warp.setFilter(exposure.getFilter()) 

510 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

511 # PSF replaced with CoaddPsf after loop if and only if 

512 # creating direct warp. 

513 warp.setPsf(exposure.getPsf()) 

514 didSetMetadata[warpType] = True 

515 

516 # Need inputRecorder for CoaddApCorrMap for both direct and 

517 # PSF-matched. 

518 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

519 

520 except Exception as e: 

521 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

522 continue 

523 

524 for warpType in warpTypeList: 

525 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

526 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

527 

528 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

529 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType]) 

530 if warpType == "direct": 

531 warps[warpType].setPsf( 

532 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

533 self.config.coaddPsf.makeControl())) 

534 else: 

535 if not self.config.doWriteEmptyWarps: 

536 # No good pixels. Exposure still empty. 

537 warps[warpType] = None 

538 # NoWorkFound is unnecessary as the downstream tasks will 

539 # adjust the quantum accordingly. 

540 

541 result = pipeBase.Struct(exposures=warps) 

542 return result 

543 

544 def filterInputs(self, indices, inputs): 

545 """Filter task inputs by their indices. 

546 

547 Parameters 

548 ---------- 

549 indices : `list` [`int`] 

550 inputs : `dict` [`list`] 

551 A dictionary of input connections to be passed to run. 

552 

553 Returns 

554 ------- 

555 inputs : `dict` [`list`] 

556 Task inputs with their lists filtered by indices. 

557 """ 

558 for key in inputs.keys(): 

559 # Only down-select on list inputs 

560 if isinstance(inputs[key], list): 

561 inputs[key] = [inputs[key][ind] for ind in indices] 

562 return inputs 

563 

564 @deprecated(reason="This method is deprecated in favor of its leading underscore version, " 

565 "_prepareCalibratedfExposures(). Will be removed after v25.", 

566 version="v25.0", category=FutureWarning) 

567 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

568 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

569 finalizedPsfApCorrCatalog=None, 

570 **kwargs): 

571 """Deprecated function. 

572 

573 Please use _prepareCalibratedExposure(), which this delegates to and 

574 noting its slightly updated API, instead. 

575 """ 

576 # Read in all calexps. 

577 calExpList = [ref.get() for ref in calExpList] 

578 # Populate wcsList as required by new underscored version of function. 

579 wcsList = [calexp.getWcs() for calexp in calExpList] 

580 

581 indices = self._prepareCalibratedExposures(calExpList=calExpList, wcsList=wcsList, 

582 backgroundList=backgroundList, skyCorrList=skyCorrList, 

583 externalSkyWcsCatalog=externalSkyWcsCatalog, 

584 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

585 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

586 return indices 

587 

588 def _prepareCalibratedExposures(self, calExpList=[], wcsList=None, backgroundList=None, skyCorrList=None, 

589 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

590 finalizedPsfApCorrCatalog=None, **kwargs): 

591 """Calibrate and add backgrounds to input calExpList in place. 

592 

593 Parameters 

594 ---------- 

595 calExpList : `list` [`lsst.afw.image.Exposure` or 

596 `lsst.daf.butler.DeferredDatasetHandle`] 

597 Sequence of calexps to be modified in place. 

598 wcsList : `list` [`lsst.afw.geom.SkyWcs`] 

599 The WCSs of the calexps in ``calExpList``. When 

600 ``externalSkyCatalog`` is `None`, these are used to determine if 

601 the calexp should be included in the warp, namely checking that it 

602 is not `None`. If ``externalSkyCatalog`` is not `None`, this list 

603 will be dynamically updated with the external sky WCS. 

604 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional 

605 Sequence of backgrounds to be added back in if bgSubtracted=False. 

606 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional 

607 Sequence of background corrections to be subtracted if 

608 doApplySkyCorr=True. 

609 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

610 Exposure catalog with external skyWcs to be applied 

611 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

612 for the catalog id, sorted on id for fast lookup. 

613 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

614 Exposure catalog with external photoCalib to be applied 

615 if config.doApplyExternalPhotoCalib=True. Catalog uses the 

616 detector id for the catalog id, sorted on id for fast lookup. 

617 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

618 Exposure catalog with finalized psf models and aperture correction 

619 maps to be applied if config.doApplyFinalizedPsf=True. Catalog 

620 uses the detector id for the catalog id, sorted on id for fast 

621 lookup. 

622 **kwargs 

623 Additional keyword arguments. 

624 

625 Returns 

626 ------- 

627 indices : `list` [`int`] 

628 Indices of ``calExpList`` and friends that have valid 

629 photoCalib/skyWcs. 

630 """ 

631 wcsList = len(calExpList)*[None] if wcsList is None else wcsList 

632 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

633 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

634 

635 includeCalibVar = self.config.includeCalibVar 

636 

637 indices = [] 

638 for index, (calexp, wcs, background, skyCorr) in enumerate(zip(calExpList, 

639 wcsList, 

640 backgroundList, 

641 skyCorrList)): 

642 if externalSkyWcsCatalog is None and wcs is None: 

643 self.log.warning("Detector id %d for visit %d has None for skyWcs and will not be " 

644 "used in the warp", calexp.dataId["detector"], calexp.dataId["visit"]) 

645 continue 

646 

647 if isinstance(calexp, DeferredDatasetHandle): 

648 calexp = calexp.get() 

649 

650 if not self.config.bgSubtracted: 

651 calexp.maskedImage += background.getImage() 

652 

653 detectorId = calexp.getInfo().getDetector().getId() 

654 

655 # Find the external photoCalib. 

656 if externalPhotoCalibCatalog is not None: 

657 row = externalPhotoCalibCatalog.find(detectorId) 

658 if row is None: 

659 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

660 "and will not be used in the warp.", detectorId) 

661 continue 

662 photoCalib = row.getPhotoCalib() 

663 if photoCalib is None: 

664 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

665 "and will not be used in the warp.", detectorId) 

666 continue 

667 calexp.setPhotoCalib(photoCalib) 

668 else: 

669 photoCalib = calexp.getPhotoCalib() 

670 if photoCalib is None: 

671 self.log.warning("Detector id %s has None for photoCalib in the calexp " 

672 "and will not be used in the warp.", detectorId) 

673 continue 

674 

675 # Find and apply external skyWcs. 

676 if externalSkyWcsCatalog is not None: 

677 row = externalSkyWcsCatalog.find(detectorId) 

678 if row is None: 

679 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

680 "and will not be used in the warp.", detectorId) 

681 continue 

682 skyWcs = row.getWcs() 

683 wcsList[index] = skyWcs 

684 if skyWcs is None: 

685 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

686 "and will not be used in the warp.", detectorId) 

687 continue 

688 calexp.setWcs(skyWcs) 

689 else: 

690 skyWcs = calexp.getWcs() 

691 wcsList[index] = skyWcs 

692 if skyWcs is None: 

693 self.log.warning("Detector id %s has None for skyWcs in the calexp " 

694 "and will not be used in the warp.", detectorId) 

695 continue 

696 

697 # Find and apply finalized psf and aperture correction. 

698 if finalizedPsfApCorrCatalog is not None: 

699 row = finalizedPsfApCorrCatalog.find(detectorId) 

700 if row is None: 

701 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog " 

702 "and will not be used in the warp.", detectorId) 

703 continue 

704 psf = row.getPsf() 

705 if psf is None: 

706 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog " 

707 "and will not be used in the warp.", detectorId) 

708 continue 

709 calexp.setPsf(psf) 

710 apCorrMap = row.getApCorrMap() 

711 if apCorrMap is None: 

712 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog " 

713 "and will not be used in the warp.", detectorId) 

714 continue 

715 calexp.info.setApCorrMap(apCorrMap) 

716 

717 # Calibrate the image. 

718 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

719 includeScaleUncertainty=includeCalibVar) 

720 calexp.maskedImage /= photoCalib.getCalibrationMean() 

721 # TODO: The images will have a calibration of 1.0 everywhere once 

722 # RFC-545 is implemented. 

723 # exposure.setCalib(afwImage.Calib(1.0)) 

724 

725 # Apply skycorr 

726 if self.config.doApplySkyCorr: 

727 calexp.maskedImage -= skyCorr.getImage() 

728 

729 indices.append(index) 

730 calExpList[index] = calexp 

731 

732 return indices 

733 

734 @staticmethod 

735 def _prepareEmptyExposure(skyInfo): 

736 """Produce an empty exposure for a given patch. 

737 

738 Parameters 

739 ---------- 

740 skyInfo : `lsst.pipe.base.Struct` 

741 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

742 geometric information about the patch. 

743 

744 Returns 

745 ------- 

746 exp : `lsst.afw.image.exposure.ExposureF` 

747 An empty exposure for a given patch. 

748 """ 

749 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

750 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

751 .getPlaneBitMask("NO_DATA"), numpy.inf) 

752 return exp 

753 

754 def getWarpTypeList(self): 

755 """Return list of requested warp types per the config. 

756 """ 

757 warpTypeList = [] 

758 if self.config.makeDirect: 

759 warpTypeList.append("direct") 

760 if self.config.makePsfMatched: 

761 warpTypeList.append("psfMatched") 

762 return warpTypeList 

763 

764 

765def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

766 """Reorder inputRefs per outputSortKeyOrder. 

767 

768 Any inputRefs which are lists will be resorted per specified key e.g., 

769 'detector.' Only iterables will be reordered, and values can be of type 

770 `lsst.pipe.base.connections.DeferredDatasetRef` or 

771 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

772 

773 Returned lists of refs have the same length as the outputSortKeyOrder. 

774 If an outputSortKey not in the inputRef, then it will be padded with None. 

775 If an inputRef contains an inputSortKey that is not in the 

776 outputSortKeyOrder it will be removed. 

777 

778 Parameters 

779 ---------- 

780 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

781 Input references to be reordered and padded. 

782 outputSortKeyOrder : `iterable` 

783 Iterable of values to be compared with inputRef's dataId[dataIdKey]. 

784 dataIdKey : `str` 

785 The data ID key in the dataRefs to compare with the outputSortKeyOrder. 

786 

787 Returns 

788 ------- 

789 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

790 Quantized Connection with sorted DatasetRef values sorted if iterable. 

791 """ 

792 for connectionName, refs in inputRefs: 

793 if isinstance(refs, Iterable): 

794 if hasattr(refs[0], "dataId"): 

795 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

796 else: 

797 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

798 if inputSortKeyOrder != outputSortKeyOrder: 

799 setattr(inputRefs, connectionName, 

800 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

801 return inputRefs