Coverage for python/lsst/pipe/tasks/makeWarp.py: 15%

285 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-27 03:36 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["MakeWarpTask", "MakeWarpConfig"] 

23 

24from deprecated.sphinx import deprecated 

25import logging 

26import numpy 

27 

28import lsst.pex.config as pexConfig 

29import lsst.afw.image as afwImage 

30import lsst.coadd.utils as coaddUtils 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as connectionTypes 

33import lsst.utils as utils 

34import lsst.geom 

35from lsst.daf.butler import DeferredDatasetHandle 

36from lsst.meas.base import DetectorVisitIdGeneratorConfig 

37from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

38from lsst.skymap import BaseSkyMap 

39from lsst.utils.timer import timeMethod 

40from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

41from .warpAndPsfMatch import WarpAndPsfMatchTask 

42from collections.abc import Iterable 

43 

44log = logging.getLogger(__name__) 

45 

46 

47class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

48 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

49 defaultTemplates={"coaddName": "deep", 

50 "skyWcsName": "gbdesAstrometricFit", 

51 "photoCalibName": "fgcm", 

52 "calexpType": ""}): 

53 calExpList = connectionTypes.Input( 

54 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

55 name="{calexpType}calexp", 

56 storageClass="ExposureF", 

57 dimensions=("instrument", "visit", "detector"), 

58 multiple=True, 

59 deferLoad=True, 

60 ) 

61 backgroundList = connectionTypes.Input( 

62 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

63 name="calexpBackground", 

64 storageClass="Background", 

65 dimensions=("instrument", "visit", "detector"), 

66 multiple=True, 

67 ) 

68 skyCorrList = connectionTypes.Input( 

69 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

70 name="skyCorr", 

71 storageClass="Background", 

72 dimensions=("instrument", "visit", "detector"), 

73 multiple=True, 

74 ) 

75 skyMap = connectionTypes.Input( 

76 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

78 storageClass="SkyMap", 

79 dimensions=("skymap",), 

80 ) 

81 externalSkyWcsTractCatalog = connectionTypes.Input( 

82 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

83 "id for the catalog id, sorted on id for fast lookup."), 

84 name="{skyWcsName}SkyWcsCatalog", 

85 storageClass="ExposureCatalog", 

86 dimensions=("instrument", "visit", "tract"), 

87 ) 

88 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

89 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

90 "These catalogs use the detector id for the catalog id, sorted on id for " 

91 "fast lookup."), 

92 name="finalVisitSummary", 

93 storageClass="ExposureCatalog", 

94 dimensions=("instrument", "visit"), 

95 ) 

96 externalPhotoCalibTractCatalog = connectionTypes.Input( 

97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

98 "detector id for the catalog id, sorted on id for fast lookup."), 

99 name="{photoCalibName}PhotoCalibCatalog", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit", "tract"), 

102 ) 

103 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

104 doc=("Per-visit photometric calibrations computed globally (with no tract " 

105 "information). These catalogs use the detector id for the catalog id, " 

106 "sorted on id for fast lookup."), 

107 name="finalVisitSummary", 

108 storageClass="ExposureCatalog", 

109 dimensions=("instrument", "visit"), 

110 ) 

111 finalizedPsfApCorrCatalog = connectionTypes.Input( 

112 doc=("Per-visit finalized psf models and aperture correction maps. " 

113 "These catalogs use the detector id for the catalog id, " 

114 "sorted on id for fast lookup."), 

115 name="finalVisitSummary", 

116 storageClass="ExposureCatalog", 

117 dimensions=("instrument", "visit"), 

118 ) 

119 direct = connectionTypes.Output( 

120 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

121 "calexps onto the skyMap patch geometry."), 

122 name="{coaddName}Coadd_directWarp", 

123 storageClass="ExposureF", 

124 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

125 ) 

126 psfMatched = connectionTypes.Output( 

127 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

128 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

129 name="{coaddName}Coadd_psfMatchedWarp", 

130 storageClass="ExposureF", 

131 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

132 ) 

133 # TODO DM-28769, have selectImages subtask indicate which connections they 

134 # need: 

135 wcsList = connectionTypes.Input( 

136 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

137 name="{calexpType}calexp.wcs", 

138 storageClass="Wcs", 

139 dimensions=("instrument", "visit", "detector"), 

140 multiple=True, 

141 ) 

142 bboxList = connectionTypes.Input( 

143 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

144 name="{calexpType}calexp.bbox", 

145 storageClass="Box2I", 

146 dimensions=("instrument", "visit", "detector"), 

147 multiple=True, 

148 ) 

149 visitSummary = connectionTypes.Input( 

150 doc="Consolidated exposure metadata", 

151 name="finalVisitSummary", 

152 storageClass="ExposureCatalog", 

153 dimensions=("instrument", "visit",), 

154 ) 

155 

156 def __init__(self, *, config=None): 

157 super().__init__(config=config) 

158 if config.bgSubtracted: 

159 self.inputs.remove("backgroundList") 

160 if not config.doApplySkyCorr: 

161 self.inputs.remove("skyCorrList") 

162 if config.doApplyExternalSkyWcs: 

163 if config.useGlobalExternalSkyWcs: 

164 self.inputs.remove("externalSkyWcsTractCatalog") 

165 else: 

166 self.inputs.remove("externalSkyWcsGlobalCatalog") 

167 else: 

168 self.inputs.remove("externalSkyWcsTractCatalog") 

169 self.inputs.remove("externalSkyWcsGlobalCatalog") 

170 if config.doApplyExternalPhotoCalib: 

171 if config.useGlobalExternalPhotoCalib: 

172 self.inputs.remove("externalPhotoCalibTractCatalog") 

173 else: 

174 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

175 else: 

176 self.inputs.remove("externalPhotoCalibTractCatalog") 

177 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

178 if not config.doApplyFinalizedPsf: 

179 self.inputs.remove("finalizedPsfApCorrCatalog") 

180 if not config.makeDirect: 

181 self.outputs.remove("direct") 

182 if not config.makePsfMatched: 

183 self.outputs.remove("psfMatched") 

184 # TODO DM-28769: add connection per selectImages connections 

185 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

186 self.inputs.remove("visitSummary") 

187 

188 

189class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, 

190 pipelineConnections=MakeWarpConnections): 

191 """Config for MakeWarpTask.""" 

192 

193 warpAndPsfMatch = pexConfig.ConfigurableField( 

194 target=WarpAndPsfMatchTask, 

195 doc="Task to warp and PSF-match calexp", 

196 ) 

197 doWrite = pexConfig.Field( 

198 doc="persist <coaddName>Coadd_<warpType>Warp", 

199 dtype=bool, 

200 default=True, 

201 ) 

202 bgSubtracted = pexConfig.Field( 

203 doc="Work with a background subtracted calexp?", 

204 dtype=bool, 

205 default=True, 

206 ) 

207 coaddPsf = pexConfig.ConfigField( 

208 doc="Configuration for CoaddPsf", 

209 dtype=CoaddPsfConfig, 

210 ) 

211 makeDirect = pexConfig.Field( 

212 doc="Make direct Warp/Coadds", 

213 dtype=bool, 

214 default=True, 

215 ) 

216 makePsfMatched = pexConfig.Field( 

217 doc="Make Psf-Matched Warp/Coadd?", 

218 dtype=bool, 

219 default=False, 

220 ) 

221 doWriteEmptyWarps = pexConfig.Field( 

222 dtype=bool, 

223 default=False, 

224 doc="Write out warps even if they are empty" 

225 ) 

226 hasFakes = pexConfig.Field( 

227 doc="Should be set to True if fake sources have been inserted into the input data.", 

228 dtype=bool, 

229 default=False, 

230 ) 

231 doApplySkyCorr = pexConfig.Field( 

232 dtype=bool, 

233 default=False, 

234 doc="Apply sky correction?", 

235 ) 

236 doApplyFinalizedPsf = pexConfig.Field( 

237 doc="Whether to apply finalized psf models and aperture correction map.", 

238 dtype=bool, 

239 default=True, 

240 ) 

241 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

242 

243 def validate(self): 

244 CoaddBaseTask.ConfigClass.validate(self) 

245 

246 if not self.makePsfMatched and not self.makeDirect: 

247 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

248 if self.doPsfMatch: 

249 # Backwards compatibility. 

250 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

251 self.makePsfMatched = True 

252 self.makeDirect = False 

253 

254 def setDefaults(self): 

255 CoaddBaseTask.ConfigClass.setDefaults(self) 

256 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

257 

258 

259class MakeWarpTask(CoaddBaseTask): 

260 """Warp and optionally PSF-Match calexps onto an a common projection. 

261 

262 Warp and optionally PSF-Match calexps onto a common projection, by 

263 performing the following operations: 

264 - Group calexps by visit/run 

265 - For each visit, generate a Warp by calling method @ref run. 

266 `run` loops over the visit's calexps calling 

267 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit 

268 

269 Notes 

270 ----- 

271 WarpType identifies the types of convolutions applied to Warps 

272 (previously CoaddTempExps). Only two types are available: direct 

273 (for regular Warps/Coadds) and psfMatched(for Warps/Coadds with 

274 homogenized PSFs). We expect to add a third type, likelihood, for 

275 generating likelihood Coadds with Warps that have been correlated with 

276 their own PSF. 

277 

278 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

279 `~lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask` 

280 is responsible for the PSF-Matching, and its config is accessed via 

281 `config.warpAndPsfMatch.psfMatch`. 

282 

283 The optimal configuration depends on aspects of dataset: the pixel scale, 

284 average PSF FWHM and dimensions of the PSF kernel. These configs include 

285 the requested model PSF, the matching kernel size, padding of the science 

286 PSF thumbnail and spatial sampling frequency of the PSF. 

287 

288 *Config Guidelines*: The user must specify the size of the model PSF to 

289 which to match by setting `config.modelPsf.defaultFwhm` in units of pixels. 

290 The appropriate values depends on science case. In general, for a set of 

291 input images, this config should equal the FWHM of the visit with the worst 

292 seeing. The smallest it should be set to is the median FWHM. The defaults 

293 of the other config options offer a reasonable starting point. 

294 

295 The following list presents the most common problems that arise from a 

296 misconfigured 

297 @link ip::diffim::modelPsfMatch::ModelPsfMatchTask ModelPsfMatchTask 

298 @endlink 

299 and corresponding solutions. All assume the default Alard-Lupton kernel, 

300 with configs accessed via 

301 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list 

302 is formatted as: 

303 Problem: Explanation. *Solution* 

304 

305 *Troublshooting PSF-Matching Configuration:* 

306 - Matched PSFs look boxy: The matching kernel is too small. 

307 _Increase the matching kernel size. 

308 For example:_ 

309 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

310 # default 21 

311 Note that increasing the kernel size also increases runtime. 

312 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find 

313 good solution for matching kernel. 

314 _Provide the matcher with more data by either increasing 

315 the spatial sampling by decreasing the spatial cell size,_ 

316 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 

317 # default 128 

318 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 

319 # default 128 

320 _or increasing the padding around the Science PSF, for example:_ 

321 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

322 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF 

323 dimensions to the matching kernel dimensions, thus increasing the 

324 number of pixels available to fit after convolving the PSF with the 

325 matching kernel. Optionally, for debugging the effects of padding, the 

326 level of padding may be manually controlled by setting turning off the 

327 automatic padding and setting the number of pixels by which to pad the 

328 PSF: 

329 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False 

330 # default True 

331 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 

332 # pixels. default 0 

333 - Deconvolution: Matching a large PSF to a smaller PSF produces 

334 a telltale noise pattern which looks like ripples or a brain. 

335 _Increase the size of the requested model PSF. For example:_ 

336 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of 

337 pixels. 

338 - High frequency (sometimes checkered) noise: The matching basis functions 

339 are too small. 

340 _Increase the width of the Gaussian basis functions. For example:_ 

341 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss= 

342 [1.5, 3.0, 6.0] # from default [0.7, 1.5, 3.0] 

343 """ 

344 ConfigClass = MakeWarpConfig 

345 _DefaultName = "makeWarp" 

346 

347 def __init__(self, **kwargs): 

348 CoaddBaseTask.__init__(self, **kwargs) 

349 self.makeSubtask("warpAndPsfMatch") 

350 if self.config.hasFakes: 

351 self.calexpType = "fakes_calexp" 

352 else: 

353 self.calexpType = "calexp" 

354 

355 @utils.inheritDoc(pipeBase.PipelineTask) 

356 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

357 # Obtain the list of input detectors from calExpList. Sort them by 

358 # detector order (to ensure reproducibility). Then ensure all input 

359 # lists are in the same sorted detector order. 

360 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

361 detectorOrder.sort() 

362 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

363 

364 # Read in all inputs. 

365 inputs = butlerQC.get(inputRefs) 

366 

367 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

368 # from the dictionary so we can pass it as kwargs later. 

369 skyMap = inputs.pop("skyMap") 

370 quantumDataId = butlerQC.quantum.dataId 

371 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

372 

373 # Construct list of input DataIds expected by `run`. 

374 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

375 # Construct list of packed integer IDs expected by `run`. 

376 ccdIdList = [ 

377 self.config.idGenerator.apply(dataId).catalog_id 

378 for dataId in dataIdList 

379 ] 

380 

381 if self.config.doApplyExternalSkyWcs: 

382 if self.config.useGlobalExternalSkyWcs: 

383 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

384 else: 

385 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

386 else: 

387 externalSkyWcsCatalog = None 

388 

389 if self.config.doApplyExternalPhotoCalib: 

390 if self.config.useGlobalExternalPhotoCalib: 

391 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

392 else: 

393 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

394 else: 

395 externalPhotoCalibCatalog = None 

396 

397 if self.config.doApplyFinalizedPsf: 

398 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog") 

399 else: 

400 finalizedPsfApCorrCatalog = None 

401 

402 # Do an initial selection on inputs with complete wcs/photoCalib info. 

403 # Qualifying calexps will be read in the following call. 

404 completeIndices = self._prepareCalibratedExposures( 

405 **inputs, 

406 externalSkyWcsCatalog=externalSkyWcsCatalog, 

407 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

408 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

409 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

410 

411 # Do another selection based on the configured selection task 

412 # (using updated WCSs to determine patch overlap if an external 

413 # calibration was applied). 

414 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

415 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

416 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

417 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

418 

419 # Extract integer visitId requested by `run`. 

420 visitId = dataIdList[0]["visit"] 

421 

422 results = self.run(**inputs, visitId=visitId, 

423 ccdIdList=[ccdIdList[i] for i in goodIndices], 

424 dataIdList=[dataIdList[i] for i in goodIndices], 

425 skyInfo=skyInfo) 

426 if self.config.makeDirect and results.exposures["direct"] is not None: 

427 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

428 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

429 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

430 

431 @timeMethod 

432 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

433 """Create a Warp from inputs. 

434 

435 We iterate over the multiple calexps in a single exposure to construct 

436 the warp (previously called a coaddTempExp) of that exposure to the 

437 supplied tract/patch. 

438 

439 Pixels that receive no pixels are set to NAN; this is not correct 

440 (violates LSST algorithms group policy), but will be fixed up by 

441 interpolating after the coaddition. 

442 

443 calexpRefList : `list` 

444 List of data references for calexps that (may) 

445 overlap the patch of interest. 

446 skyInfo : `lsst.pipe.base.Struct` 

447 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

448 geometric information about the patch. 

449 visitId : `int` 

450 Integer identifier for visit, for the table that will 

451 produce the CoaddPsf. 

452 

453 Returns 

454 ------- 

455 result : `lsst.pipe.base.Struct` 

456 Results as a struct with attributes: 

457 

458 ``exposures`` 

459 A dictionary containing the warps requested: 

460 "direct": direct warp if ``config.makeDirect`` 

461 "psfMatched": PSF-matched warp if ``config.makePsfMatched`` 

462 (`dict`). 

463 """ 

464 warpTypeList = self.getWarpTypeList() 

465 

466 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

467 didSetMetadata = {warpType: False for warpType in warpTypeList} 

468 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

469 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

470 for warpType in warpTypeList} 

471 

472 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

473 if dataIdList is None: 

474 dataIdList = ccdIdList 

475 

476 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

477 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

478 calExpInd+1, len(calExpList), dataId) 

479 # TODO: The following conditional is only required for backwards 

480 # compatibility with the deprecated prepareCalibratedExposures() 

481 # method. Can remove with its removal after the deprecation 

482 # period. 

483 if isinstance(calExp, DeferredDatasetHandle): 

484 calExp = calExp.get() 

485 try: 

486 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

487 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

488 makeDirect=self.config.makeDirect, 

489 makePsfMatched=self.config.makePsfMatched) 

490 except Exception as e: 

491 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

492 continue 

493 try: 

494 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

495 for warpType in warpTypeList: 

496 exposure = warpedAndMatched.getDict()[warpType] 

497 if exposure is None: 

498 continue 

499 warp = warps[warpType] 

500 if didSetMetadata[warpType]: 

501 mimg = exposure.getMaskedImage() 

502 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

503 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

504 del mimg 

505 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

506 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

507 totGoodPix[warpType] += numGoodPix[warpType] 

508 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

509 dataId, numGoodPix[warpType], 

510 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

511 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

512 warp.info.id = exposure.info.id 

513 warp.setPhotoCalib(exposure.getPhotoCalib()) 

514 warp.setFilter(exposure.getFilter()) 

515 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

516 # PSF replaced with CoaddPsf after loop if and only if 

517 # creating direct warp. 

518 warp.setPsf(exposure.getPsf()) 

519 didSetMetadata[warpType] = True 

520 

521 # Need inputRecorder for CoaddApCorrMap for both direct and 

522 # PSF-matched. 

523 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

524 

525 except Exception as e: 

526 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

527 continue 

528 

529 for warpType in warpTypeList: 

530 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

531 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

532 

533 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

534 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType]) 

535 if warpType == "direct": 

536 warps[warpType].setPsf( 

537 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

538 self.config.coaddPsf.makeControl())) 

539 else: 

540 if not self.config.doWriteEmptyWarps: 

541 # No good pixels. Exposure still empty. 

542 warps[warpType] = None 

543 # NoWorkFound is unnecessary as the downstream tasks will 

544 # adjust the quantum accordingly. 

545 

546 result = pipeBase.Struct(exposures=warps) 

547 return result 

548 

549 def filterInputs(self, indices, inputs): 

550 """Filter task inputs by their indices. 

551 

552 Parameters 

553 ---------- 

554 indices : `list` [`int`] 

555 inputs : `dict` [`list`] 

556 A dictionary of input connections to be passed to run. 

557 

558 Returns 

559 ------- 

560 inputs : `dict` [`list`] 

561 Task inputs with their lists filtered by indices. 

562 """ 

563 for key in inputs.keys(): 

564 # Only down-select on list inputs 

565 if isinstance(inputs[key], list): 

566 inputs[key] = [inputs[key][ind] for ind in indices] 

567 return inputs 

568 

569 @deprecated(reason="This method is deprecated in favor of its leading underscore version, " 

570 "_prepareCalibratedfExposures(). Will be removed after v25.", 

571 version="v25.0", category=FutureWarning) 

572 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

573 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

574 finalizedPsfApCorrCatalog=None, 

575 **kwargs): 

576 """Deprecated function. 

577 

578 Please use _prepareCalibratedExposure(), which this delegates to and 

579 noting its slightly updated API, instead. 

580 """ 

581 # Read in all calexps. 

582 calExpList = [ref.get() for ref in calExpList] 

583 # Populate wcsList as required by new underscored version of function. 

584 wcsList = [calexp.getWcs() for calexp in calExpList] 

585 

586 indices = self._prepareCalibratedExposures(calExpList=calExpList, wcsList=wcsList, 

587 backgroundList=backgroundList, skyCorrList=skyCorrList, 

588 externalSkyWcsCatalog=externalSkyWcsCatalog, 

589 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

590 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

591 return indices 

592 

593 def _prepareCalibratedExposures(self, calExpList=[], wcsList=None, backgroundList=None, skyCorrList=None, 

594 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

595 finalizedPsfApCorrCatalog=None, **kwargs): 

596 """Calibrate and add backgrounds to input calExpList in place. 

597 

598 Parameters 

599 ---------- 

600 calExpList : `list` [`lsst.afw.image.Exposure` or 

601 `lsst.daf.butler.DeferredDatasetHandle`] 

602 Sequence of calexps to be modified in place. 

603 wcsList : `list` [`lsst.afw.geom.SkyWcs`] 

604 The WCSs of the calexps in ``calExpList``. When 

605 ``externalSkyCatalog`` is `None`, these are used to determine if 

606 the calexp should be included in the warp, namely checking that it 

607 is not `None`. If ``externalSkyCatalog`` is not `None`, this list 

608 will be dynamically updated with the external sky WCS. 

609 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional 

610 Sequence of backgrounds to be added back in if bgSubtracted=False. 

611 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional 

612 Sequence of background corrections to be subtracted if 

613 doApplySkyCorr=True. 

614 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

615 Exposure catalog with external skyWcs to be applied 

616 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

617 for the catalog id, sorted on id for fast lookup. 

618 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

619 Exposure catalog with external photoCalib to be applied 

620 if config.doApplyExternalPhotoCalib=True. Catalog uses the 

621 detector id for the catalog id, sorted on id for fast lookup. 

622 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

623 Exposure catalog with finalized psf models and aperture correction 

624 maps to be applied if config.doApplyFinalizedPsf=True. Catalog 

625 uses the detector id for the catalog id, sorted on id for fast 

626 lookup. 

627 **kwargs 

628 Additional keyword arguments. 

629 

630 Returns 

631 ------- 

632 indices : `list` [`int`] 

633 Indices of ``calExpList`` and friends that have valid 

634 photoCalib/skyWcs. 

635 """ 

636 wcsList = len(calExpList)*[None] if wcsList is None else wcsList 

637 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

638 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

639 

640 includeCalibVar = self.config.includeCalibVar 

641 

642 indices = [] 

643 for index, (calexp, wcs, background, skyCorr) in enumerate(zip(calExpList, 

644 wcsList, 

645 backgroundList, 

646 skyCorrList)): 

647 if externalSkyWcsCatalog is None and wcs is None: 

648 self.log.warning("Detector id %d for visit %d has None for skyWcs and will not be " 

649 "used in the warp", calexp.dataId["detector"], calexp.dataId["visit"]) 

650 continue 

651 

652 if isinstance(calexp, DeferredDatasetHandle): 

653 calexp = calexp.get() 

654 

655 if not self.config.bgSubtracted: 

656 calexp.maskedImage += background.getImage() 

657 

658 detectorId = calexp.info.getDetector().getId() 

659 

660 # Find the external photoCalib. 

661 if externalPhotoCalibCatalog is not None: 

662 row = externalPhotoCalibCatalog.find(detectorId) 

663 if row is None: 

664 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

665 "and will not be used in the warp.", detectorId) 

666 continue 

667 photoCalib = row.getPhotoCalib() 

668 if photoCalib is None: 

669 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

670 "and will not be used in the warp.", detectorId) 

671 continue 

672 calexp.setPhotoCalib(photoCalib) 

673 else: 

674 photoCalib = calexp.getPhotoCalib() 

675 if photoCalib is None: 

676 self.log.warning("Detector id %s has None for photoCalib in the calexp " 

677 "and will not be used in the warp.", detectorId) 

678 continue 

679 

680 # Find and apply external skyWcs. 

681 if externalSkyWcsCatalog is not None: 

682 row = externalSkyWcsCatalog.find(detectorId) 

683 if row is None: 

684 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

685 "and will not be used in the warp.", detectorId) 

686 continue 

687 skyWcs = row.getWcs() 

688 wcsList[index] = skyWcs 

689 if skyWcs is None: 

690 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

691 "and will not be used in the warp.", detectorId) 

692 continue 

693 calexp.setWcs(skyWcs) 

694 else: 

695 skyWcs = calexp.getWcs() 

696 wcsList[index] = skyWcs 

697 if skyWcs is None: 

698 self.log.warning("Detector id %s has None for skyWcs in the calexp " 

699 "and will not be used in the warp.", detectorId) 

700 continue 

701 

702 # Find and apply finalized psf and aperture correction. 

703 if finalizedPsfApCorrCatalog is not None: 

704 row = finalizedPsfApCorrCatalog.find(detectorId) 

705 if row is None: 

706 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog " 

707 "and will not be used in the warp.", detectorId) 

708 continue 

709 psf = row.getPsf() 

710 if psf is None: 

711 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog " 

712 "and will not be used in the warp.", detectorId) 

713 continue 

714 calexp.setPsf(psf) 

715 apCorrMap = row.getApCorrMap() 

716 if apCorrMap is None: 

717 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog " 

718 "and will not be used in the warp.", detectorId) 

719 continue 

720 calexp.info.setApCorrMap(apCorrMap) 

721 else: 

722 # Ensure that calexp has valid aperture correction map. 

723 if calexp.info.getApCorrMap() is None: 

724 self.log.warning("Detector id %s has None for ApCorrMap in the calexp " 

725 "and will not be used in the warp.", detectorId) 

726 continue 

727 

728 # Calibrate the image. 

729 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

730 includeScaleUncertainty=includeCalibVar) 

731 calexp.maskedImage /= photoCalib.getCalibrationMean() 

732 # TODO: The images will have a calibration of 1.0 everywhere once 

733 # RFC-545 is implemented. 

734 # exposure.setCalib(afwImage.Calib(1.0)) 

735 

736 # Apply skycorr 

737 if self.config.doApplySkyCorr: 

738 calexp.maskedImage -= skyCorr.getImage() 

739 

740 indices.append(index) 

741 calExpList[index] = calexp 

742 

743 return indices 

744 

745 @staticmethod 

746 def _prepareEmptyExposure(skyInfo): 

747 """Produce an empty exposure for a given patch. 

748 

749 Parameters 

750 ---------- 

751 skyInfo : `lsst.pipe.base.Struct` 

752 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

753 geometric information about the patch. 

754 

755 Returns 

756 ------- 

757 exp : `lsst.afw.image.exposure.ExposureF` 

758 An empty exposure for a given patch. 

759 """ 

760 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

761 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

762 .getPlaneBitMask("NO_DATA"), numpy.inf) 

763 return exp 

764 

765 def getWarpTypeList(self): 

766 """Return list of requested warp types per the config. 

767 """ 

768 warpTypeList = [] 

769 if self.config.makeDirect: 

770 warpTypeList.append("direct") 

771 if self.config.makePsfMatched: 

772 warpTypeList.append("psfMatched") 

773 return warpTypeList 

774 

775 

776def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

777 """Reorder inputRefs per outputSortKeyOrder. 

778 

779 Any inputRefs which are lists will be resorted per specified key e.g., 

780 'detector.' Only iterables will be reordered, and values can be of type 

781 `lsst.pipe.base.connections.DeferredDatasetRef` or 

782 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

783 

784 Returned lists of refs have the same length as the outputSortKeyOrder. 

785 If an outputSortKey not in the inputRef, then it will be padded with None. 

786 If an inputRef contains an inputSortKey that is not in the 

787 outputSortKeyOrder it will be removed. 

788 

789 Parameters 

790 ---------- 

791 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

792 Input references to be reordered and padded. 

793 outputSortKeyOrder : `iterable` 

794 Iterable of values to be compared with inputRef's dataId[dataIdKey]. 

795 dataIdKey : `str` 

796 The data ID key in the dataRefs to compare with the outputSortKeyOrder. 

797 

798 Returns 

799 ------- 

800 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

801 Quantized Connection with sorted DatasetRef values sorted if iterable. 

802 """ 

803 for connectionName, refs in inputRefs: 

804 if isinstance(refs, Iterable): 

805 if hasattr(refs[0], "dataId"): 

806 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

807 else: 

808 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

809 if inputSortKeyOrder != outputSortKeyOrder: 

810 setattr(inputRefs, connectionName, 

811 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

812 return inputRefs