Coverage for python/lsst/pipe/tasks/makeWarp.py: 15%

305 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-27 03:33 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["MakeWarpTask", "MakeWarpConfig"] 

23 

24import logging 

25import numpy 

26 

27import lsst.pex.config as pexConfig 

28import lsst.afw.image as afwImage 

29import lsst.coadd.utils as coaddUtils 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as connectionTypes 

32import lsst.utils as utils 

33import lsst.geom 

34from lsst.daf.butler import DeferredDatasetHandle 

35from lsst.meas.base import DetectorVisitIdGeneratorConfig 

36from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

37from lsst.skymap import BaseSkyMap 

38from lsst.utils.timer import timeMethod 

39from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

40from .warpAndPsfMatch import WarpAndPsfMatchTask 

41from collections.abc import Iterable 

42 

43log = logging.getLogger(__name__) 

44 

45 

46class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

48 defaultTemplates={"coaddName": "deep", 

49 "skyWcsName": "gbdesAstrometricFit", 

50 "photoCalibName": "fgcm", 

51 "calexpType": ""}, 

52 # TODO: remove on DM-39854. 

53 deprecatedTemplates={"skyWcsName": "Deprecated; will be removed after v26.", 

54 "photoCalibName": "Deprecated; will be removed after v26."}): 

55 calExpList = connectionTypes.Input( 

56 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

57 name="{calexpType}calexp", 

58 storageClass="ExposureF", 

59 dimensions=("instrument", "visit", "detector"), 

60 multiple=True, 

61 deferLoad=True, 

62 ) 

63 backgroundList = connectionTypes.Input( 

64 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

65 name="calexpBackground", 

66 storageClass="Background", 

67 dimensions=("instrument", "visit", "detector"), 

68 multiple=True, 

69 ) 

70 skyCorrList = connectionTypes.Input( 

71 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

72 name="skyCorr", 

73 storageClass="Background", 

74 dimensions=("instrument", "visit", "detector"), 

75 multiple=True, 

76 ) 

77 skyMap = connectionTypes.Input( 

78 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

79 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

80 storageClass="SkyMap", 

81 dimensions=("skymap",), 

82 ) 

83 externalSkyWcsTractCatalog = connectionTypes.Input( 

84 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

85 "id for the catalog id, sorted on id for fast lookup."), 

86 name="{skyWcsName}SkyWcsCatalog", 

87 storageClass="ExposureCatalog", 

88 dimensions=("instrument", "visit", "tract"), 

89 # TODO: remove on DM-39854. 

90 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

91 ) 

92 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

93 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

94 "These catalogs use the detector id for the catalog id, sorted on id for " 

95 "fast lookup."), 

96 name="finalVisitSummary", 

97 storageClass="ExposureCatalog", 

98 dimensions=("instrument", "visit"), 

99 # TODO: remove on DM-39854. 

100 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

101 ) 

102 externalPhotoCalibTractCatalog = connectionTypes.Input( 

103 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

104 "detector id for the catalog id, sorted on id for fast lookup."), 

105 name="{photoCalibName}PhotoCalibCatalog", 

106 storageClass="ExposureCatalog", 

107 dimensions=("instrument", "visit", "tract"), 

108 # TODO: remove on DM-39854. 

109 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

110 ) 

111 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

112 doc=("Per-visit photometric calibrations computed globally (with no tract " 

113 "information). These catalogs use the detector id for the catalog id, " 

114 "sorted on id for fast lookup."), 

115 name="finalVisitSummary", 

116 storageClass="ExposureCatalog", 

117 dimensions=("instrument", "visit"), 

118 # TODO: remove on DM-39854. 

119 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

120 ) 

121 finalizedPsfApCorrCatalog = connectionTypes.Input( 

122 doc=("Per-visit finalized psf models and aperture correction maps. " 

123 "These catalogs use the detector id for the catalog id, " 

124 "sorted on id for fast lookup."), 

125 name="finalVisitSummary", 

126 storageClass="ExposureCatalog", 

127 dimensions=("instrument", "visit"), 

128 # TODO: remove on DM-39854. 

129 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

130 ) 

131 direct = connectionTypes.Output( 

132 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

133 "calexps onto the skyMap patch geometry."), 

134 name="{coaddName}Coadd_directWarp", 

135 storageClass="ExposureF", 

136 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

137 ) 

138 psfMatched = connectionTypes.Output( 

139 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

140 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

141 name="{coaddName}Coadd_psfMatchedWarp", 

142 storageClass="ExposureF", 

143 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

144 ) 

145 wcsList = connectionTypes.Input( 

146 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

147 name="{calexpType}calexp.wcs", 

148 storageClass="Wcs", 

149 dimensions=("instrument", "visit", "detector"), 

150 multiple=True, 

151 # TODO: remove on DM-39854 

152 deprecated=( 

153 "Deprecated in favor of the 'visitSummary' connection (and already ignored). " 

154 "Will be removed after v26." 

155 ) 

156 ) 

157 bboxList = connectionTypes.Input( 

158 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

159 name="{calexpType}calexp.bbox", 

160 storageClass="Box2I", 

161 dimensions=("instrument", "visit", "detector"), 

162 multiple=True, 

163 # TODO: remove on DM-39854 

164 deprecated=( 

165 "Deprecated in favor of the 'visitSummary' connection (and already ignored). " 

166 "Will be removed after v26." 

167 ) 

168 ) 

169 visitSummary = connectionTypes.Input( 

170 doc="Input visit-summary catalog with updated calibration objects.", 

171 name="finalVisitSummary", 

172 storageClass="ExposureCatalog", 

173 dimensions=("instrument", "visit",), 

174 ) 

175 

176 def __init__(self, *, config=None): 

177 if config.bgSubtracted: 

178 del self.backgroundList 

179 if not config.doApplySkyCorr: 

180 del self.skyCorrList 

181 # TODO: remove all "external" checks on DM-39854 

182 if config.doApplyExternalSkyWcs: 

183 if config.useGlobalExternalSkyWcs: 

184 del self.externalSkyWcsTractCatalog 

185 else: 

186 del self.externalSkyWcsGlobalCatalog 

187 else: 

188 del self.externalSkyWcsTractCatalog 

189 del self.externalSkyWcsGlobalCatalog 

190 if config.doApplyExternalPhotoCalib: 

191 if config.useGlobalExternalPhotoCalib: 

192 del self.externalPhotoCalibTractCatalog 

193 else: 

194 del self.externalPhotoCalibGlobalCatalog 

195 else: 

196 del self.externalPhotoCalibTractCatalog 

197 del self.externalPhotoCalibGlobalCatalog 

198 if not config.doApplyFinalizedPsf: 

199 del self.finalizedPsfApCorrCatalog 

200 if not config.makeDirect: 

201 del self.direct 

202 if not config.makePsfMatched: 

203 del self.psfMatched 

204 # We always drop the deprecated wcsList and bboxList connections, 

205 # since we can always get equivalents from the visitSummary dataset. 

206 # Removing them here avoids the deprecation warning, but we do have 

207 # to deprecate rather than immediately remove them to keep old configs 

208 # usable for a bit. 

209 # TODO: remove on DM-39854 

210 del self.bboxList 

211 del self.wcsList 

212 

213 

214class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, 

215 pipelineConnections=MakeWarpConnections): 

216 """Config for MakeWarpTask.""" 

217 

218 warpAndPsfMatch = pexConfig.ConfigurableField( 

219 target=WarpAndPsfMatchTask, 

220 doc="Task to warp and PSF-match calexp", 

221 ) 

222 doWrite = pexConfig.Field( 

223 doc="persist <coaddName>Coadd_<warpType>Warp", 

224 dtype=bool, 

225 default=True, 

226 ) 

227 bgSubtracted = pexConfig.Field( 

228 doc="Work with a background subtracted calexp?", 

229 dtype=bool, 

230 default=True, 

231 ) 

232 coaddPsf = pexConfig.ConfigField( 

233 doc="Configuration for CoaddPsf", 

234 dtype=CoaddPsfConfig, 

235 ) 

236 makeDirect = pexConfig.Field( 

237 doc="Make direct Warp/Coadds", 

238 dtype=bool, 

239 default=True, 

240 ) 

241 makePsfMatched = pexConfig.Field( 

242 doc="Make Psf-Matched Warp/Coadd?", 

243 dtype=bool, 

244 default=False, 

245 ) 

246 useVisitSummaryPsf = pexConfig.Field( 

247 doc=( 

248 "If True, use the PSF model and aperture corrections from the 'visitSummary' connection. " 

249 "If False, use the PSF model and aperture corrections from the 'exposure' connection. " 

250 # TODO: remove this next sentence on DM-39854. 

251 "The finalizedPsfApCorrCatalog connection (if enabled) takes precedence over either." 

252 ), 

253 dtype=bool, 

254 default=True, 

255 ) 

256 doWriteEmptyWarps = pexConfig.Field( 

257 dtype=bool, 

258 default=False, 

259 doc="Write out warps even if they are empty" 

260 ) 

261 hasFakes = pexConfig.Field( 

262 doc="Should be set to True if fake sources have been inserted into the input data.", 

263 dtype=bool, 

264 default=False, 

265 ) 

266 doApplySkyCorr = pexConfig.Field( 

267 dtype=bool, 

268 default=False, 

269 doc="Apply sky correction?", 

270 ) 

271 doApplyFinalizedPsf = pexConfig.Field( 

272 doc="Whether to apply finalized psf models and aperture correction map.", 

273 dtype=bool, 

274 default=False, 

275 # TODO: remove on DM-39854. 

276 deprecated="Deprecated in favor of useVisitSummaryPsf. Will be removed after v26.", 

277 ) 

278 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

279 

280 def validate(self): 

281 CoaddBaseTask.ConfigClass.validate(self) 

282 

283 if not self.makePsfMatched and not self.makeDirect: 

284 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

285 if self.doPsfMatch: # TODO: Remove this in DM-39841 

286 # Backwards compatibility. 

287 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

288 self.makePsfMatched = True 

289 self.makeDirect = False 

290 

291 def setDefaults(self): 

292 CoaddBaseTask.ConfigClass.setDefaults(self) 

293 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

294 

295 

296class MakeWarpTask(CoaddBaseTask): 

297 """Warp and optionally PSF-Match calexps onto an a common projection. 

298 

299 Warp and optionally PSF-Match calexps onto a common projection, by 

300 performing the following operations: 

301 - Group calexps by visit/run 

302 - For each visit, generate a Warp by calling method @ref run. 

303 `run` loops over the visit's calexps calling 

304 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit 

305 

306 """ 

307 ConfigClass = MakeWarpConfig 

308 _DefaultName = "makeWarp" 

309 

310 def __init__(self, **kwargs): 

311 CoaddBaseTask.__init__(self, **kwargs) 

312 self.makeSubtask("warpAndPsfMatch") 

313 if self.config.hasFakes: 

314 self.calexpType = "fakes_calexp" 

315 else: 

316 self.calexpType = "calexp" 

317 

318 @utils.inheritDoc(pipeBase.PipelineTask) 

319 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

320 # Docstring to be augmented with info from PipelineTask.runQuantum 

321 """Notes 

322 ----- 

323 Obtain the list of input detectors from calExpList. Sort them by 

324 detector order (to ensure reproducibility). Then ensure all input 

325 lists are in the same sorted detector order. 

326 """ 

327 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

328 detectorOrder.sort() 

329 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

330 

331 # Read in all inputs. 

332 inputs = butlerQC.get(inputRefs) 

333 

334 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

335 # from the dictionary so we can pass it as kwargs later. 

336 skyMap = inputs.pop("skyMap") 

337 quantumDataId = butlerQC.quantum.dataId 

338 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

339 

340 # Construct list of input DataIds expected by `run`. 

341 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

342 # Construct list of packed integer IDs expected by `run`. 

343 ccdIdList = [ 

344 self.config.idGenerator.apply(dataId).catalog_id 

345 for dataId in dataIdList 

346 ] 

347 

348 visitSummary = inputs["visitSummary"] 

349 bboxList = [] 

350 wcsList = [] 

351 for dataId in dataIdList: 

352 row = visitSummary.find(dataId["detector"]) 

353 if row is None: 

354 raise RuntimeError( 

355 f"Unexpectedly incomplete visitSummary provided to makeWarp: {dataId} is missing." 

356 ) 

357 bboxList.append(row.getBBox()) 

358 wcsList.append(row.getWcs()) 

359 inputs["bboxList"] = bboxList 

360 inputs["wcsList"] = wcsList 

361 

362 if self.config.doApplyExternalSkyWcs: 

363 if self.config.useGlobalExternalSkyWcs: 

364 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

365 else: 

366 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

367 else: 

368 externalSkyWcsCatalog = None 

369 

370 if self.config.doApplyExternalPhotoCalib: 

371 if self.config.useGlobalExternalPhotoCalib: 

372 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

373 else: 

374 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

375 else: 

376 externalPhotoCalibCatalog = None 

377 

378 if self.config.doApplyFinalizedPsf: 

379 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog") 

380 else: 

381 finalizedPsfApCorrCatalog = None 

382 

383 # Do an initial selection on inputs with complete wcs/photoCalib info. 

384 # Qualifying calexps will be read in the following call. 

385 completeIndices = self._prepareCalibratedExposures( 

386 **inputs, 

387 externalSkyWcsCatalog=externalSkyWcsCatalog, 

388 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

389 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog, 

390 ) 

391 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

392 

393 # Do another selection based on the configured selection task 

394 # (using updated WCSs to determine patch overlap if an external 

395 # calibration was applied). 

396 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

397 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

398 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

399 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

400 

401 # Extract integer visitId requested by `run`. 

402 visitId = dataIdList[0]["visit"] 

403 

404 results = self.run(**inputs, 

405 visitId=visitId, 

406 ccdIdList=[ccdIdList[i] for i in goodIndices], 

407 dataIdList=[dataIdList[i] for i in goodIndices], 

408 skyInfo=skyInfo) 

409 if self.config.makeDirect and results.exposures["direct"] is not None: 

410 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

411 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

412 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

413 

414 @timeMethod 

415 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

416 """Create a Warp from inputs. 

417 

418 We iterate over the multiple calexps in a single exposure to construct 

419 the warp (previously called a coaddTempExp) of that exposure to the 

420 supplied tract/patch. 

421 

422 Pixels that receive no pixels are set to NAN; this is not correct 

423 (violates LSST algorithms group policy), but will be fixed up by 

424 interpolating after the coaddition. 

425 

426 calexpRefList : `list` 

427 List of data references for calexps that (may) 

428 overlap the patch of interest. 

429 skyInfo : `lsst.pipe.base.Struct` 

430 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

431 geometric information about the patch. 

432 visitId : `int` 

433 Integer identifier for visit, for the table that will 

434 produce the CoaddPsf. 

435 

436 Returns 

437 ------- 

438 result : `lsst.pipe.base.Struct` 

439 Results as a struct with attributes: 

440 

441 ``exposures`` 

442 A dictionary containing the warps requested: 

443 "direct": direct warp if ``config.makeDirect`` 

444 "psfMatched": PSF-matched warp if ``config.makePsfMatched`` 

445 (`dict`). 

446 """ 

447 warpTypeList = self.getWarpTypeList() 

448 

449 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

450 didSetMetadata = {warpType: False for warpType in warpTypeList} 

451 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

452 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

453 for warpType in warpTypeList} 

454 

455 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

456 if dataIdList is None: 

457 dataIdList = ccdIdList 

458 

459 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

460 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

461 calExpInd+1, len(calExpList), dataId) 

462 try: 

463 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

464 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

465 makeDirect=self.config.makeDirect, 

466 makePsfMatched=self.config.makePsfMatched) 

467 except Exception as e: 

468 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

469 continue 

470 try: 

471 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

472 for warpType in warpTypeList: 

473 exposure = warpedAndMatched.getDict()[warpType] 

474 if exposure is None: 

475 continue 

476 warp = warps[warpType] 

477 if didSetMetadata[warpType]: 

478 mimg = exposure.getMaskedImage() 

479 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

480 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

481 del mimg 

482 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

483 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

484 totGoodPix[warpType] += numGoodPix[warpType] 

485 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

486 dataId, numGoodPix[warpType], 

487 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

488 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

489 warp.info.id = exposure.info.id 

490 warp.setPhotoCalib(exposure.getPhotoCalib()) 

491 warp.setFilter(exposure.getFilter()) 

492 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

493 # PSF replaced with CoaddPsf after loop if and only if 

494 # creating direct warp. 

495 warp.setPsf(exposure.getPsf()) 

496 didSetMetadata[warpType] = True 

497 

498 # Need inputRecorder for CoaddApCorrMap for both direct and 

499 # PSF-matched. 

500 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

501 

502 except Exception as e: 

503 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

504 continue 

505 

506 for warpType in warpTypeList: 

507 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

508 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

509 

510 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

511 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType]) 

512 if warpType == "direct": 

513 warps[warpType].setPsf( 

514 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

515 self.config.coaddPsf.makeControl())) 

516 else: 

517 if not self.config.doWriteEmptyWarps: 

518 # No good pixels. Exposure still empty. 

519 warps[warpType] = None 

520 # NoWorkFound is unnecessary as the downstream tasks will 

521 # adjust the quantum accordingly. 

522 

523 result = pipeBase.Struct(exposures=warps) 

524 return result 

525 

526 def filterInputs(self, indices, inputs): 

527 """Filter task inputs by their indices. 

528 

529 Parameters 

530 ---------- 

531 indices : `list` [`int`] 

532 inputs : `dict` [`list`] 

533 A dictionary of input connections to be passed to run. 

534 

535 Returns 

536 ------- 

537 inputs : `dict` [`list`] 

538 Task inputs with their lists filtered by indices. 

539 """ 

540 for key in inputs.keys(): 

541 # Only down-select on list inputs 

542 if isinstance(inputs[key], list): 

543 inputs[key] = [inputs[key][ind] for ind in indices] 

544 return inputs 

545 

546 def _prepareCalibratedExposures(self, calExpList=[], wcsList=None, backgroundList=None, skyCorrList=None, 

547 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

548 finalizedPsfApCorrCatalog=None, visitSummary=None, **kwargs): 

549 """Calibrate and add backgrounds to input calExpList in place. 

550 

551 Parameters 

552 ---------- 

553 calExpList : `list` [`lsst.afw.image.Exposure` or 

554 `lsst.daf.butler.DeferredDatasetHandle`] 

555 Sequence of calexps to be modified in place. 

556 wcsList : `list` [`lsst.afw.geom.SkyWcs`] 

557 The WCSs of the calexps in ``calExpList``. When 

558 ``externalSkyCatalog`` is `None`, these are used to determine if 

559 the calexp should be included in the warp, namely checking that it 

560 is not `None`. If ``externalSkyCatalog`` is not `None`, this list 

561 will be dynamically updated with the external sky WCS. 

562 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional 

563 Sequence of backgrounds to be added back in if bgSubtracted=False. 

564 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional 

565 Sequence of background corrections to be subtracted if 

566 doApplySkyCorr=True. 

567 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

568 Exposure catalog with external skyWcs to be applied 

569 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

570 for the catalog id, sorted on id for fast lookup. 

571 Deprecated and will be removed after v26. 

572 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

573 Exposure catalog with external photoCalib to be applied 

574 if config.doApplyExternalPhotoCalib=True. Catalog uses the 

575 detector id for the catalog id, sorted on id for fast lookup. 

576 Deprecated and will be removed after v26. 

577 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

578 Exposure catalog with finalized psf models and aperture correction 

579 maps to be applied if config.doApplyFinalizedPsf=True. Catalog 

580 uses the detector id for the catalog id, sorted on id for fast 

581 lookup. 

582 Deprecated and will be removed after v26. 

583 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

584 Exposure catalog with potentially all calibrations. Attributes set 

585 to `None` are ignored. 

586 **kwargs 

587 Additional keyword arguments. 

588 

589 Returns 

590 ------- 

591 indices : `list` [`int`] 

592 Indices of ``calExpList`` and friends that have valid 

593 photoCalib/skyWcs. 

594 """ 

595 wcsList = len(calExpList)*[None] if wcsList is None else wcsList 

596 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

597 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

598 

599 includeCalibVar = self.config.includeCalibVar 

600 

601 indices = [] 

602 for index, (calexp, wcs, background, skyCorr) in enumerate(zip(calExpList, 

603 wcsList, 

604 backgroundList, 

605 skyCorrList)): 

606 if externalSkyWcsCatalog is None and wcs is None: 

607 self.log.warning("Detector id %d for visit %d has None for skyWcs and will not be " 

608 "used in the warp", calexp.dataId["detector"], calexp.dataId["visit"]) 

609 continue 

610 

611 if isinstance(calexp, DeferredDatasetHandle): 

612 calexp = calexp.get() 

613 

614 if not self.config.bgSubtracted: 

615 calexp.maskedImage += background.getImage() 

616 

617 detectorId = calexp.info.getDetector().getId() 

618 

619 # Load all calibrations from visitSummary. 

620 if visitSummary is not None: 

621 row = visitSummary.find(detectorId) 

622 if row is None: 

623 raise RuntimeError( 

624 f"Unexpectedly incomplete visitSummary: detector={detectorId} is missing." 

625 ) 

626 if (photoCalib := row.getPhotoCalib()) is not None: 

627 calexp.setPhotoCalib(photoCalib) 

628 if (skyWcs := row.getWcs()) is not None: 

629 calexp.setWcs(skyWcs) 

630 wcsList[index] = skyWcs 

631 if self.config.useVisitSummaryPsf: 

632 if (psf := row.getPsf()) is not None: 

633 calexp.setPsf(psf) 

634 if (apCorrMap := row.getApCorrMap()) is not None: 

635 calexp.info.setApCorrMap(apCorrMap) 

636 # TODO: on DM-39854 the logic in the 'elif' blocks below could 

637 # be moved into 'else' blocks above (or otherwise simplified 

638 # substantially) after the 'external' arguments are removed. 

639 

640 # Find the external photoCalib. 

641 if externalPhotoCalibCatalog is not None: 

642 row = externalPhotoCalibCatalog.find(detectorId) 

643 if row is None: 

644 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

645 "and will not be used in the warp.", detectorId) 

646 continue 

647 photoCalib = row.getPhotoCalib() 

648 if photoCalib is None: 

649 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

650 "and will not be used in the warp.", detectorId) 

651 continue 

652 calexp.setPhotoCalib(photoCalib) 

653 elif photoCalib is None: 

654 self.log.warning("Detector id %s has None for photoCalib in the visit summary " 

655 "and will not be used in the warp.", detectorId) 

656 continue 

657 

658 # Find and apply external skyWcs. 

659 if externalSkyWcsCatalog is not None: 

660 row = externalSkyWcsCatalog.find(detectorId) 

661 if row is None: 

662 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

663 "and will not be used in the warp.", detectorId) 

664 continue 

665 skyWcs = row.getWcs() 

666 wcsList[index] = skyWcs 

667 if skyWcs is None: 

668 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

669 "and will not be used in the warp.", detectorId) 

670 continue 

671 calexp.setWcs(skyWcs) 

672 elif skyWcs is None: 

673 self.log.warning("Detector id %s has None for skyWcs in the visit summary " 

674 "and will not be used in the warp.", detectorId) 

675 continue 

676 

677 # Find and apply finalized psf and aperture correction. 

678 if finalizedPsfApCorrCatalog is not None: 

679 row = finalizedPsfApCorrCatalog.find(detectorId) 

680 if row is None: 

681 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog " 

682 "and will not be used in the warp.", detectorId) 

683 continue 

684 psf = row.getPsf() 

685 if psf is None: 

686 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog " 

687 "and will not be used in the warp.", detectorId) 

688 continue 

689 calexp.setPsf(psf) 

690 apCorrMap = row.getApCorrMap() 

691 if apCorrMap is None: 

692 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog " 

693 "and will not be used in the warp.", detectorId) 

694 continue 

695 calexp.info.setApCorrMap(apCorrMap) 

696 elif self.config.useVisitSummaryPsf: 

697 if psf is None: 

698 self.log.warning("Detector id %s has None for PSF in the visit summary " 

699 "and will not be used in the warp.", detectorId) 

700 if apCorrMap is None: 

701 self.log.warning("Detector id %s has None for ApCorrMap in the visit summary " 

702 "and will not be used in the warp.", detectorId) 

703 else: 

704 if calexp.getPsf() is None: 

705 self.log.warning("Detector id %s has None for PSF in the calexp " 

706 "and will not be used in the warp.", detectorId) 

707 if calexp.info.getApCorrMap() is None: 

708 self.log.warning("Detector id %s has None for ApCorrMap in the calexp " 

709 "and will not be used in the warp.", detectorId) 

710 continue 

711 

712 # Calibrate the image. 

713 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

714 includeScaleUncertainty=includeCalibVar) 

715 calexp.maskedImage /= photoCalib.getCalibrationMean() 

716 # TODO: The images will have a calibration of 1.0 everywhere once 

717 # RFC-545 is implemented. 

718 # exposure.setCalib(afwImage.Calib(1.0)) 

719 

720 # Apply skycorr 

721 if self.config.doApplySkyCorr: 

722 calexp.maskedImage -= skyCorr.getImage() 

723 

724 indices.append(index) 

725 calExpList[index] = calexp 

726 

727 return indices 

728 

729 @staticmethod 

730 def _prepareEmptyExposure(skyInfo): 

731 """Produce an empty exposure for a given patch. 

732 

733 Parameters 

734 ---------- 

735 skyInfo : `lsst.pipe.base.Struct` 

736 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

737 geometric information about the patch. 

738 

739 Returns 

740 ------- 

741 exp : `lsst.afw.image.exposure.ExposureF` 

742 An empty exposure for a given patch. 

743 """ 

744 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

745 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

746 .getPlaneBitMask("NO_DATA"), numpy.inf) 

747 return exp 

748 

749 def getWarpTypeList(self): 

750 """Return list of requested warp types per the config. 

751 """ 

752 warpTypeList = [] 

753 if self.config.makeDirect: 

754 warpTypeList.append("direct") 

755 if self.config.makePsfMatched: 

756 warpTypeList.append("psfMatched") 

757 return warpTypeList 

758 

759 

760def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

761 """Reorder inputRefs per outputSortKeyOrder. 

762 

763 Any inputRefs which are lists will be resorted per specified key e.g., 

764 'detector.' Only iterables will be reordered, and values can be of type 

765 `lsst.pipe.base.connections.DeferredDatasetRef` or 

766 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

767 

768 Returned lists of refs have the same length as the outputSortKeyOrder. 

769 If an outputSortKey not in the inputRef, then it will be padded with None. 

770 If an inputRef contains an inputSortKey that is not in the 

771 outputSortKeyOrder it will be removed. 

772 

773 Parameters 

774 ---------- 

775 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

776 Input references to be reordered and padded. 

777 outputSortKeyOrder : `iterable` 

778 Iterable of values to be compared with inputRef's dataId[dataIdKey]. 

779 dataIdKey : `str` 

780 The data ID key in the dataRefs to compare with the outputSortKeyOrder. 

781 

782 Returns 

783 ------- 

784 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

785 Quantized Connection with sorted DatasetRef values sorted if iterable. 

786 """ 

787 for connectionName, refs in inputRefs: 

788 if isinstance(refs, Iterable): 

789 if hasattr(refs[0], "dataId"): 

790 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

791 else: 

792 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

793 if inputSortKeyOrder != outputSortKeyOrder: 

794 setattr(inputRefs, connectionName, 

795 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

796 return inputRefs