Coverage for python/lsst/pipe/tasks/makeWarp.py: 15%

305 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-12 10:21 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["MakeWarpTask", "MakeWarpConfig"] 

23 

24import logging 

25import numpy 

26 

27import lsst.pex.config as pexConfig 

28import lsst.afw.image as afwImage 

29import lsst.coadd.utils as coaddUtils 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as connectionTypes 

32import lsst.utils as utils 

33import lsst.geom 

34from lsst.daf.butler import DeferredDatasetHandle 

35from lsst.meas.base import DetectorVisitIdGeneratorConfig 

36from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

37from lsst.skymap import BaseSkyMap 

38from lsst.utils.timer import timeMethod 

39from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

40from .warpAndPsfMatch import WarpAndPsfMatchTask 

41from collections.abc import Iterable 

42 

43log = logging.getLogger(__name__) 

44 

45 

46class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

48 defaultTemplates={"coaddName": "deep", 

49 "skyWcsName": "gbdesAstrometricFit", 

50 "photoCalibName": "fgcm", 

51 "calexpType": ""}, 

52 # TODO: remove on DM-39854. 

53 deprecatedTemplates={"skyWcsName": "Deprecated; will be removed after v26.", 

54 "photoCalibName": "Deprecated; will be removed after v26."}): 

55 calExpList = connectionTypes.Input( 

56 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

57 name="{calexpType}calexp", 

58 storageClass="ExposureF", 

59 dimensions=("instrument", "visit", "detector"), 

60 multiple=True, 

61 deferLoad=True, 

62 ) 

63 backgroundList = connectionTypes.Input( 

64 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

65 name="calexpBackground", 

66 storageClass="Background", 

67 dimensions=("instrument", "visit", "detector"), 

68 multiple=True, 

69 ) 

70 skyCorrList = connectionTypes.Input( 

71 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

72 name="skyCorr", 

73 storageClass="Background", 

74 dimensions=("instrument", "visit", "detector"), 

75 multiple=True, 

76 ) 

77 skyMap = connectionTypes.Input( 

78 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

79 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

80 storageClass="SkyMap", 

81 dimensions=("skymap",), 

82 ) 

83 externalSkyWcsTractCatalog = connectionTypes.Input( 

84 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

85 "id for the catalog id, sorted on id for fast lookup."), 

86 name="{skyWcsName}SkyWcsCatalog", 

87 storageClass="ExposureCatalog", 

88 dimensions=("instrument", "visit", "tract"), 

89 # TODO: remove on DM-39854. 

90 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

91 ) 

92 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

93 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

94 "These catalogs use the detector id for the catalog id, sorted on id for " 

95 "fast lookup."), 

96 name="finalVisitSummary", 

97 storageClass="ExposureCatalog", 

98 dimensions=("instrument", "visit"), 

99 # TODO: remove on DM-39854. 

100 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

101 ) 

102 externalPhotoCalibTractCatalog = connectionTypes.Input( 

103 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

104 "detector id for the catalog id, sorted on id for fast lookup."), 

105 name="{photoCalibName}PhotoCalibCatalog", 

106 storageClass="ExposureCatalog", 

107 dimensions=("instrument", "visit", "tract"), 

108 # TODO: remove on DM-39854. 

109 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

110 ) 

111 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

112 doc=("Per-visit photometric calibrations computed globally (with no tract " 

113 "information). These catalogs use the detector id for the catalog id, " 

114 "sorted on id for fast lookup."), 

115 name="finalVisitSummary", 

116 storageClass="ExposureCatalog", 

117 dimensions=("instrument", "visit"), 

118 # TODO: remove on DM-39854. 

119 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

120 ) 

121 finalizedPsfApCorrCatalog = connectionTypes.Input( 

122 doc=("Per-visit finalized psf models and aperture correction maps. " 

123 "These catalogs use the detector id for the catalog id, " 

124 "sorted on id for fast lookup."), 

125 name="finalVisitSummary", 

126 storageClass="ExposureCatalog", 

127 dimensions=("instrument", "visit"), 

128 # TODO: remove on DM-39854. 

129 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

130 ) 

131 direct = connectionTypes.Output( 

132 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

133 "calexps onto the skyMap patch geometry."), 

134 name="{coaddName}Coadd_directWarp", 

135 storageClass="ExposureF", 

136 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

137 ) 

138 psfMatched = connectionTypes.Output( 

139 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

140 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

141 name="{coaddName}Coadd_psfMatchedWarp", 

142 storageClass="ExposureF", 

143 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

144 ) 

145 wcsList = connectionTypes.Input( 

146 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

147 name="{calexpType}calexp.wcs", 

148 storageClass="Wcs", 

149 dimensions=("instrument", "visit", "detector"), 

150 multiple=True, 

151 # TODO: remove on DM-39854 

152 deprecated=( 

153 "Deprecated in favor of the 'visitSummary' connection (and already ignored). " 

154 "Will be removed after v26." 

155 ) 

156 ) 

157 bboxList = connectionTypes.Input( 

158 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

159 name="{calexpType}calexp.bbox", 

160 storageClass="Box2I", 

161 dimensions=("instrument", "visit", "detector"), 

162 multiple=True, 

163 # TODO: remove on DM-39854 

164 deprecated=( 

165 "Deprecated in favor of the 'visitSummary' connection (and already ignored). " 

166 "Will be removed after v26." 

167 ) 

168 ) 

169 visitSummary = connectionTypes.Input( 

170 doc="Input visit-summary catalog with updated calibration objects.", 

171 name="finalVisitSummary", 

172 storageClass="ExposureCatalog", 

173 dimensions=("instrument", "visit",), 

174 ) 

175 

176 def __init__(self, *, config=None): 

177 if config.bgSubtracted: 

178 del self.backgroundList 

179 if not config.doApplySkyCorr: 

180 del self.skyCorrList 

181 # TODO: remove all "external" checks on DM-39854 

182 if config.doApplyExternalSkyWcs: 

183 if config.useGlobalExternalSkyWcs: 

184 del self.externalSkyWcsTractCatalog 

185 else: 

186 del self.externalSkyWcsGlobalCatalog 

187 else: 

188 del self.externalSkyWcsTractCatalog 

189 del self.externalSkyWcsGlobalCatalog 

190 if config.doApplyExternalPhotoCalib: 

191 if config.useGlobalExternalPhotoCalib: 

192 del self.externalPhotoCalibTractCatalog 

193 else: 

194 del self.externalPhotoCalibGlobalCatalog 

195 else: 

196 del self.externalPhotoCalibTractCatalog 

197 del self.externalPhotoCalibGlobalCatalog 

198 if not config.doApplyFinalizedPsf: 

199 del self.finalizedPsfApCorrCatalog 

200 if not config.makeDirect: 

201 del self.direct 

202 if not config.makePsfMatched: 

203 del self.psfMatched 

204 # We always drop the deprecated wcsList and bboxList connections, 

205 # since we can always get equivalents from the visitSummary dataset. 

206 # Removing them here avoids the deprecation warning, but we do have 

207 # to deprecate rather than immediately remove them to keep old configs 

208 # usable for a bit. 

209 # TODO: remove on DM-39854 

210 del self.bboxList 

211 del self.wcsList 

212 

213 

214class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, 

215 pipelineConnections=MakeWarpConnections): 

216 """Config for MakeWarpTask.""" 

217 

218 warpAndPsfMatch = pexConfig.ConfigurableField( 

219 target=WarpAndPsfMatchTask, 

220 doc="Task to warp and PSF-match calexp", 

221 ) 

222 doWrite = pexConfig.Field( 

223 doc="persist <coaddName>Coadd_<warpType>Warp", 

224 dtype=bool, 

225 default=True, 

226 ) 

227 bgSubtracted = pexConfig.Field( 

228 doc="Work with a background subtracted calexp?", 

229 dtype=bool, 

230 default=True, 

231 ) 

232 coaddPsf = pexConfig.ConfigField( 

233 doc="Configuration for CoaddPsf", 

234 dtype=CoaddPsfConfig, 

235 ) 

236 makeDirect = pexConfig.Field( 

237 doc="Make direct Warp/Coadds", 

238 dtype=bool, 

239 default=True, 

240 ) 

241 makePsfMatched = pexConfig.Field( 

242 doc="Make Psf-Matched Warp/Coadd?", 

243 dtype=bool, 

244 default=False, 

245 ) 

246 useVisitSummaryPsf = pexConfig.Field( 

247 doc=( 

248 "If True, use the PSF model and aperture corrections from the 'visitSummary' connection. " 

249 "If False, use the PSF model and aperture corrections from the 'exposure' connection. " 

250 # TODO: remove this next sentence on DM-39854. 

251 "The finalizedPsfApCorrCatalog connection (if enabled) takes precedence over either." 

252 ), 

253 dtype=bool, 

254 default=True, 

255 ) 

256 doWriteEmptyWarps = pexConfig.Field( 

257 dtype=bool, 

258 default=False, 

259 doc="Write out warps even if they are empty" 

260 ) 

261 hasFakes = pexConfig.Field( 

262 doc="Should be set to True if fake sources have been inserted into the input data.", 

263 dtype=bool, 

264 default=False, 

265 ) 

266 doApplySkyCorr = pexConfig.Field( 

267 dtype=bool, 

268 default=False, 

269 doc="Apply sky correction?", 

270 ) 

271 doApplyFinalizedPsf = pexConfig.Field( 

272 doc="Whether to apply finalized psf models and aperture correction map.", 

273 dtype=bool, 

274 default=True, 

275 # TODO: remove on DM-39854. 

276 deprecated="Deprecated in favor of useVisitSummaryPsf. Will be removed after v26.", 

277 ) 

278 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

279 

280 def validate(self): 

281 CoaddBaseTask.ConfigClass.validate(self) 

282 

283 if not self.makePsfMatched and not self.makeDirect: 

284 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

285 if self.doPsfMatch: # TODO: Remove this in DM-39841 

286 # Backwards compatibility. 

287 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

288 self.makePsfMatched = True 

289 self.makeDirect = False 

290 

291 def setDefaults(self): 

292 CoaddBaseTask.ConfigClass.setDefaults(self) 

293 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

294 

295 

296class MakeWarpTask(CoaddBaseTask): 

297 """Warp and optionally PSF-Match calexps onto an a common projection. 

298 

299 Warp and optionally PSF-Match calexps onto a common projection, by 

300 performing the following operations: 

301 - Group calexps by visit/run 

302 - For each visit, generate a Warp by calling method @ref run. 

303 `run` loops over the visit's calexps calling 

304 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit 

305 

306 """ 

307 ConfigClass = MakeWarpConfig 

308 _DefaultName = "makeWarp" 

309 

310 def __init__(self, **kwargs): 

311 CoaddBaseTask.__init__(self, **kwargs) 

312 self.makeSubtask("warpAndPsfMatch") 

313 if self.config.hasFakes: 

314 self.calexpType = "fakes_calexp" 

315 else: 

316 self.calexpType = "calexp" 

317 

318 @utils.inheritDoc(pipeBase.PipelineTask) 

319 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

320 # Obtain the list of input detectors from calExpList. Sort them by 

321 # detector order (to ensure reproducibility). Then ensure all input 

322 # lists are in the same sorted detector order. 

323 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

324 detectorOrder.sort() 

325 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

326 

327 # Read in all inputs. 

328 inputs = butlerQC.get(inputRefs) 

329 

330 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

331 # from the dictionary so we can pass it as kwargs later. 

332 skyMap = inputs.pop("skyMap") 

333 quantumDataId = butlerQC.quantum.dataId 

334 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

335 

336 # Construct list of input DataIds expected by `run`. 

337 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

338 # Construct list of packed integer IDs expected by `run`. 

339 ccdIdList = [ 

340 self.config.idGenerator.apply(dataId).catalog_id 

341 for dataId in dataIdList 

342 ] 

343 

344 visitSummary = inputs["visitSummary"] 

345 bboxList = [] 

346 wcsList = [] 

347 for dataId in dataIdList: 

348 row = visitSummary.find(dataId["detector"]) 

349 if row is None: 

350 raise RuntimeError( 

351 f"Unexpectedly incomplete visitSummary provided to makeWarp: {dataId} is missing." 

352 ) 

353 bboxList.append(row.getBBox()) 

354 wcsList.append(row.getWcs()) 

355 inputs["bboxList"] = bboxList 

356 inputs["wcsList"] = wcsList 

357 

358 if self.config.doApplyExternalSkyWcs: 

359 if self.config.useGlobalExternalSkyWcs: 

360 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

361 else: 

362 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

363 else: 

364 externalSkyWcsCatalog = None 

365 

366 if self.config.doApplyExternalPhotoCalib: 

367 if self.config.useGlobalExternalPhotoCalib: 

368 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

369 else: 

370 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

371 else: 

372 externalPhotoCalibCatalog = None 

373 

374 if self.config.doApplyFinalizedPsf: 

375 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog") 

376 else: 

377 finalizedPsfApCorrCatalog = None 

378 

379 # Do an initial selection on inputs with complete wcs/photoCalib info. 

380 # Qualifying calexps will be read in the following call. 

381 completeIndices = self._prepareCalibratedExposures( 

382 **inputs, 

383 externalSkyWcsCatalog=externalSkyWcsCatalog, 

384 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

385 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog, 

386 ) 

387 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

388 

389 # Do another selection based on the configured selection task 

390 # (using updated WCSs to determine patch overlap if an external 

391 # calibration was applied). 

392 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

393 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

394 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

395 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

396 

397 # Extract integer visitId requested by `run`. 

398 visitId = dataIdList[0]["visit"] 

399 

400 results = self.run(**inputs, 

401 visitId=visitId, 

402 ccdIdList=[ccdIdList[i] for i in goodIndices], 

403 dataIdList=[dataIdList[i] for i in goodIndices], 

404 skyInfo=skyInfo) 

405 if self.config.makeDirect and results.exposures["direct"] is not None: 

406 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

407 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

408 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

409 

410 @timeMethod 

411 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

412 """Create a Warp from inputs. 

413 

414 We iterate over the multiple calexps in a single exposure to construct 

415 the warp (previously called a coaddTempExp) of that exposure to the 

416 supplied tract/patch. 

417 

418 Pixels that receive no pixels are set to NAN; this is not correct 

419 (violates LSST algorithms group policy), but will be fixed up by 

420 interpolating after the coaddition. 

421 

422 calexpRefList : `list` 

423 List of data references for calexps that (may) 

424 overlap the patch of interest. 

425 skyInfo : `lsst.pipe.base.Struct` 

426 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

427 geometric information about the patch. 

428 visitId : `int` 

429 Integer identifier for visit, for the table that will 

430 produce the CoaddPsf. 

431 

432 Returns 

433 ------- 

434 result : `lsst.pipe.base.Struct` 

435 Results as a struct with attributes: 

436 

437 ``exposures`` 

438 A dictionary containing the warps requested: 

439 "direct": direct warp if ``config.makeDirect`` 

440 "psfMatched": PSF-matched warp if ``config.makePsfMatched`` 

441 (`dict`). 

442 """ 

443 warpTypeList = self.getWarpTypeList() 

444 

445 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

446 didSetMetadata = {warpType: False for warpType in warpTypeList} 

447 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

448 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

449 for warpType in warpTypeList} 

450 

451 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

452 if dataIdList is None: 

453 dataIdList = ccdIdList 

454 

455 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

456 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

457 calExpInd+1, len(calExpList), dataId) 

458 try: 

459 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

460 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

461 makeDirect=self.config.makeDirect, 

462 makePsfMatched=self.config.makePsfMatched) 

463 except Exception as e: 

464 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

465 continue 

466 try: 

467 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

468 for warpType in warpTypeList: 

469 exposure = warpedAndMatched.getDict()[warpType] 

470 if exposure is None: 

471 continue 

472 warp = warps[warpType] 

473 if didSetMetadata[warpType]: 

474 mimg = exposure.getMaskedImage() 

475 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

476 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

477 del mimg 

478 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

479 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

480 totGoodPix[warpType] += numGoodPix[warpType] 

481 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

482 dataId, numGoodPix[warpType], 

483 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

484 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

485 warp.info.id = exposure.info.id 

486 warp.setPhotoCalib(exposure.getPhotoCalib()) 

487 warp.setFilter(exposure.getFilter()) 

488 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

489 # PSF replaced with CoaddPsf after loop if and only if 

490 # creating direct warp. 

491 warp.setPsf(exposure.getPsf()) 

492 didSetMetadata[warpType] = True 

493 

494 # Need inputRecorder for CoaddApCorrMap for both direct and 

495 # PSF-matched. 

496 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

497 

498 except Exception as e: 

499 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

500 continue 

501 

502 for warpType in warpTypeList: 

503 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

504 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

505 

506 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

507 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType]) 

508 if warpType == "direct": 

509 warps[warpType].setPsf( 

510 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

511 self.config.coaddPsf.makeControl())) 

512 else: 

513 if not self.config.doWriteEmptyWarps: 

514 # No good pixels. Exposure still empty. 

515 warps[warpType] = None 

516 # NoWorkFound is unnecessary as the downstream tasks will 

517 # adjust the quantum accordingly. 

518 

519 result = pipeBase.Struct(exposures=warps) 

520 return result 

521 

522 def filterInputs(self, indices, inputs): 

523 """Filter task inputs by their indices. 

524 

525 Parameters 

526 ---------- 

527 indices : `list` [`int`] 

528 inputs : `dict` [`list`] 

529 A dictionary of input connections to be passed to run. 

530 

531 Returns 

532 ------- 

533 inputs : `dict` [`list`] 

534 Task inputs with their lists filtered by indices. 

535 """ 

536 for key in inputs.keys(): 

537 # Only down-select on list inputs 

538 if isinstance(inputs[key], list): 

539 inputs[key] = [inputs[key][ind] for ind in indices] 

540 return inputs 

541 

542 def _prepareCalibratedExposures(self, calExpList=[], wcsList=None, backgroundList=None, skyCorrList=None, 

543 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

544 finalizedPsfApCorrCatalog=None, visitSummary=None, **kwargs): 

545 """Calibrate and add backgrounds to input calExpList in place. 

546 

547 Parameters 

548 ---------- 

549 calExpList : `list` [`lsst.afw.image.Exposure` or 

550 `lsst.daf.butler.DeferredDatasetHandle`] 

551 Sequence of calexps to be modified in place. 

552 wcsList : `list` [`lsst.afw.geom.SkyWcs`] 

553 The WCSs of the calexps in ``calExpList``. When 

554 ``externalSkyCatalog`` is `None`, these are used to determine if 

555 the calexp should be included in the warp, namely checking that it 

556 is not `None`. If ``externalSkyCatalog`` is not `None`, this list 

557 will be dynamically updated with the external sky WCS. 

558 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional 

559 Sequence of backgrounds to be added back in if bgSubtracted=False. 

560 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional 

561 Sequence of background corrections to be subtracted if 

562 doApplySkyCorr=True. 

563 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

564 Exposure catalog with external skyWcs to be applied 

565 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

566 for the catalog id, sorted on id for fast lookup. 

567 Deprecated and will be removed after v26. 

568 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

569 Exposure catalog with external photoCalib to be applied 

570 if config.doApplyExternalPhotoCalib=True. Catalog uses the 

571 detector id for the catalog id, sorted on id for fast lookup. 

572 Deprecated and will be removed after v26. 

573 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

574 Exposure catalog with finalized psf models and aperture correction 

575 maps to be applied if config.doApplyFinalizedPsf=True. Catalog 

576 uses the detector id for the catalog id, sorted on id for fast 

577 lookup. 

578 Deprecated and will be removed after v26. 

579 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

580 Exposure catalog with potentially all calibrations. Attributes set 

581 to `None` are ignored. 

582 **kwargs 

583 Additional keyword arguments. 

584 

585 Returns 

586 ------- 

587 indices : `list` [`int`] 

588 Indices of ``calExpList`` and friends that have valid 

589 photoCalib/skyWcs. 

590 """ 

591 wcsList = len(calExpList)*[None] if wcsList is None else wcsList 

592 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

593 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

594 

595 includeCalibVar = self.config.includeCalibVar 

596 

597 indices = [] 

598 for index, (calexp, wcs, background, skyCorr) in enumerate(zip(calExpList, 

599 wcsList, 

600 backgroundList, 

601 skyCorrList)): 

602 if externalSkyWcsCatalog is None and wcs is None: 

603 self.log.warning("Detector id %d for visit %d has None for skyWcs and will not be " 

604 "used in the warp", calexp.dataId["detector"], calexp.dataId["visit"]) 

605 continue 

606 

607 if isinstance(calexp, DeferredDatasetHandle): 

608 calexp = calexp.get() 

609 

610 if not self.config.bgSubtracted: 

611 calexp.maskedImage += background.getImage() 

612 

613 detectorId = calexp.info.getDetector().getId() 

614 

615 # Load all calibrations from visitSummary. 

616 if visitSummary is not None: 

617 row = visitSummary.find(detectorId) 

618 if row is None: 

619 raise RuntimeError( 

620 f"Unexpectedly incomplete visitSummary: detector={detectorId} is missing." 

621 ) 

622 if (photoCalib := row.getPhotoCalib()) is not None: 

623 calexp.setPhotoCalib(photoCalib) 

624 if (skyWcs := row.getWcs()) is not None: 

625 calexp.setWcs(skyWcs) 

626 wcsList[index] = skyWcs 

627 if self.config.useVisitSummaryPsf: 

628 if (psf := row.getPsf()) is not None: 

629 calexp.setPsf(psf) 

630 if (apCorrMap := row.getApCorrMap()) is not None: 

631 calexp.info.setApCorrMap(apCorrMap) 

632 # TODO: on DM-39854 the logic in the 'elif' blocks below could 

633 # be moved into 'else' blocks above (or otherwise simplified 

634 # substantially) after the 'external' arguments are removed. 

635 

636 # Find the external photoCalib. 

637 if externalPhotoCalibCatalog is not None: 

638 row = externalPhotoCalibCatalog.find(detectorId) 

639 if row is None: 

640 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

641 "and will not be used in the warp.", detectorId) 

642 continue 

643 photoCalib = row.getPhotoCalib() 

644 if photoCalib is None: 

645 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

646 "and will not be used in the warp.", detectorId) 

647 continue 

648 calexp.setPhotoCalib(photoCalib) 

649 elif photoCalib is None: 

650 self.log.warning("Detector id %s has None for photoCalib in the visit summary " 

651 "and will not be used in the warp.", detectorId) 

652 continue 

653 

654 # Find and apply external skyWcs. 

655 if externalSkyWcsCatalog is not None: 

656 row = externalSkyWcsCatalog.find(detectorId) 

657 if row is None: 

658 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

659 "and will not be used in the warp.", detectorId) 

660 continue 

661 skyWcs = row.getWcs() 

662 wcsList[index] = skyWcs 

663 if skyWcs is None: 

664 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

665 "and will not be used in the warp.", detectorId) 

666 continue 

667 calexp.setWcs(skyWcs) 

668 elif skyWcs is None: 

669 self.log.warning("Detector id %s has None for skyWcs in the visit summary " 

670 "and will not be used in the warp.", detectorId) 

671 continue 

672 

673 # Find and apply finalized psf and aperture correction. 

674 if finalizedPsfApCorrCatalog is not None: 

675 row = finalizedPsfApCorrCatalog.find(detectorId) 

676 if row is None: 

677 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog " 

678 "and will not be used in the warp.", detectorId) 

679 continue 

680 psf = row.getPsf() 

681 if psf is None: 

682 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog " 

683 "and will not be used in the warp.", detectorId) 

684 continue 

685 calexp.setPsf(psf) 

686 apCorrMap = row.getApCorrMap() 

687 if apCorrMap is None: 

688 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog " 

689 "and will not be used in the warp.", detectorId) 

690 continue 

691 calexp.info.setApCorrMap(apCorrMap) 

692 elif self.config.useVisitSummaryPsf: 

693 if psf is None: 

694 self.log.warning("Detector id %s has None for PSF in the visit summary " 

695 "and will not be used in the warp.", detectorId) 

696 if apCorrMap is None: 

697 self.log.warning("Detector id %s has None for ApCorrMap in the visit summary " 

698 "and will not be used in the warp.", detectorId) 

699 else: 

700 if calexp.getPsf() is None: 

701 self.log.warning("Detector id %s has None for PSF in the calexp " 

702 "and will not be used in the warp.", detectorId) 

703 if calexp.info.getApCorrMap() is None: 

704 self.log.warning("Detector id %s has None for ApCorrMap in the calexp " 

705 "and will not be used in the warp.", detectorId) 

706 continue 

707 

708 # Calibrate the image. 

709 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

710 includeScaleUncertainty=includeCalibVar) 

711 calexp.maskedImage /= photoCalib.getCalibrationMean() 

712 # TODO: The images will have a calibration of 1.0 everywhere once 

713 # RFC-545 is implemented. 

714 # exposure.setCalib(afwImage.Calib(1.0)) 

715 

716 # Apply skycorr 

717 if self.config.doApplySkyCorr: 

718 calexp.maskedImage -= skyCorr.getImage() 

719 

720 indices.append(index) 

721 calExpList[index] = calexp 

722 

723 return indices 

724 

725 @staticmethod 

726 def _prepareEmptyExposure(skyInfo): 

727 """Produce an empty exposure for a given patch. 

728 

729 Parameters 

730 ---------- 

731 skyInfo : `lsst.pipe.base.Struct` 

732 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

733 geometric information about the patch. 

734 

735 Returns 

736 ------- 

737 exp : `lsst.afw.image.exposure.ExposureF` 

738 An empty exposure for a given patch. 

739 """ 

740 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

741 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

742 .getPlaneBitMask("NO_DATA"), numpy.inf) 

743 return exp 

744 

745 def getWarpTypeList(self): 

746 """Return list of requested warp types per the config. 

747 """ 

748 warpTypeList = [] 

749 if self.config.makeDirect: 

750 warpTypeList.append("direct") 

751 if self.config.makePsfMatched: 

752 warpTypeList.append("psfMatched") 

753 return warpTypeList 

754 

755 

756def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

757 """Reorder inputRefs per outputSortKeyOrder. 

758 

759 Any inputRefs which are lists will be resorted per specified key e.g., 

760 'detector.' Only iterables will be reordered, and values can be of type 

761 `lsst.pipe.base.connections.DeferredDatasetRef` or 

762 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

763 

764 Returned lists of refs have the same length as the outputSortKeyOrder. 

765 If an outputSortKey not in the inputRef, then it will be padded with None. 

766 If an inputRef contains an inputSortKey that is not in the 

767 outputSortKeyOrder it will be removed. 

768 

769 Parameters 

770 ---------- 

771 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

772 Input references to be reordered and padded. 

773 outputSortKeyOrder : `iterable` 

774 Iterable of values to be compared with inputRef's dataId[dataIdKey]. 

775 dataIdKey : `str` 

776 The data ID key in the dataRefs to compare with the outputSortKeyOrder. 

777 

778 Returns 

779 ------- 

780 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

781 Quantized Connection with sorted DatasetRef values sorted if iterable. 

782 """ 

783 for connectionName, refs in inputRefs: 

784 if isinstance(refs, Iterable): 

785 if hasattr(refs[0], "dataId"): 

786 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

787 else: 

788 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

789 if inputSortKeyOrder != outputSortKeyOrder: 

790 setattr(inputRefs, connectionName, 

791 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

792 return inputRefs