Coverage for python/lsst/pipe/tasks/makeWarp.py: 15%

285 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-02 10:25 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["MakeWarpTask", "MakeWarpConfig"] 

23 

24from deprecated.sphinx import deprecated 

25import logging 

26import numpy 

27 

28import lsst.pex.config as pexConfig 

29import lsst.afw.image as afwImage 

30import lsst.coadd.utils as coaddUtils 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as connectionTypes 

33import lsst.utils as utils 

34import lsst.geom 

35from lsst.daf.butler import DeferredDatasetHandle 

36from lsst.meas.base import DetectorVisitIdGeneratorConfig 

37from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

38from lsst.skymap import BaseSkyMap 

39from lsst.utils.timer import timeMethod 

40from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

41from .warpAndPsfMatch import WarpAndPsfMatchTask 

42from collections.abc import Iterable 

43 

44log = logging.getLogger(__name__) 

45 

46 

47class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

48 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

49 defaultTemplates={"coaddName": "deep", 

50 "skyWcsName": "gbdesAstrometricFit", 

51 "photoCalibName": "fgcm", 

52 "calexpType": ""}): 

53 calExpList = connectionTypes.Input( 

54 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

55 name="{calexpType}calexp", 

56 storageClass="ExposureF", 

57 dimensions=("instrument", "visit", "detector"), 

58 multiple=True, 

59 deferLoad=True, 

60 ) 

61 backgroundList = connectionTypes.Input( 

62 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

63 name="calexpBackground", 

64 storageClass="Background", 

65 dimensions=("instrument", "visit", "detector"), 

66 multiple=True, 

67 ) 

68 skyCorrList = connectionTypes.Input( 

69 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

70 name="skyCorr", 

71 storageClass="Background", 

72 dimensions=("instrument", "visit", "detector"), 

73 multiple=True, 

74 ) 

75 skyMap = connectionTypes.Input( 

76 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

78 storageClass="SkyMap", 

79 dimensions=("skymap",), 

80 ) 

81 externalSkyWcsTractCatalog = connectionTypes.Input( 

82 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

83 "id for the catalog id, sorted on id for fast lookup."), 

84 name="{skyWcsName}SkyWcsCatalog", 

85 storageClass="ExposureCatalog", 

86 dimensions=("instrument", "visit", "tract"), 

87 ) 

88 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

89 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

90 "These catalogs use the detector id for the catalog id, sorted on id for " 

91 "fast lookup."), 

92 name="finalVisitSummary", 

93 storageClass="ExposureCatalog", 

94 dimensions=("instrument", "visit"), 

95 ) 

96 externalPhotoCalibTractCatalog = connectionTypes.Input( 

97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

98 "detector id for the catalog id, sorted on id for fast lookup."), 

99 name="{photoCalibName}PhotoCalibCatalog", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit", "tract"), 

102 ) 

103 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

104 doc=("Per-visit photometric calibrations computed globally (with no tract " 

105 "information). These catalogs use the detector id for the catalog id, " 

106 "sorted on id for fast lookup."), 

107 name="finalVisitSummary", 

108 storageClass="ExposureCatalog", 

109 dimensions=("instrument", "visit"), 

110 ) 

111 finalizedPsfApCorrCatalog = connectionTypes.Input( 

112 doc=("Per-visit finalized psf models and aperture correction maps. " 

113 "These catalogs use the detector id for the catalog id, " 

114 "sorted on id for fast lookup."), 

115 name="finalVisitSummary", 

116 storageClass="ExposureCatalog", 

117 dimensions=("instrument", "visit"), 

118 ) 

119 direct = connectionTypes.Output( 

120 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

121 "calexps onto the skyMap patch geometry."), 

122 name="{coaddName}Coadd_directWarp", 

123 storageClass="ExposureF", 

124 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

125 ) 

126 psfMatched = connectionTypes.Output( 

127 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

128 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

129 name="{coaddName}Coadd_psfMatchedWarp", 

130 storageClass="ExposureF", 

131 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

132 ) 

133 # TODO DM-28769, have selectImages subtask indicate which connections they 

134 # need: 

135 wcsList = connectionTypes.Input( 

136 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

137 name="{calexpType}calexp.wcs", 

138 storageClass="Wcs", 

139 dimensions=("instrument", "visit", "detector"), 

140 multiple=True, 

141 ) 

142 bboxList = connectionTypes.Input( 

143 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

144 name="{calexpType}calexp.bbox", 

145 storageClass="Box2I", 

146 dimensions=("instrument", "visit", "detector"), 

147 multiple=True, 

148 ) 

149 visitSummary = connectionTypes.Input( 

150 doc="Consolidated exposure metadata", 

151 name="finalVisitSummary", 

152 storageClass="ExposureCatalog", 

153 dimensions=("instrument", "visit",), 

154 ) 

155 

156 def __init__(self, *, config=None): 

157 super().__init__(config=config) 

158 if config.bgSubtracted: 

159 self.inputs.remove("backgroundList") 

160 if not config.doApplySkyCorr: 

161 self.inputs.remove("skyCorrList") 

162 if config.doApplyExternalSkyWcs: 

163 if config.useGlobalExternalSkyWcs: 

164 self.inputs.remove("externalSkyWcsTractCatalog") 

165 else: 

166 self.inputs.remove("externalSkyWcsGlobalCatalog") 

167 else: 

168 self.inputs.remove("externalSkyWcsTractCatalog") 

169 self.inputs.remove("externalSkyWcsGlobalCatalog") 

170 if config.doApplyExternalPhotoCalib: 

171 if config.useGlobalExternalPhotoCalib: 

172 self.inputs.remove("externalPhotoCalibTractCatalog") 

173 else: 

174 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

175 else: 

176 self.inputs.remove("externalPhotoCalibTractCatalog") 

177 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

178 if not config.doApplyFinalizedPsf: 

179 self.inputs.remove("finalizedPsfApCorrCatalog") 

180 if not config.makeDirect: 

181 self.outputs.remove("direct") 

182 if not config.makePsfMatched: 

183 self.outputs.remove("psfMatched") 

184 # TODO DM-28769: add connection per selectImages connections 

185 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

186 self.inputs.remove("visitSummary") 

187 

188 

189class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, 

190 pipelineConnections=MakeWarpConnections): 

191 """Config for MakeWarpTask.""" 

192 

193 warpAndPsfMatch = pexConfig.ConfigurableField( 

194 target=WarpAndPsfMatchTask, 

195 doc="Task to warp and PSF-match calexp", 

196 ) 

197 doWrite = pexConfig.Field( 

198 doc="persist <coaddName>Coadd_<warpType>Warp", 

199 dtype=bool, 

200 default=True, 

201 ) 

202 bgSubtracted = pexConfig.Field( 

203 doc="Work with a background subtracted calexp?", 

204 dtype=bool, 

205 default=True, 

206 ) 

207 coaddPsf = pexConfig.ConfigField( 

208 doc="Configuration for CoaddPsf", 

209 dtype=CoaddPsfConfig, 

210 ) 

211 makeDirect = pexConfig.Field( 

212 doc="Make direct Warp/Coadds", 

213 dtype=bool, 

214 default=True, 

215 ) 

216 makePsfMatched = pexConfig.Field( 

217 doc="Make Psf-Matched Warp/Coadd?", 

218 dtype=bool, 

219 default=False, 

220 ) 

221 doWriteEmptyWarps = pexConfig.Field( 

222 dtype=bool, 

223 default=False, 

224 doc="Write out warps even if they are empty" 

225 ) 

226 hasFakes = pexConfig.Field( 

227 doc="Should be set to True if fake sources have been inserted into the input data.", 

228 dtype=bool, 

229 default=False, 

230 ) 

231 doApplySkyCorr = pexConfig.Field( 

232 dtype=bool, 

233 default=False, 

234 doc="Apply sky correction?", 

235 ) 

236 doApplyFinalizedPsf = pexConfig.Field( 

237 doc="Whether to apply finalized psf models and aperture correction map.", 

238 dtype=bool, 

239 default=True, 

240 ) 

241 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

242 

243 def validate(self): 

244 CoaddBaseTask.ConfigClass.validate(self) 

245 

246 if not self.makePsfMatched and not self.makeDirect: 

247 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

248 if self.doPsfMatch: 

249 # Backwards compatibility. 

250 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

251 self.makePsfMatched = True 

252 self.makeDirect = False 

253 

254 def setDefaults(self): 

255 CoaddBaseTask.ConfigClass.setDefaults(self) 

256 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

257 

258 

259class MakeWarpTask(CoaddBaseTask): 

260 """Warp and optionally PSF-Match calexps onto an a common projection. 

261 

262 Warp and optionally PSF-Match calexps onto a common projection, by 

263 performing the following operations: 

264 - Group calexps by visit/run 

265 - For each visit, generate a Warp by calling method @ref run. 

266 `run` loops over the visit's calexps calling 

267 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit 

268 

269 """ 

270 ConfigClass = MakeWarpConfig 

271 _DefaultName = "makeWarp" 

272 

273 def __init__(self, **kwargs): 

274 CoaddBaseTask.__init__(self, **kwargs) 

275 self.makeSubtask("warpAndPsfMatch") 

276 if self.config.hasFakes: 

277 self.calexpType = "fakes_calexp" 

278 else: 

279 self.calexpType = "calexp" 

280 

281 @utils.inheritDoc(pipeBase.PipelineTask) 

282 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

283 # Obtain the list of input detectors from calExpList. Sort them by 

284 # detector order (to ensure reproducibility). Then ensure all input 

285 # lists are in the same sorted detector order. 

286 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

287 detectorOrder.sort() 

288 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

289 

290 # Read in all inputs. 

291 inputs = butlerQC.get(inputRefs) 

292 

293 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

294 # from the dictionary so we can pass it as kwargs later. 

295 skyMap = inputs.pop("skyMap") 

296 quantumDataId = butlerQC.quantum.dataId 

297 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

298 

299 # Construct list of input DataIds expected by `run`. 

300 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

301 # Construct list of packed integer IDs expected by `run`. 

302 ccdIdList = [ 

303 self.config.idGenerator.apply(dataId).catalog_id 

304 for dataId in dataIdList 

305 ] 

306 

307 if self.config.doApplyExternalSkyWcs: 

308 if self.config.useGlobalExternalSkyWcs: 

309 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

310 else: 

311 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

312 else: 

313 externalSkyWcsCatalog = None 

314 

315 if self.config.doApplyExternalPhotoCalib: 

316 if self.config.useGlobalExternalPhotoCalib: 

317 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

318 else: 

319 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

320 else: 

321 externalPhotoCalibCatalog = None 

322 

323 if self.config.doApplyFinalizedPsf: 

324 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog") 

325 else: 

326 finalizedPsfApCorrCatalog = None 

327 

328 # Do an initial selection on inputs with complete wcs/photoCalib info. 

329 # Qualifying calexps will be read in the following call. 

330 completeIndices = self._prepareCalibratedExposures( 

331 **inputs, 

332 externalSkyWcsCatalog=externalSkyWcsCatalog, 

333 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

334 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

335 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

336 

337 # Do another selection based on the configured selection task 

338 # (using updated WCSs to determine patch overlap if an external 

339 # calibration was applied). 

340 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

341 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

342 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

343 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

344 

345 # Extract integer visitId requested by `run`. 

346 visitId = dataIdList[0]["visit"] 

347 

348 results = self.run(**inputs, visitId=visitId, 

349 ccdIdList=[ccdIdList[i] for i in goodIndices], 

350 dataIdList=[dataIdList[i] for i in goodIndices], 

351 skyInfo=skyInfo) 

352 if self.config.makeDirect and results.exposures["direct"] is not None: 

353 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

354 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

355 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

356 

357 @timeMethod 

358 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

359 """Create a Warp from inputs. 

360 

361 We iterate over the multiple calexps in a single exposure to construct 

362 the warp (previously called a coaddTempExp) of that exposure to the 

363 supplied tract/patch. 

364 

365 Pixels that receive no pixels are set to NAN; this is not correct 

366 (violates LSST algorithms group policy), but will be fixed up by 

367 interpolating after the coaddition. 

368 

369 calexpRefList : `list` 

370 List of data references for calexps that (may) 

371 overlap the patch of interest. 

372 skyInfo : `lsst.pipe.base.Struct` 

373 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

374 geometric information about the patch. 

375 visitId : `int` 

376 Integer identifier for visit, for the table that will 

377 produce the CoaddPsf. 

378 

379 Returns 

380 ------- 

381 result : `lsst.pipe.base.Struct` 

382 Results as a struct with attributes: 

383 

384 ``exposures`` 

385 A dictionary containing the warps requested: 

386 "direct": direct warp if ``config.makeDirect`` 

387 "psfMatched": PSF-matched warp if ``config.makePsfMatched`` 

388 (`dict`). 

389 """ 

390 warpTypeList = self.getWarpTypeList() 

391 

392 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

393 didSetMetadata = {warpType: False for warpType in warpTypeList} 

394 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

395 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

396 for warpType in warpTypeList} 

397 

398 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

399 if dataIdList is None: 

400 dataIdList = ccdIdList 

401 

402 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

403 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

404 calExpInd+1, len(calExpList), dataId) 

405 # TODO: The following conditional is only required for backwards 

406 # compatibility with the deprecated prepareCalibratedExposures() 

407 # method. Can remove with its removal after the deprecation 

408 # period. 

409 if isinstance(calExp, DeferredDatasetHandle): 

410 calExp = calExp.get() 

411 try: 

412 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

413 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

414 makeDirect=self.config.makeDirect, 

415 makePsfMatched=self.config.makePsfMatched) 

416 except Exception as e: 

417 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

418 continue 

419 try: 

420 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

421 for warpType in warpTypeList: 

422 exposure = warpedAndMatched.getDict()[warpType] 

423 if exposure is None: 

424 continue 

425 warp = warps[warpType] 

426 if didSetMetadata[warpType]: 

427 mimg = exposure.getMaskedImage() 

428 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

429 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

430 del mimg 

431 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

432 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

433 totGoodPix[warpType] += numGoodPix[warpType] 

434 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

435 dataId, numGoodPix[warpType], 

436 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

437 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

438 warp.info.id = exposure.info.id 

439 warp.setPhotoCalib(exposure.getPhotoCalib()) 

440 warp.setFilter(exposure.getFilter()) 

441 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

442 # PSF replaced with CoaddPsf after loop if and only if 

443 # creating direct warp. 

444 warp.setPsf(exposure.getPsf()) 

445 didSetMetadata[warpType] = True 

446 

447 # Need inputRecorder for CoaddApCorrMap for both direct and 

448 # PSF-matched. 

449 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

450 

451 except Exception as e: 

452 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

453 continue 

454 

455 for warpType in warpTypeList: 

456 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

457 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

458 

459 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

460 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType]) 

461 if warpType == "direct": 

462 warps[warpType].setPsf( 

463 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

464 self.config.coaddPsf.makeControl())) 

465 else: 

466 if not self.config.doWriteEmptyWarps: 

467 # No good pixels. Exposure still empty. 

468 warps[warpType] = None 

469 # NoWorkFound is unnecessary as the downstream tasks will 

470 # adjust the quantum accordingly. 

471 

472 result = pipeBase.Struct(exposures=warps) 

473 return result 

474 

475 def filterInputs(self, indices, inputs): 

476 """Filter task inputs by their indices. 

477 

478 Parameters 

479 ---------- 

480 indices : `list` [`int`] 

481 inputs : `dict` [`list`] 

482 A dictionary of input connections to be passed to run. 

483 

484 Returns 

485 ------- 

486 inputs : `dict` [`list`] 

487 Task inputs with their lists filtered by indices. 

488 """ 

489 for key in inputs.keys(): 

490 # Only down-select on list inputs 

491 if isinstance(inputs[key], list): 

492 inputs[key] = [inputs[key][ind] for ind in indices] 

493 return inputs 

494 

495 @deprecated(reason="This method is deprecated in favor of its leading underscore version, " 

496 "_prepareCalibratedfExposures(). Will be removed after v25.", 

497 version="v25.0", category=FutureWarning) 

498 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

499 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

500 finalizedPsfApCorrCatalog=None, 

501 **kwargs): 

502 """Deprecated function. 

503 

504 Please use _prepareCalibratedExposure(), which this delegates to and 

505 noting its slightly updated API, instead. 

506 """ 

507 # Read in all calexps. 

508 calExpList = [ref.get() for ref in calExpList] 

509 # Populate wcsList as required by new underscored version of function. 

510 wcsList = [calexp.getWcs() for calexp in calExpList] 

511 

512 indices = self._prepareCalibratedExposures(calExpList=calExpList, wcsList=wcsList, 

513 backgroundList=backgroundList, skyCorrList=skyCorrList, 

514 externalSkyWcsCatalog=externalSkyWcsCatalog, 

515 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

516 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

517 return indices 

518 

519 def _prepareCalibratedExposures(self, calExpList=[], wcsList=None, backgroundList=None, skyCorrList=None, 

520 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

521 finalizedPsfApCorrCatalog=None, **kwargs): 

522 """Calibrate and add backgrounds to input calExpList in place. 

523 

524 Parameters 

525 ---------- 

526 calExpList : `list` [`lsst.afw.image.Exposure` or 

527 `lsst.daf.butler.DeferredDatasetHandle`] 

528 Sequence of calexps to be modified in place. 

529 wcsList : `list` [`lsst.afw.geom.SkyWcs`] 

530 The WCSs of the calexps in ``calExpList``. When 

531 ``externalSkyCatalog`` is `None`, these are used to determine if 

532 the calexp should be included in the warp, namely checking that it 

533 is not `None`. If ``externalSkyCatalog`` is not `None`, this list 

534 will be dynamically updated with the external sky WCS. 

535 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional 

536 Sequence of backgrounds to be added back in if bgSubtracted=False. 

537 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional 

538 Sequence of background corrections to be subtracted if 

539 doApplySkyCorr=True. 

540 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

541 Exposure catalog with external skyWcs to be applied 

542 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

543 for the catalog id, sorted on id for fast lookup. 

544 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

545 Exposure catalog with external photoCalib to be applied 

546 if config.doApplyExternalPhotoCalib=True. Catalog uses the 

547 detector id for the catalog id, sorted on id for fast lookup. 

548 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

549 Exposure catalog with finalized psf models and aperture correction 

550 maps to be applied if config.doApplyFinalizedPsf=True. Catalog 

551 uses the detector id for the catalog id, sorted on id for fast 

552 lookup. 

553 **kwargs 

554 Additional keyword arguments. 

555 

556 Returns 

557 ------- 

558 indices : `list` [`int`] 

559 Indices of ``calExpList`` and friends that have valid 

560 photoCalib/skyWcs. 

561 """ 

562 wcsList = len(calExpList)*[None] if wcsList is None else wcsList 

563 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

564 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

565 

566 includeCalibVar = self.config.includeCalibVar 

567 

568 indices = [] 

569 for index, (calexp, wcs, background, skyCorr) in enumerate(zip(calExpList, 

570 wcsList, 

571 backgroundList, 

572 skyCorrList)): 

573 if externalSkyWcsCatalog is None and wcs is None: 

574 self.log.warning("Detector id %d for visit %d has None for skyWcs and will not be " 

575 "used in the warp", calexp.dataId["detector"], calexp.dataId["visit"]) 

576 continue 

577 

578 if isinstance(calexp, DeferredDatasetHandle): 

579 calexp = calexp.get() 

580 

581 if not self.config.bgSubtracted: 

582 calexp.maskedImage += background.getImage() 

583 

584 detectorId = calexp.info.getDetector().getId() 

585 

586 # Find the external photoCalib. 

587 if externalPhotoCalibCatalog is not None: 

588 row = externalPhotoCalibCatalog.find(detectorId) 

589 if row is None: 

590 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

591 "and will not be used in the warp.", detectorId) 

592 continue 

593 photoCalib = row.getPhotoCalib() 

594 if photoCalib is None: 

595 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

596 "and will not be used in the warp.", detectorId) 

597 continue 

598 calexp.setPhotoCalib(photoCalib) 

599 else: 

600 photoCalib = calexp.getPhotoCalib() 

601 if photoCalib is None: 

602 self.log.warning("Detector id %s has None for photoCalib in the calexp " 

603 "and will not be used in the warp.", detectorId) 

604 continue 

605 

606 # Find and apply external skyWcs. 

607 if externalSkyWcsCatalog is not None: 

608 row = externalSkyWcsCatalog.find(detectorId) 

609 if row is None: 

610 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

611 "and will not be used in the warp.", detectorId) 

612 continue 

613 skyWcs = row.getWcs() 

614 wcsList[index] = skyWcs 

615 if skyWcs is None: 

616 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

617 "and will not be used in the warp.", detectorId) 

618 continue 

619 calexp.setWcs(skyWcs) 

620 else: 

621 skyWcs = calexp.getWcs() 

622 wcsList[index] = skyWcs 

623 if skyWcs is None: 

624 self.log.warning("Detector id %s has None for skyWcs in the calexp " 

625 "and will not be used in the warp.", detectorId) 

626 continue 

627 

628 # Find and apply finalized psf and aperture correction. 

629 if finalizedPsfApCorrCatalog is not None: 

630 row = finalizedPsfApCorrCatalog.find(detectorId) 

631 if row is None: 

632 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog " 

633 "and will not be used in the warp.", detectorId) 

634 continue 

635 psf = row.getPsf() 

636 if psf is None: 

637 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog " 

638 "and will not be used in the warp.", detectorId) 

639 continue 

640 calexp.setPsf(psf) 

641 apCorrMap = row.getApCorrMap() 

642 if apCorrMap is None: 

643 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog " 

644 "and will not be used in the warp.", detectorId) 

645 continue 

646 calexp.info.setApCorrMap(apCorrMap) 

647 else: 

648 # Ensure that calexp has valid aperture correction map. 

649 if calexp.info.getApCorrMap() is None: 

650 self.log.warning("Detector id %s has None for ApCorrMap in the calexp " 

651 "and will not be used in the warp.", detectorId) 

652 continue 

653 

654 # Calibrate the image. 

655 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

656 includeScaleUncertainty=includeCalibVar) 

657 calexp.maskedImage /= photoCalib.getCalibrationMean() 

658 # TODO: The images will have a calibration of 1.0 everywhere once 

659 # RFC-545 is implemented. 

660 # exposure.setCalib(afwImage.Calib(1.0)) 

661 

662 # Apply skycorr 

663 if self.config.doApplySkyCorr: 

664 calexp.maskedImage -= skyCorr.getImage() 

665 

666 indices.append(index) 

667 calExpList[index] = calexp 

668 

669 return indices 

670 

671 @staticmethod 

672 def _prepareEmptyExposure(skyInfo): 

673 """Produce an empty exposure for a given patch. 

674 

675 Parameters 

676 ---------- 

677 skyInfo : `lsst.pipe.base.Struct` 

678 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

679 geometric information about the patch. 

680 

681 Returns 

682 ------- 

683 exp : `lsst.afw.image.exposure.ExposureF` 

684 An empty exposure for a given patch. 

685 """ 

686 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

687 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

688 .getPlaneBitMask("NO_DATA"), numpy.inf) 

689 return exp 

690 

691 def getWarpTypeList(self): 

692 """Return list of requested warp types per the config. 

693 """ 

694 warpTypeList = [] 

695 if self.config.makeDirect: 

696 warpTypeList.append("direct") 

697 if self.config.makePsfMatched: 

698 warpTypeList.append("psfMatched") 

699 return warpTypeList 

700 

701 

702def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

703 """Reorder inputRefs per outputSortKeyOrder. 

704 

705 Any inputRefs which are lists will be resorted per specified key e.g., 

706 'detector.' Only iterables will be reordered, and values can be of type 

707 `lsst.pipe.base.connections.DeferredDatasetRef` or 

708 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

709 

710 Returned lists of refs have the same length as the outputSortKeyOrder. 

711 If an outputSortKey not in the inputRef, then it will be padded with None. 

712 If an inputRef contains an inputSortKey that is not in the 

713 outputSortKeyOrder it will be removed. 

714 

715 Parameters 

716 ---------- 

717 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

718 Input references to be reordered and padded. 

719 outputSortKeyOrder : `iterable` 

720 Iterable of values to be compared with inputRef's dataId[dataIdKey]. 

721 dataIdKey : `str` 

722 The data ID key in the dataRefs to compare with the outputSortKeyOrder. 

723 

724 Returns 

725 ------- 

726 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

727 Quantized Connection with sorted DatasetRef values sorted if iterable. 

728 """ 

729 for connectionName, refs in inputRefs: 

730 if isinstance(refs, Iterable): 

731 if hasattr(refs[0], "dataId"): 

732 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

733 else: 

734 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

735 if inputSortKeyOrder != outputSortKeyOrder: 

736 setattr(inputRefs, connectionName, 

737 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

738 return inputRefs