Coverage for python/lsst/pipe/tasks/makeWarp.py: 15%

276 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-12 10:09 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["MakeWarpTask", "MakeWarpConfig"] 

23 

24import logging 

25import numpy 

26 

27import lsst.pex.config as pexConfig 

28import lsst.afw.image as afwImage 

29import lsst.coadd.utils as coaddUtils 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as connectionTypes 

32import lsst.utils as utils 

33import lsst.geom 

34from lsst.daf.butler import DeferredDatasetHandle 

35from lsst.meas.base import DetectorVisitIdGeneratorConfig 

36from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

37from lsst.skymap import BaseSkyMap 

38from lsst.utils.timer import timeMethod 

39from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

40from .warpAndPsfMatch import WarpAndPsfMatchTask 

41from collections.abc import Iterable 

42 

43log = logging.getLogger(__name__) 

44 

45 

46class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

48 defaultTemplates={"coaddName": "deep", 

49 "skyWcsName": "gbdesAstrometricFit", 

50 "photoCalibName": "fgcm", 

51 "calexpType": ""}): 

52 calExpList = connectionTypes.Input( 

53 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

54 name="{calexpType}calexp", 

55 storageClass="ExposureF", 

56 dimensions=("instrument", "visit", "detector"), 

57 multiple=True, 

58 deferLoad=True, 

59 ) 

60 backgroundList = connectionTypes.Input( 

61 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

62 name="calexpBackground", 

63 storageClass="Background", 

64 dimensions=("instrument", "visit", "detector"), 

65 multiple=True, 

66 ) 

67 skyCorrList = connectionTypes.Input( 

68 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

69 name="skyCorr", 

70 storageClass="Background", 

71 dimensions=("instrument", "visit", "detector"), 

72 multiple=True, 

73 ) 

74 skyMap = connectionTypes.Input( 

75 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

76 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

77 storageClass="SkyMap", 

78 dimensions=("skymap",), 

79 ) 

80 externalSkyWcsTractCatalog = connectionTypes.Input( 

81 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

82 "id for the catalog id, sorted on id for fast lookup."), 

83 name="{skyWcsName}SkyWcsCatalog", 

84 storageClass="ExposureCatalog", 

85 dimensions=("instrument", "visit", "tract"), 

86 ) 

87 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

88 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

89 "These catalogs use the detector id for the catalog id, sorted on id for " 

90 "fast lookup."), 

91 name="finalVisitSummary", 

92 storageClass="ExposureCatalog", 

93 dimensions=("instrument", "visit"), 

94 ) 

95 externalPhotoCalibTractCatalog = connectionTypes.Input( 

96 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

97 "detector id for the catalog id, sorted on id for fast lookup."), 

98 name="{photoCalibName}PhotoCalibCatalog", 

99 storageClass="ExposureCatalog", 

100 dimensions=("instrument", "visit", "tract"), 

101 ) 

102 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

103 doc=("Per-visit photometric calibrations computed globally (with no tract " 

104 "information). These catalogs use the detector id for the catalog id, " 

105 "sorted on id for fast lookup."), 

106 name="finalVisitSummary", 

107 storageClass="ExposureCatalog", 

108 dimensions=("instrument", "visit"), 

109 ) 

110 finalizedPsfApCorrCatalog = connectionTypes.Input( 

111 doc=("Per-visit finalized psf models and aperture correction maps. " 

112 "These catalogs use the detector id for the catalog id, " 

113 "sorted on id for fast lookup."), 

114 name="finalVisitSummary", 

115 storageClass="ExposureCatalog", 

116 dimensions=("instrument", "visit"), 

117 ) 

118 direct = connectionTypes.Output( 

119 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

120 "calexps onto the skyMap patch geometry."), 

121 name="{coaddName}Coadd_directWarp", 

122 storageClass="ExposureF", 

123 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

124 ) 

125 psfMatched = connectionTypes.Output( 

126 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

127 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

128 name="{coaddName}Coadd_psfMatchedWarp", 

129 storageClass="ExposureF", 

130 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

131 ) 

132 # TODO DM-28769, have selectImages subtask indicate which connections they 

133 # need: 

134 wcsList = connectionTypes.Input( 

135 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

136 name="{calexpType}calexp.wcs", 

137 storageClass="Wcs", 

138 dimensions=("instrument", "visit", "detector"), 

139 multiple=True, 

140 ) 

141 bboxList = connectionTypes.Input( 

142 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

143 name="{calexpType}calexp.bbox", 

144 storageClass="Box2I", 

145 dimensions=("instrument", "visit", "detector"), 

146 multiple=True, 

147 ) 

148 visitSummary = connectionTypes.Input( 

149 doc="Consolidated exposure metadata", 

150 name="finalVisitSummary", 

151 storageClass="ExposureCatalog", 

152 dimensions=("instrument", "visit",), 

153 ) 

154 

155 def __init__(self, *, config=None): 

156 super().__init__(config=config) 

157 if config.bgSubtracted: 

158 self.inputs.remove("backgroundList") 

159 if not config.doApplySkyCorr: 

160 self.inputs.remove("skyCorrList") 

161 if config.doApplyExternalSkyWcs: 

162 if config.useGlobalExternalSkyWcs: 

163 self.inputs.remove("externalSkyWcsTractCatalog") 

164 else: 

165 self.inputs.remove("externalSkyWcsGlobalCatalog") 

166 else: 

167 self.inputs.remove("externalSkyWcsTractCatalog") 

168 self.inputs.remove("externalSkyWcsGlobalCatalog") 

169 if config.doApplyExternalPhotoCalib: 

170 if config.useGlobalExternalPhotoCalib: 

171 self.inputs.remove("externalPhotoCalibTractCatalog") 

172 else: 

173 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

174 else: 

175 self.inputs.remove("externalPhotoCalibTractCatalog") 

176 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

177 if not config.doApplyFinalizedPsf: 

178 self.inputs.remove("finalizedPsfApCorrCatalog") 

179 if not config.makeDirect: 

180 self.outputs.remove("direct") 

181 if not config.makePsfMatched: 

182 self.outputs.remove("psfMatched") 

183 # TODO DM-28769: add connection per selectImages connections 

184 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

185 self.inputs.remove("visitSummary") 

186 

187 

188class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, 

189 pipelineConnections=MakeWarpConnections): 

190 """Config for MakeWarpTask.""" 

191 

192 warpAndPsfMatch = pexConfig.ConfigurableField( 

193 target=WarpAndPsfMatchTask, 

194 doc="Task to warp and PSF-match calexp", 

195 ) 

196 doWrite = pexConfig.Field( 

197 doc="persist <coaddName>Coadd_<warpType>Warp", 

198 dtype=bool, 

199 default=True, 

200 ) 

201 bgSubtracted = pexConfig.Field( 

202 doc="Work with a background subtracted calexp?", 

203 dtype=bool, 

204 default=True, 

205 ) 

206 coaddPsf = pexConfig.ConfigField( 

207 doc="Configuration for CoaddPsf", 

208 dtype=CoaddPsfConfig, 

209 ) 

210 makeDirect = pexConfig.Field( 

211 doc="Make direct Warp/Coadds", 

212 dtype=bool, 

213 default=True, 

214 ) 

215 makePsfMatched = pexConfig.Field( 

216 doc="Make Psf-Matched Warp/Coadd?", 

217 dtype=bool, 

218 default=False, 

219 ) 

220 doWriteEmptyWarps = pexConfig.Field( 

221 dtype=bool, 

222 default=False, 

223 doc="Write out warps even if they are empty" 

224 ) 

225 hasFakes = pexConfig.Field( 

226 doc="Should be set to True if fake sources have been inserted into the input data.", 

227 dtype=bool, 

228 default=False, 

229 ) 

230 doApplySkyCorr = pexConfig.Field( 

231 dtype=bool, 

232 default=False, 

233 doc="Apply sky correction?", 

234 ) 

235 doApplyFinalizedPsf = pexConfig.Field( 

236 doc="Whether to apply finalized psf models and aperture correction map.", 

237 dtype=bool, 

238 default=True, 

239 ) 

240 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

241 

242 def validate(self): 

243 CoaddBaseTask.ConfigClass.validate(self) 

244 

245 if not self.makePsfMatched and not self.makeDirect: 

246 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

247 if self.doPsfMatch: # TODO: Remove this in DM-39841 

248 # Backwards compatibility. 

249 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

250 self.makePsfMatched = True 

251 self.makeDirect = False 

252 

253 def setDefaults(self): 

254 CoaddBaseTask.ConfigClass.setDefaults(self) 

255 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

256 

257 

258class MakeWarpTask(CoaddBaseTask): 

259 """Warp and optionally PSF-Match calexps onto an a common projection. 

260 

261 Warp and optionally PSF-Match calexps onto a common projection, by 

262 performing the following operations: 

263 - Group calexps by visit/run 

264 - For each visit, generate a Warp by calling method @ref run. 

265 `run` loops over the visit's calexps calling 

266 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit 

267 

268 """ 

269 ConfigClass = MakeWarpConfig 

270 _DefaultName = "makeWarp" 

271 

272 def __init__(self, **kwargs): 

273 CoaddBaseTask.__init__(self, **kwargs) 

274 self.makeSubtask("warpAndPsfMatch") 

275 if self.config.hasFakes: 

276 self.calexpType = "fakes_calexp" 

277 else: 

278 self.calexpType = "calexp" 

279 

280 @utils.inheritDoc(pipeBase.PipelineTask) 

281 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

282 # Obtain the list of input detectors from calExpList. Sort them by 

283 # detector order (to ensure reproducibility). Then ensure all input 

284 # lists are in the same sorted detector order. 

285 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

286 detectorOrder.sort() 

287 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

288 

289 # Read in all inputs. 

290 inputs = butlerQC.get(inputRefs) 

291 

292 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

293 # from the dictionary so we can pass it as kwargs later. 

294 skyMap = inputs.pop("skyMap") 

295 quantumDataId = butlerQC.quantum.dataId 

296 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

297 

298 # Construct list of input DataIds expected by `run`. 

299 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

300 # Construct list of packed integer IDs expected by `run`. 

301 ccdIdList = [ 

302 self.config.idGenerator.apply(dataId).catalog_id 

303 for dataId in dataIdList 

304 ] 

305 

306 if self.config.doApplyExternalSkyWcs: 

307 if self.config.useGlobalExternalSkyWcs: 

308 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

309 else: 

310 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

311 else: 

312 externalSkyWcsCatalog = None 

313 

314 if self.config.doApplyExternalPhotoCalib: 

315 if self.config.useGlobalExternalPhotoCalib: 

316 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

317 else: 

318 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

319 else: 

320 externalPhotoCalibCatalog = None 

321 

322 if self.config.doApplyFinalizedPsf: 

323 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog") 

324 else: 

325 finalizedPsfApCorrCatalog = None 

326 

327 # Do an initial selection on inputs with complete wcs/photoCalib info. 

328 # Qualifying calexps will be read in the following call. 

329 completeIndices = self._prepareCalibratedExposures( 

330 **inputs, 

331 externalSkyWcsCatalog=externalSkyWcsCatalog, 

332 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

333 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

334 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

335 

336 # Do another selection based on the configured selection task 

337 # (using updated WCSs to determine patch overlap if an external 

338 # calibration was applied). 

339 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

340 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

341 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

342 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

343 

344 # Extract integer visitId requested by `run`. 

345 visitId = dataIdList[0]["visit"] 

346 

347 results = self.run(**inputs, visitId=visitId, 

348 ccdIdList=[ccdIdList[i] for i in goodIndices], 

349 dataIdList=[dataIdList[i] for i in goodIndices], 

350 skyInfo=skyInfo) 

351 if self.config.makeDirect and results.exposures["direct"] is not None: 

352 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

353 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

354 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

355 

356 @timeMethod 

357 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

358 """Create a Warp from inputs. 

359 

360 We iterate over the multiple calexps in a single exposure to construct 

361 the warp (previously called a coaddTempExp) of that exposure to the 

362 supplied tract/patch. 

363 

364 Pixels that receive no pixels are set to NAN; this is not correct 

365 (violates LSST algorithms group policy), but will be fixed up by 

366 interpolating after the coaddition. 

367 

368 calexpRefList : `list` 

369 List of data references for calexps that (may) 

370 overlap the patch of interest. 

371 skyInfo : `lsst.pipe.base.Struct` 

372 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

373 geometric information about the patch. 

374 visitId : `int` 

375 Integer identifier for visit, for the table that will 

376 produce the CoaddPsf. 

377 

378 Returns 

379 ------- 

380 result : `lsst.pipe.base.Struct` 

381 Results as a struct with attributes: 

382 

383 ``exposures`` 

384 A dictionary containing the warps requested: 

385 "direct": direct warp if ``config.makeDirect`` 

386 "psfMatched": PSF-matched warp if ``config.makePsfMatched`` 

387 (`dict`). 

388 """ 

389 warpTypeList = self.getWarpTypeList() 

390 

391 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

392 didSetMetadata = {warpType: False for warpType in warpTypeList} 

393 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

394 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

395 for warpType in warpTypeList} 

396 

397 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

398 if dataIdList is None: 

399 dataIdList = ccdIdList 

400 

401 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

402 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

403 calExpInd+1, len(calExpList), dataId) 

404 try: 

405 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

406 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

407 makeDirect=self.config.makeDirect, 

408 makePsfMatched=self.config.makePsfMatched) 

409 except Exception as e: 

410 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

411 continue 

412 try: 

413 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

414 for warpType in warpTypeList: 

415 exposure = warpedAndMatched.getDict()[warpType] 

416 if exposure is None: 

417 continue 

418 warp = warps[warpType] 

419 if didSetMetadata[warpType]: 

420 mimg = exposure.getMaskedImage() 

421 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

422 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

423 del mimg 

424 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

425 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

426 totGoodPix[warpType] += numGoodPix[warpType] 

427 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

428 dataId, numGoodPix[warpType], 

429 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

430 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

431 warp.info.id = exposure.info.id 

432 warp.setPhotoCalib(exposure.getPhotoCalib()) 

433 warp.setFilter(exposure.getFilter()) 

434 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

435 # PSF replaced with CoaddPsf after loop if and only if 

436 # creating direct warp. 

437 warp.setPsf(exposure.getPsf()) 

438 didSetMetadata[warpType] = True 

439 

440 # Need inputRecorder for CoaddApCorrMap for both direct and 

441 # PSF-matched. 

442 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

443 

444 except Exception as e: 

445 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

446 continue 

447 

448 for warpType in warpTypeList: 

449 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

450 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

451 

452 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

453 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType]) 

454 if warpType == "direct": 

455 warps[warpType].setPsf( 

456 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

457 self.config.coaddPsf.makeControl())) 

458 else: 

459 if not self.config.doWriteEmptyWarps: 

460 # No good pixels. Exposure still empty. 

461 warps[warpType] = None 

462 # NoWorkFound is unnecessary as the downstream tasks will 

463 # adjust the quantum accordingly. 

464 

465 result = pipeBase.Struct(exposures=warps) 

466 return result 

467 

468 def filterInputs(self, indices, inputs): 

469 """Filter task inputs by their indices. 

470 

471 Parameters 

472 ---------- 

473 indices : `list` [`int`] 

474 inputs : `dict` [`list`] 

475 A dictionary of input connections to be passed to run. 

476 

477 Returns 

478 ------- 

479 inputs : `dict` [`list`] 

480 Task inputs with their lists filtered by indices. 

481 """ 

482 for key in inputs.keys(): 

483 # Only down-select on list inputs 

484 if isinstance(inputs[key], list): 

485 inputs[key] = [inputs[key][ind] for ind in indices] 

486 return inputs 

487 

488 def _prepareCalibratedExposures(self, calExpList=[], wcsList=None, backgroundList=None, skyCorrList=None, 

489 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

490 finalizedPsfApCorrCatalog=None, **kwargs): 

491 """Calibrate and add backgrounds to input calExpList in place. 

492 

493 Parameters 

494 ---------- 

495 calExpList : `list` [`lsst.afw.image.Exposure` or 

496 `lsst.daf.butler.DeferredDatasetHandle`] 

497 Sequence of calexps to be modified in place. 

498 wcsList : `list` [`lsst.afw.geom.SkyWcs`] 

499 The WCSs of the calexps in ``calExpList``. When 

500 ``externalSkyCatalog`` is `None`, these are used to determine if 

501 the calexp should be included in the warp, namely checking that it 

502 is not `None`. If ``externalSkyCatalog`` is not `None`, this list 

503 will be dynamically updated with the external sky WCS. 

504 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional 

505 Sequence of backgrounds to be added back in if bgSubtracted=False. 

506 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional 

507 Sequence of background corrections to be subtracted if 

508 doApplySkyCorr=True. 

509 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

510 Exposure catalog with external skyWcs to be applied 

511 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

512 for the catalog id, sorted on id for fast lookup. 

513 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

514 Exposure catalog with external photoCalib to be applied 

515 if config.doApplyExternalPhotoCalib=True. Catalog uses the 

516 detector id for the catalog id, sorted on id for fast lookup. 

517 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

518 Exposure catalog with finalized psf models and aperture correction 

519 maps to be applied if config.doApplyFinalizedPsf=True. Catalog 

520 uses the detector id for the catalog id, sorted on id for fast 

521 lookup. 

522 **kwargs 

523 Additional keyword arguments. 

524 

525 Returns 

526 ------- 

527 indices : `list` [`int`] 

528 Indices of ``calExpList`` and friends that have valid 

529 photoCalib/skyWcs. 

530 """ 

531 wcsList = len(calExpList)*[None] if wcsList is None else wcsList 

532 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

533 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

534 

535 includeCalibVar = self.config.includeCalibVar 

536 

537 indices = [] 

538 for index, (calexp, wcs, background, skyCorr) in enumerate(zip(calExpList, 

539 wcsList, 

540 backgroundList, 

541 skyCorrList)): 

542 if externalSkyWcsCatalog is None and wcs is None: 

543 self.log.warning("Detector id %d for visit %d has None for skyWcs and will not be " 

544 "used in the warp", calexp.dataId["detector"], calexp.dataId["visit"]) 

545 continue 

546 

547 if isinstance(calexp, DeferredDatasetHandle): 

548 calexp = calexp.get() 

549 

550 if not self.config.bgSubtracted: 

551 calexp.maskedImage += background.getImage() 

552 

553 detectorId = calexp.info.getDetector().getId() 

554 

555 # Find the external photoCalib. 

556 if externalPhotoCalibCatalog is not None: 

557 row = externalPhotoCalibCatalog.find(detectorId) 

558 if row is None: 

559 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

560 "and will not be used in the warp.", detectorId) 

561 continue 

562 photoCalib = row.getPhotoCalib() 

563 if photoCalib is None: 

564 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

565 "and will not be used in the warp.", detectorId) 

566 continue 

567 calexp.setPhotoCalib(photoCalib) 

568 else: 

569 photoCalib = calexp.getPhotoCalib() 

570 if photoCalib is None: 

571 self.log.warning("Detector id %s has None for photoCalib in the calexp " 

572 "and will not be used in the warp.", detectorId) 

573 continue 

574 

575 # Find and apply external skyWcs. 

576 if externalSkyWcsCatalog is not None: 

577 row = externalSkyWcsCatalog.find(detectorId) 

578 if row is None: 

579 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

580 "and will not be used in the warp.", detectorId) 

581 continue 

582 skyWcs = row.getWcs() 

583 wcsList[index] = skyWcs 

584 if skyWcs is None: 

585 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

586 "and will not be used in the warp.", detectorId) 

587 continue 

588 calexp.setWcs(skyWcs) 

589 else: 

590 skyWcs = calexp.getWcs() 

591 wcsList[index] = skyWcs 

592 if skyWcs is None: 

593 self.log.warning("Detector id %s has None for skyWcs in the calexp " 

594 "and will not be used in the warp.", detectorId) 

595 continue 

596 

597 # Find and apply finalized psf and aperture correction. 

598 if finalizedPsfApCorrCatalog is not None: 

599 row = finalizedPsfApCorrCatalog.find(detectorId) 

600 if row is None: 

601 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog " 

602 "and will not be used in the warp.", detectorId) 

603 continue 

604 psf = row.getPsf() 

605 if psf is None: 

606 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog " 

607 "and will not be used in the warp.", detectorId) 

608 continue 

609 calexp.setPsf(psf) 

610 apCorrMap = row.getApCorrMap() 

611 if apCorrMap is None: 

612 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog " 

613 "and will not be used in the warp.", detectorId) 

614 continue 

615 calexp.info.setApCorrMap(apCorrMap) 

616 else: 

617 # Ensure that calexp has valid aperture correction map. 

618 if calexp.info.getApCorrMap() is None: 

619 self.log.warning("Detector id %s has None for ApCorrMap in the calexp " 

620 "and will not be used in the warp.", detectorId) 

621 continue 

622 

623 # Calibrate the image. 

624 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

625 includeScaleUncertainty=includeCalibVar) 

626 calexp.maskedImage /= photoCalib.getCalibrationMean() 

627 # TODO: The images will have a calibration of 1.0 everywhere once 

628 # RFC-545 is implemented. 

629 # exposure.setCalib(afwImage.Calib(1.0)) 

630 

631 # Apply skycorr 

632 if self.config.doApplySkyCorr: 

633 calexp.maskedImage -= skyCorr.getImage() 

634 

635 indices.append(index) 

636 calExpList[index] = calexp 

637 

638 return indices 

639 

640 @staticmethod 

641 def _prepareEmptyExposure(skyInfo): 

642 """Produce an empty exposure for a given patch. 

643 

644 Parameters 

645 ---------- 

646 skyInfo : `lsst.pipe.base.Struct` 

647 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with 

648 geometric information about the patch. 

649 

650 Returns 

651 ------- 

652 exp : `lsst.afw.image.exposure.ExposureF` 

653 An empty exposure for a given patch. 

654 """ 

655 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

656 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

657 .getPlaneBitMask("NO_DATA"), numpy.inf) 

658 return exp 

659 

660 def getWarpTypeList(self): 

661 """Return list of requested warp types per the config. 

662 """ 

663 warpTypeList = [] 

664 if self.config.makeDirect: 

665 warpTypeList.append("direct") 

666 if self.config.makePsfMatched: 

667 warpTypeList.append("psfMatched") 

668 return warpTypeList 

669 

670 

671def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

672 """Reorder inputRefs per outputSortKeyOrder. 

673 

674 Any inputRefs which are lists will be resorted per specified key e.g., 

675 'detector.' Only iterables will be reordered, and values can be of type 

676 `lsst.pipe.base.connections.DeferredDatasetRef` or 

677 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

678 

679 Returned lists of refs have the same length as the outputSortKeyOrder. 

680 If an outputSortKey not in the inputRef, then it will be padded with None. 

681 If an inputRef contains an inputSortKey that is not in the 

682 outputSortKeyOrder it will be removed. 

683 

684 Parameters 

685 ---------- 

686 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

687 Input references to be reordered and padded. 

688 outputSortKeyOrder : `iterable` 

689 Iterable of values to be compared with inputRef's dataId[dataIdKey]. 

690 dataIdKey : `str` 

691 The data ID key in the dataRefs to compare with the outputSortKeyOrder. 

692 

693 Returns 

694 ------- 

695 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

696 Quantized Connection with sorted DatasetRef values sorted if iterable. 

697 """ 

698 for connectionName, refs in inputRefs: 

699 if isinstance(refs, Iterable): 

700 if hasattr(refs[0], "dataId"): 

701 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

702 else: 

703 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

704 if inputSortKeyOrder != outputSortKeyOrder: 

705 setattr(inputRefs, connectionName, 

706 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

707 return inputRefs