Coverage for python/lsst/pipe/tasks/makeWarp.py: 16%

263 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-30 10:45 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["MakeWarpTask", "MakeWarpConfig"] 

23 

24import logging 

25import numpy 

26 

27import lsst.pex.config as pexConfig 

28import lsst.afw.image as afwImage 

29import lsst.coadd.utils as coaddUtils 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as connectionTypes 

32import lsst.utils as utils 

33import lsst.geom 

34from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

35from lsst.skymap import BaseSkyMap 

36from lsst.utils.timer import timeMethod 

37from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

38from .warpAndPsfMatch import WarpAndPsfMatchTask 

39from collections.abc import Iterable 

40 

41log = logging.getLogger(__name__) 

42 

43 

44class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

45 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

46 defaultTemplates={"coaddName": "deep", 

47 "skyWcsName": "jointcal", 

48 "photoCalibName": "fgcm", 

49 "calexpType": ""}): 

50 calExpList = connectionTypes.Input( 

51 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

52 name="{calexpType}calexp", 

53 storageClass="ExposureF", 

54 dimensions=("instrument", "visit", "detector"), 

55 multiple=True, 

56 deferLoad=True, 

57 ) 

58 backgroundList = connectionTypes.Input( 

59 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

60 name="calexpBackground", 

61 storageClass="Background", 

62 dimensions=("instrument", "visit", "detector"), 

63 multiple=True, 

64 ) 

65 skyCorrList = connectionTypes.Input( 

66 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

67 name="skyCorr", 

68 storageClass="Background", 

69 dimensions=("instrument", "visit", "detector"), 

70 multiple=True, 

71 ) 

72 skyMap = connectionTypes.Input( 

73 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

75 storageClass="SkyMap", 

76 dimensions=("skymap",), 

77 ) 

78 externalSkyWcsTractCatalog = connectionTypes.Input( 

79 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

80 "id for the catalog id, sorted on id for fast lookup."), 

81 name="{skyWcsName}SkyWcsCatalog", 

82 storageClass="ExposureCatalog", 

83 dimensions=("instrument", "visit", "tract"), 

84 ) 

85 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

86 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

87 "These catalogs use the detector id for the catalog id, sorted on id for " 

88 "fast lookup."), 

89 name="{skyWcsName}SkyWcsCatalog", 

90 storageClass="ExposureCatalog", 

91 dimensions=("instrument", "visit"), 

92 ) 

93 externalPhotoCalibTractCatalog = connectionTypes.Input( 

94 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

95 "detector id for the catalog id, sorted on id for fast lookup."), 

96 name="{photoCalibName}PhotoCalibCatalog", 

97 storageClass="ExposureCatalog", 

98 dimensions=("instrument", "visit", "tract"), 

99 ) 

100 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

101 doc=("Per-visit photometric calibrations computed globally (with no tract " 

102 "information). These catalogs use the detector id for the catalog id, " 

103 "sorted on id for fast lookup."), 

104 name="{photoCalibName}PhotoCalibCatalog", 

105 storageClass="ExposureCatalog", 

106 dimensions=("instrument", "visit"), 

107 ) 

108 finalizedPsfApCorrCatalog = connectionTypes.Input( 

109 doc=("Per-visit finalized psf models and aperture correction maps. " 

110 "These catalogs use the detector id for the catalog id, " 

111 "sorted on id for fast lookup."), 

112 name="finalized_psf_ap_corr_catalog", 

113 storageClass="ExposureCatalog", 

114 dimensions=("instrument", "visit"), 

115 ) 

116 direct = connectionTypes.Output( 

117 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

118 "calexps onto the skyMap patch geometry."), 

119 name="{coaddName}Coadd_directWarp", 

120 storageClass="ExposureF", 

121 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

122 ) 

123 psfMatched = connectionTypes.Output( 

124 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

125 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

126 name="{coaddName}Coadd_psfMatchedWarp", 

127 storageClass="ExposureF", 

128 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

129 ) 

130 # TODO DM-28769, have selectImages subtask indicate which connections they need: 

131 wcsList = connectionTypes.Input( 

132 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

133 name="{calexpType}calexp.wcs", 

134 storageClass="Wcs", 

135 dimensions=("instrument", "visit", "detector"), 

136 multiple=True, 

137 ) 

138 bboxList = connectionTypes.Input( 

139 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

140 name="{calexpType}calexp.bbox", 

141 storageClass="Box2I", 

142 dimensions=("instrument", "visit", "detector"), 

143 multiple=True, 

144 ) 

145 visitSummary = connectionTypes.Input( 

146 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask", 

147 name="{calexpType}visitSummary", 

148 storageClass="ExposureCatalog", 

149 dimensions=("instrument", "visit",), 

150 ) 

151 

152 def __init__(self, *, config=None): 

153 super().__init__(config=config) 

154 if config.bgSubtracted: 

155 self.inputs.remove("backgroundList") 

156 if not config.doApplySkyCorr: 

157 self.inputs.remove("skyCorrList") 

158 if config.doApplyExternalSkyWcs: 

159 if config.useGlobalExternalSkyWcs: 

160 self.inputs.remove("externalSkyWcsTractCatalog") 

161 else: 

162 self.inputs.remove("externalSkyWcsGlobalCatalog") 

163 else: 

164 self.inputs.remove("externalSkyWcsTractCatalog") 

165 self.inputs.remove("externalSkyWcsGlobalCatalog") 

166 if config.doApplyExternalPhotoCalib: 

167 if config.useGlobalExternalPhotoCalib: 

168 self.inputs.remove("externalPhotoCalibTractCatalog") 

169 else: 

170 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

171 else: 

172 self.inputs.remove("externalPhotoCalibTractCatalog") 

173 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

174 if not config.doApplyFinalizedPsf: 

175 self.inputs.remove("finalizedPsfApCorrCatalog") 

176 if not config.makeDirect: 

177 self.outputs.remove("direct") 

178 if not config.makePsfMatched: 

179 self.outputs.remove("psfMatched") 

180 # TODO DM-28769: add connection per selectImages connections 

181 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

182 self.inputs.remove("visitSummary") 

183 

184 

185class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, 

186 pipelineConnections=MakeWarpConnections): 

187 """Config for MakeWarpTask.""" 

188 

189 warpAndPsfMatch = pexConfig.ConfigurableField( 

190 target=WarpAndPsfMatchTask, 

191 doc="Task to warp and PSF-match calexp", 

192 ) 

193 doWrite = pexConfig.Field( 

194 doc="persist <coaddName>Coadd_<warpType>Warp", 

195 dtype=bool, 

196 default=True, 

197 ) 

198 bgSubtracted = pexConfig.Field( 

199 doc="Work with a background subtracted calexp?", 

200 dtype=bool, 

201 default=True, 

202 ) 

203 coaddPsf = pexConfig.ConfigField( 

204 doc="Configuration for CoaddPsf", 

205 dtype=CoaddPsfConfig, 

206 ) 

207 makeDirect = pexConfig.Field( 

208 doc="Make direct Warp/Coadds", 

209 dtype=bool, 

210 default=True, 

211 ) 

212 makePsfMatched = pexConfig.Field( 

213 doc="Make Psf-Matched Warp/Coadd?", 

214 dtype=bool, 

215 default=False, 

216 ) 

217 doWriteEmptyWarps = pexConfig.Field( 

218 dtype=bool, 

219 default=False, 

220 doc="Write out warps even if they are empty" 

221 ) 

222 hasFakes = pexConfig.Field( 

223 doc="Should be set to True if fake sources have been inserted into the input data.", 

224 dtype=bool, 

225 default=False, 

226 ) 

227 doApplySkyCorr = pexConfig.Field( 

228 dtype=bool, 

229 default=False, 

230 doc="Apply sky correction?", 

231 ) 

232 doApplyFinalizedPsf = pexConfig.Field( 

233 doc="Whether to apply finalized psf models and aperture correction map.", 

234 dtype=bool, 

235 default=True, 

236 ) 

237 

238 def validate(self): 

239 CoaddBaseTask.ConfigClass.validate(self) 

240 

241 if not self.makePsfMatched and not self.makeDirect: 

242 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

243 if self.doPsfMatch: 

244 # Backwards compatibility. 

245 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

246 self.makePsfMatched = True 

247 self.makeDirect = False 

248 

249 def setDefaults(self): 

250 CoaddBaseTask.ConfigClass.setDefaults(self) 

251 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

252 

253 

254class MakeWarpTask(CoaddBaseTask): 

255 """Warp and optionally PSF-Match calexps onto an a common projection. 

256 

257 Warp and optionally PSF-Match calexps onto a common projection, by 

258 performing the following operations: 

259 - Group calexps by visit/run 

260 - For each visit, generate a Warp by calling method @ref run. 

261 `run` loops over the visit's calexps calling 

262 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit 

263 

264 Notes 

265 ----- 

266 WarpType identifies the types of convolutions applied to Warps 

267 (previously CoaddTempExps). Only two types are available: direct 

268 (for regular Warps/Coadds) and psfMatched(for Warps/Coadds with 

269 homogenized PSFs). We expect to add a third type, likelihood, for 

270 generating likelihood Coadds with Warps that have been correlated with 

271 their own PSF. 

272 

273 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

274 `~lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask` 

275 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

276 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

277 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

278 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

279 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

280 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

281 In general, for a set of input images, this config should equal the FWHM of the visit 

282 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

283 of the other config options offer a reasonable starting point. 

284 The following list presents the most common problems that arise from a misconfigured 

285 @link ip::diffim::modelPsfMatch::ModelPsfMatchTask ModelPsfMatchTask @endlink 

286 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

287 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

288 Problem: Explanation. *Solution* 

289 *Troublshooting PSF-Matching Configuration:* 

290 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

291 For example:_ 

292 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

293 Note that increasing the kernel size also increases runtime. 

294 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

295 for matching kernel. _Provide the matcher with more data by either increasing 

296 the spatial sampling by decreasing the spatial cell size,_ 

297 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

298 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

299 _or increasing the padding around the Science PSF, for example:_ 

300 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

301 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

302 matching kernel dimensions, thus increasing the number of pixels available to fit 

303 after convolving the PSF with the matching kernel. 

304 Optionally, for debugging the effects of padding, the level of padding may be manually 

305 controlled by setting turning off the automatic padding and setting the number 

306 of pixels by which to pad the PSF: 

307 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

308 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

309 - Deconvolution: Matching a large PSF to a smaller PSF produces 

310 a telltale noise pattern which looks like ripples or a brain. 

311 _Increase the size of the requested model PSF. For example:_ 

312 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

313 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

314 _Increase the width of the Gaussian basis functions. For example:_ 

315 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

316 # from default [0.7, 1.5, 3.0] 

317 """ 

318 ConfigClass = MakeWarpConfig 

319 _DefaultName = "makeWarp" 

320 

321 def __init__(self, **kwargs): 

322 CoaddBaseTask.__init__(self, **kwargs) 

323 self.makeSubtask("warpAndPsfMatch") 

324 if self.config.hasFakes: 

325 self.calexpType = "fakes_calexp" 

326 else: 

327 self.calexpType = "calexp" 

328 

329 @utils.inheritDoc(pipeBase.PipelineTask) 

330 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

331 # Obtain the list of input detectors from calExpList. Sort them by 

332 # detector order (to ensure reproducibility). Then ensure all input 

333 # lists are in the same sorted detector order. 

334 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

335 detectorOrder.sort() 

336 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

337 

338 # Read in all inputs. 

339 inputs = butlerQC.get(inputRefs) 

340 

341 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

342 # from the dictionary so we can pass it as kwargs later. 

343 skyMap = inputs.pop("skyMap") 

344 quantumDataId = butlerQC.quantum.dataId 

345 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

346 

347 # Construct list of input DataIds expected by `run` 

348 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

349 # Construct list of packed integer IDs expected by `run` 

350 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

351 

352 # Run the selector and filter out calexps that were not selected 

353 # primarily because they do not overlap the patch 

354 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

355 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

356 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

357 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

358 

359 # Read from disk only the selected calexps 

360 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']] 

361 

362 # Extract integer visitId requested by `run` 

363 visits = [dataId['visit'] for dataId in dataIdList] 

364 visitId = visits[0] 

365 

366 if self.config.doApplyExternalSkyWcs: 

367 if self.config.useGlobalExternalSkyWcs: 

368 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

369 else: 

370 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

371 else: 

372 externalSkyWcsCatalog = None 

373 

374 if self.config.doApplyExternalPhotoCalib: 

375 if self.config.useGlobalExternalPhotoCalib: 

376 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

377 else: 

378 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

379 else: 

380 externalPhotoCalibCatalog = None 

381 

382 if self.config.doApplyFinalizedPsf: 

383 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog") 

384 else: 

385 finalizedPsfApCorrCatalog = None 

386 

387 completeIndices = self.prepareCalibratedExposures(**inputs, 

388 externalSkyWcsCatalog=externalSkyWcsCatalog, 

389 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

390 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

391 # Redo the input selection with inputs with complete wcs/photocalib info. 

392 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

393 

394 results = self.run(**inputs, visitId=visitId, 

395 ccdIdList=[ccdIdList[i] for i in goodIndices], 

396 dataIdList=[dataIdList[i] for i in goodIndices], 

397 skyInfo=skyInfo) 

398 if self.config.makeDirect and results.exposures["direct"] is not None: 

399 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

400 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

401 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

402 

403 @timeMethod 

404 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

405 """Create a Warp from inputs. 

406 

407 We iterate over the multiple calexps in a single exposure to construct 

408 the warp (previously called a coaddTempExp) of that exposure to the 

409 supplied tract/patch. 

410 

411 Pixels that receive no pixels are set to NAN; this is not correct 

412 (violates LSST algorithms group policy), but will be fixed up by 

413 interpolating after the coaddition. 

414 

415 calexpRefList : `list` 

416 List of data references for calexps that (may) 

417 overlap the patch of interest. 

418 skyInfo : `lsst.pipe.base.Struct` 

419 Struct from CoaddBaseTask.getSkyInfo() with geometric 

420 information about the patch. 

421 visitId : `int` 

422 Integer identifier for visit, for the table that will 

423 produce the CoaddPsf. 

424 

425 Returns 

426 ------- 

427 result : `lsst.pipe.base.Struct` 

428 Results as a struct with attributes: 

429 

430 ``exposures`` 

431 A dictionary containing the warps requested: 

432 "direct": direct warp if ``config.makeDirect`` 

433 "psfMatched": PSF-matched warp if ``config.makePsfMatched`` (`dict`). 

434 """ 

435 warpTypeList = self.getWarpTypeList() 

436 

437 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

438 didSetMetadata = {warpType: False for warpType in warpTypeList} 

439 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

440 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

441 for warpType in warpTypeList} 

442 

443 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

444 if dataIdList is None: 

445 dataIdList = ccdIdList 

446 

447 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

448 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

449 calExpInd+1, len(calExpList), dataId) 

450 

451 try: 

452 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

453 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

454 makeDirect=self.config.makeDirect, 

455 makePsfMatched=self.config.makePsfMatched) 

456 except Exception as e: 

457 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

458 continue 

459 try: 

460 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

461 for warpType in warpTypeList: 

462 exposure = warpedAndMatched.getDict()[warpType] 

463 if exposure is None: 

464 continue 

465 warp = warps[warpType] 

466 if didSetMetadata[warpType]: 

467 mimg = exposure.getMaskedImage() 

468 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

469 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

470 del mimg 

471 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

472 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

473 totGoodPix[warpType] += numGoodPix[warpType] 

474 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

475 dataId, numGoodPix[warpType], 

476 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

477 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

478 warp.info.id = exposure.info.id 

479 warp.setPhotoCalib(exposure.getPhotoCalib()) 

480 warp.setFilter(exposure.getFilter()) 

481 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

482 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

483 warp.setPsf(exposure.getPsf()) 

484 didSetMetadata[warpType] = True 

485 

486 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

487 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

488 

489 except Exception as e: 

490 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

491 continue 

492 

493 for warpType in warpTypeList: 

494 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

495 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

496 

497 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

498 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType]) 

499 if warpType == "direct": 

500 warps[warpType].setPsf( 

501 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

502 self.config.coaddPsf.makeControl())) 

503 else: 

504 if not self.config.doWriteEmptyWarps: 

505 # No good pixels. Exposure still empty 

506 warps[warpType] = None 

507 # NoWorkFound is unnecessary as the downstream tasks will 

508 # adjust the quantum accordingly. 

509 

510 result = pipeBase.Struct(exposures=warps) 

511 return result 

512 

513 def filterInputs(self, indices, inputs): 

514 """Filter task inputs by their indices. 

515 

516 Parameters 

517 ---------- 

518 indices : `list` of `int` 

519 inputs : `dict` of `list` 

520 A dictionary of input connections to be passed to run. 

521 

522 Returns 

523 ------- 

524 inputs : `dict` of `list` 

525 Task inputs with their lists filtered by indices. 

526 """ 

527 for key in inputs.keys(): 

528 # Only down-select on list inputs 

529 if isinstance(inputs[key], list): 

530 inputs[key] = [inputs[key][ind] for ind in indices] 

531 return inputs 

532 

533 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

534 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

535 finalizedPsfApCorrCatalog=None, 

536 **kwargs): 

537 """Calibrate and add backgrounds to input calExpList in place. 

538 

539 Parameters 

540 ---------- 

541 calExpList : `list` of `lsst.afw.image.Exposure` 

542 Sequence of calexps to be modified in place. 

543 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional 

544 Sequence of backgrounds to be added back in if bgSubtracted=False. 

545 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional 

546 Sequence of background corrections to be subtracted if doApplySkyCorr=True. 

547 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

548 Exposure catalog with external skyWcs to be applied 

549 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

550 for the catalog id, sorted on id for fast lookup. 

551 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

552 Exposure catalog with external photoCalib to be applied 

553 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

554 id for the catalog id, sorted on id for fast lookup. 

555 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

556 Exposure catalog with finalized psf models and aperture correction 

557 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses 

558 the detector id for the catalog id, sorted on id for fast lookup. 

559 **kwargs 

560 Additional keyword arguments. 

561 

562 Returns 

563 ------- 

564 indices : `list` [`int`] 

565 Indices of calExpList and friends that have valid photoCalib/skyWcs. 

566 """ 

567 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

568 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

569 

570 includeCalibVar = self.config.includeCalibVar 

571 

572 indices = [] 

573 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList, 

574 backgroundList, 

575 skyCorrList)): 

576 if not self.config.bgSubtracted: 

577 calexp.maskedImage += background.getImage() 

578 

579 detectorId = calexp.getInfo().getDetector().getId() 

580 

581 # Find the external photoCalib 

582 if externalPhotoCalibCatalog is not None: 

583 row = externalPhotoCalibCatalog.find(detectorId) 

584 if row is None: 

585 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

586 "and will not be used in the warp.", detectorId) 

587 continue 

588 photoCalib = row.getPhotoCalib() 

589 if photoCalib is None: 

590 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

591 "and will not be used in the warp.", detectorId) 

592 continue 

593 calexp.setPhotoCalib(photoCalib) 

594 else: 

595 photoCalib = calexp.getPhotoCalib() 

596 if photoCalib is None: 

597 self.log.warning("Detector id %s has None for photoCalib in the calexp " 

598 "and will not be used in the warp.", detectorId) 

599 continue 

600 

601 # Find and apply external skyWcs 

602 if externalSkyWcsCatalog is not None: 

603 row = externalSkyWcsCatalog.find(detectorId) 

604 if row is None: 

605 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

606 "and will not be used in the warp.", detectorId) 

607 continue 

608 skyWcs = row.getWcs() 

609 if skyWcs is None: 

610 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

611 "and will not be used in the warp.", detectorId) 

612 continue 

613 calexp.setWcs(skyWcs) 

614 else: 

615 skyWcs = calexp.getWcs() 

616 if skyWcs is None: 

617 self.log.warning("Detector id %s has None for skyWcs in the calexp " 

618 "and will not be used in the warp.", detectorId) 

619 continue 

620 

621 # Find and apply finalized psf and aperture correction 

622 if finalizedPsfApCorrCatalog is not None: 

623 row = finalizedPsfApCorrCatalog.find(detectorId) 

624 if row is None: 

625 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog " 

626 "and will not be used in the warp.", detectorId) 

627 continue 

628 psf = row.getPsf() 

629 if psf is None: 

630 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog " 

631 "and will not be used in the warp.", detectorId) 

632 continue 

633 calexp.setPsf(psf) 

634 apCorrMap = row.getApCorrMap() 

635 if apCorrMap is None: 

636 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog " 

637 "and will not be used in the warp.", detectorId) 

638 continue 

639 calexp.info.setApCorrMap(apCorrMap) 

640 

641 # Calibrate the image 

642 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

643 includeScaleUncertainty=includeCalibVar) 

644 calexp.maskedImage /= photoCalib.getCalibrationMean() 

645 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

646 # exposure.setCalib(afwImage.Calib(1.0)) 

647 

648 # Apply skycorr 

649 if self.config.doApplySkyCorr: 

650 calexp.maskedImage -= skyCorr.getImage() 

651 

652 indices.append(index) 

653 

654 return indices 

655 

656 @staticmethod 

657 def _prepareEmptyExposure(skyInfo): 

658 """Produce an empty exposure for a given patch. 

659 

660 Parameters 

661 ---------- 

662 skyInfo : `lsst.pipe.base.Struct` 

663 Struct from CoaddBaseTask.getSkyInfo() with geometric 

664 information about the patch. 

665 

666 Returns 

667 ------- 

668 exp : `lsst.afw.image.exposure.ExposureF` 

669 An empty exposure for a given patch. 

670 """ 

671 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

672 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

673 .getPlaneBitMask("NO_DATA"), numpy.inf) 

674 return exp 

675 

676 def getWarpTypeList(self): 

677 """Return list of requested warp types per the config. 

678 """ 

679 warpTypeList = [] 

680 if self.config.makeDirect: 

681 warpTypeList.append("direct") 

682 if self.config.makePsfMatched: 

683 warpTypeList.append("psfMatched") 

684 return warpTypeList 

685 

686 

687def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

688 """Reorder inputRefs per outputSortKeyOrder. 

689 

690 Any inputRefs which are lists will be resorted per specified key e.g., 

691 'detector.' Only iterables will be reordered, and values can be of type 

692 `lsst.pipe.base.connections.DeferredDatasetRef` or 

693 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

694 

695 Returned lists of refs have the same length as the outputSortKeyOrder. 

696 If an outputSortKey not in the inputRef, then it will be padded with None. 

697 If an inputRef contains an inputSortKey that is not in the 

698 outputSortKeyOrder it will be removed. 

699 

700 Parameters 

701 ---------- 

702 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

703 Input references to be reordered and padded. 

704 outputSortKeyOrder : `iterable` 

705 Iterable of values to be compared with inputRef's dataId[dataIdKey]. 

706 dataIdKey : `str` 

707 The data ID key in the dataRefs to compare with the outputSortKeyOrder. 

708 

709 Returns 

710 ------- 

711 inputRefs: `lsst.pipe.base.connections.QuantizedConnection` 

712 Quantized Connection with sorted DatasetRef values sorted if iterable. 

713 """ 

714 for connectionName, refs in inputRefs: 

715 if isinstance(refs, Iterable): 

716 if hasattr(refs[0], "dataId"): 

717 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

718 else: 

719 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

720 if inputSortKeyOrder != outputSortKeyOrder: 

721 setattr(inputRefs, connectionName, 

722 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

723 return inputRefs