Coverage for python/lsst/pipe/tasks/makeCoaddTempExp.py: 48%

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23import logging 

24 

25import lsst.pex.config as pexConfig 

26import lsst.daf.persistence as dafPersist 

27import lsst.afw.image as afwImage 

28import lsst.coadd.utils as coaddUtils 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as connectionTypes 

31import lsst.utils as utils 

32import lsst.geom 

33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

34from lsst.skymap import BaseSkyMap 

35from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

36from .selectImages import PsfWcsSelectImagesTask 

37from .warpAndPsfMatch import WarpAndPsfMatchTask 

38from .coaddHelpers import groupPatchExposures, getGroupDataRef 

39from collections.abc import Iterable 

40 

41__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

42 

43log = logging.getLogger(__name__.partition(".")[2]) 

44 

45 

46class MissingExposureError(Exception): 

47 """Raised when data cannot be retrieved for an exposure. 

48 When processing patches, sometimes one exposure is missing; this lets us 

49 distinguish bewteen that case, and other errors. 

50 """ 

51 pass 

52 

53 

54class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

55 """Config for MakeCoaddTempExpTask 

56 """ 

57 warpAndPsfMatch = pexConfig.ConfigurableField( 

58 target=WarpAndPsfMatchTask, 

59 doc="Task to warp and PSF-match calexp", 

60 ) 

61 doWrite = pexConfig.Field( 

62 doc="persist <coaddName>Coadd_<warpType>Warp", 

63 dtype=bool, 

64 default=True, 

65 ) 

66 bgSubtracted = pexConfig.Field( 

67 doc="Work with a background subtracted calexp?", 

68 dtype=bool, 

69 default=True, 

70 ) 

71 coaddPsf = pexConfig.ConfigField( 

72 doc="Configuration for CoaddPsf", 

73 dtype=CoaddPsfConfig, 

74 ) 

75 makeDirect = pexConfig.Field( 

76 doc="Make direct Warp/Coadds", 

77 dtype=bool, 

78 default=True, 

79 ) 

80 makePsfMatched = pexConfig.Field( 

81 doc="Make Psf-Matched Warp/Coadd?", 

82 dtype=bool, 

83 default=False, 

84 ) 

85 

86 doWriteEmptyWarps = pexConfig.Field( 

87 dtype=bool, 

88 default=False, 

89 doc="Write out warps even if they are empty" 

90 ) 

91 

92 hasFakes = pexConfig.Field( 

93 doc="Should be set to True if fake sources have been inserted into the input data.", 

94 dtype=bool, 

95 default=False, 

96 ) 

97 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

98 

99 def validate(self): 

100 CoaddBaseTask.ConfigClass.validate(self) 

101 if not self.makePsfMatched and not self.makeDirect: 

102 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

103 if self.doPsfMatch: 

104 # Backwards compatibility. 

105 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

106 self.makePsfMatched = True 

107 self.makeDirect = False 

108 

109 def setDefaults(self): 

110 CoaddBaseTask.ConfigClass.setDefaults(self) 

111 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

112 self.select.retarget(PsfWcsSelectImagesTask) 

113 

114## \addtogroup LSST_task_documentation 

115## \{ 

116## \page MakeCoaddTempExpTask 

117## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

118## \copybrief MakeCoaddTempExpTask 

119## \} 

120 

121 

122class MakeCoaddTempExpTask(CoaddBaseTask): 

123 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 

124 

125 @anchor MakeCoaddTempExpTask_ 

126 

127 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

128 

129 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

130 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

131 - @ref pipe_tasks_makeCoaddTempExp_IO 

132 - @ref pipe_tasks_makeCoaddTempExp_Config 

133 - @ref pipe_tasks_makeCoaddTempExp_Debug 

134 - @ref pipe_tasks_makeCoaddTempExp_Example 

135 

136 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

137 

138 Warp and optionally PSF-Match calexps onto a common projection, by 

139 performing the following operations: 

140 - Group calexps by visit/run 

141 - For each visit, generate a Warp by calling method @ref makeTempExp. 

142 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 

143 on each visit 

144 

145 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

146 

147 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

148 

149 @copydoc \_\_init\_\_ 

150 

151 This task has one special keyword argument: passing reuse=True will cause 

152 the task to skip the creation of warps that are already present in the 

153 output repositories. 

154 

155 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

156 

157 This task is primarily designed to be run from the command line. 

158 

159 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

160 to process. 

161 

162 @copydoc run 

163 

164 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

165 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

166 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

167 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

168 

169 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

170 

171 See @ref MakeCoaddTempExpConfig and parameters inherited from 

172 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 

173 

174 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

175 

176 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

177 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

178 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

179 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

180 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

181 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

182 

183 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

184 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

185 In general, for a set of input images, this config should equal the FWHM of the visit 

186 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

187 of the other config options offer a reasonable starting point. 

188 The following list presents the most common problems that arise from a misconfigured 

189 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

190 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

191 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

192 Problem: Explanation. *Solution* 

193 

194 *Troublshooting PSF-Matching Configuration:* 

195 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

196 For example:_ 

197 

198 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

199 

200 Note that increasing the kernel size also increases runtime. 

201 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

202 for matching kernel. _Provide the matcher with more data by either increasing 

203 the spatial sampling by decreasing the spatial cell size,_ 

204 

205 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

206 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

207 

208 _or increasing the padding around the Science PSF, for example:_ 

209 

210 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

211 

212 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

213 matching kernel dimensions, thus increasing the number of pixels available to fit 

214 after convolving the PSF with the matching kernel. 

215 Optionally, for debugging the effects of padding, the level of padding may be manually 

216 controlled by setting turning off the automatic padding and setting the number 

217 of pixels by which to pad the PSF: 

218 

219 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

220 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

221 

222 - Deconvolution: Matching a large PSF to a smaller PSF produces 

223 a telltale noise pattern which looks like ripples or a brain. 

224 _Increase the size of the requested model PSF. For example:_ 

225 

226 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

227 

228 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

229 _Increase the width of the Gaussian basis functions. For example:_ 

230 

231 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

232 # from default [0.7, 1.5, 3.0] 

233 

234 

235 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

236 

237 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

238 

239 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

240 

241 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

242 into the larger Data Release Processing. 

243 Set up by running: 

244 

245 setup ci_hsc 

246 cd $CI_HSC_DIR 

247 # if not built already: 

248 python $(which scons) # this will take a while 

249 

250 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

251 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

252 output respository with the desired SkyMap. The command, 

253 

254 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

255 --id patch=5,4 tract=0 filter=HSC-I \ 

256 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

257 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

258 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

259 makePsfMatched=True modelPsf.defaultFwhm=11 

260 

261 writes a direct and PSF-Matched Warp to 

262 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

263 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

264 respectively. 

265 

266 @note PSF-Matching in this particular dataset would benefit from adding 

267 `--configfile ./matchingConfig.py` to 

268 the command line arguments where `matchingConfig.py` is defined by: 

269 

270 echo " 

271 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

272 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

273 

274 

275 Add the option `--help` to see more options. 

276 """ 

277 ConfigClass = MakeCoaddTempExpConfig 

278 _DefaultName = "makeCoaddTempExp" 

279 

280 def __init__(self, reuse=False, **kwargs): 

281 CoaddBaseTask.__init__(self, **kwargs) 

282 self.reuse = reuse 

283 self.makeSubtask("warpAndPsfMatch") 

284 if self.config.hasFakes: 284 ↛ 285line 284 didn't jump to line 285, because the condition on line 284 was never true

285 self.calexpType = "fakes_calexp" 

286 else: 

287 self.calexpType = "calexp" 

288 

289 @pipeBase.timeMethod 

290 def runDataRef(self, patchRef, selectDataList=[]): 

291 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 

292 

293 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

294 plus the camera-specific filter key (e.g. "filter" or "band") 

295 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 

296 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 

297 warps are requested. 

298 

299 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

300 

301 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

302 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

303 (assembleCoadd should determine zeropoint scaling without referring to it). 

304 """ 

305 skyInfo = self.getSkyInfo(patchRef) 

306 

307 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

308 if self.config.makePsfMatched and not self.config.makeDirect: 308 ↛ 309line 308 didn't jump to line 309, because the condition on line 308 was never true

309 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

310 else: 

311 primaryWarpDataset = self.getTempExpDatasetName("direct") 

312 

313 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

314 

315 if len(calExpRefList) == 0: 315 ↛ 316line 315 didn't jump to line 316, because the condition on line 315 was never true

316 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId) 

317 return None 

318 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

319 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

320 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

321 

322 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

323 primaryWarpDataset) 

324 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

325 

326 dataRefList = [] 

327 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

328 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

329 tempExpTuple, groupData.keys) 

330 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 330 ↛ 331line 330 didn't jump to line 331, because the condition on line 330 was never true

331 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

332 dataRefList.append(tempExpRef) 

333 continue 

334 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

335 

336 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

337 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

338 # of the visit in the list. 

339 try: 

340 visitId = int(tempExpRef.dataId["visit"]) 

341 except (KeyError, ValueError): 

342 visitId = i 

343 

344 calExpList = [] 

345 ccdIdList = [] 

346 dataIdList = [] 

347 

348 for calExpInd, calExpRef in enumerate(calexpRefList): 

349 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

350 calExpRef.dataId) 

351 try: 

352 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

353 except Exception: 

354 ccdId = calExpInd 

355 try: 

356 # We augment the dataRef here with the tract, which is harmless for loading things 

357 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

358 # which do. 

359 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

360 dataId=calExpRef.dataId, 

361 tract=skyInfo.tractInfo.getId()) 

362 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

363 except Exception as e: 

364 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

365 continue 

366 

367 if self.config.doApplySkyCorr: 367 ↛ 368line 367 didn't jump to line 368, because the condition on line 367 was never true

368 self.applySkyCorr(calExpRef, calExp) 

369 

370 calExpList.append(calExp) 

371 ccdIdList.append(ccdId) 

372 dataIdList.append(calExpRef.dataId) 

373 

374 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

375 

376 if any(exps.values()): 

377 dataRefList.append(tempExpRef) 

378 else: 

379 self.log.warning("Warp %s could not be created", tempExpRef.dataId) 

380 

381 if self.config.doWrite: 381 ↛ 327line 381 didn't jump to line 327, because the condition on line 381 was never false

382 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

383 if exposure is not None: 

384 self.log.info("Persisting %s", self.getTempExpDatasetName(warpType)) 

385 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

386 

387 return dataRefList 

388 

389 @pipeBase.timeMethod 

390 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

391 """Create a Warp from inputs 

392 

393 We iterate over the multiple calexps in a single exposure to construct 

394 the warp (previously called a coaddTempExp) of that exposure to the 

395 supplied tract/patch. 

396 

397 Pixels that receive no pixels are set to NAN; this is not correct 

398 (violates LSST algorithms group policy), but will be fixed up by 

399 interpolating after the coaddition. 

400 

401 @param calexpRefList: List of data references for calexps that (may) 

402 overlap the patch of interest 

403 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

404 information about the patch 

405 @param visitId: integer identifier for visit, for the table that will 

406 produce the CoaddPsf 

407 @return a pipeBase Struct containing: 

408 - exposures: a dictionary containing the warps requested: 

409 "direct": direct warp if config.makeDirect 

410 "psfMatched": PSF-matched warp if config.makePsfMatched 

411 """ 

412 warpTypeList = self.getWarpTypeList() 

413 

414 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

415 didSetMetadata = {warpType: False for warpType in warpTypeList} 

416 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

417 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

418 for warpType in warpTypeList} 

419 

420 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

421 if dataIdList is None: 421 ↛ 422line 421 didn't jump to line 422, because the condition on line 421 was never true

422 dataIdList = ccdIdList 

423 

424 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

425 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

426 calExpInd+1, len(calExpList), dataId) 

427 

428 try: 

429 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

430 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

431 makeDirect=self.config.makeDirect, 

432 makePsfMatched=self.config.makePsfMatched) 

433 except Exception as e: 

434 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

435 continue 

436 try: 

437 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

438 for warpType in warpTypeList: 

439 exposure = warpedAndMatched.getDict()[warpType] 

440 if exposure is None: 

441 continue 

442 coaddTempExp = coaddTempExps[warpType] 

443 if didSetMetadata[warpType]: 

444 mimg = exposure.getMaskedImage() 

445 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

446 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

447 del mimg 

448 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

449 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

450 totGoodPix[warpType] += numGoodPix[warpType] 

451 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

452 dataId, numGoodPix[warpType], 

453 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

454 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

455 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

456 coaddTempExp.setFilterLabel(exposure.getFilterLabel()) 

457 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

458 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

459 coaddTempExp.setPsf(exposure.getPsf()) 

460 didSetMetadata[warpType] = True 

461 

462 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

463 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

464 

465 except Exception as e: 

466 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

467 continue 

468 

469 for warpType in warpTypeList: 

470 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

471 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

472 

473 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

474 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

475 if warpType == "direct": 

476 coaddTempExps[warpType].setPsf( 

477 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

478 self.config.coaddPsf.makeControl())) 

479 else: 

480 if not self.config.doWriteEmptyWarps: 480 ↛ 469line 480 didn't jump to line 469, because the condition on line 480 was never false

481 # No good pixels. Exposure still empty 

482 coaddTempExps[warpType] = None 

483 # NoWorkFound is unnecessary as the downstream tasks will 

484 # adjust the quantum accordingly, and it prevents gen2 

485 # MakeCoaddTempExp from continuing to loop over visits. 

486 

487 result = pipeBase.Struct(exposures=coaddTempExps) 

488 return result 

489 

490 def getCalibratedExposure(self, dataRef, bgSubtracted): 

491 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

492 

493 @param[in] dataRef a sensor-level data reference 

494 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

495 calexp's background background model and add it to the calexp. 

496 @return calibrated exposure 

497 

498 @raises MissingExposureError If data for the exposure is not available. 

499 

500 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

501 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

502 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

503 retrieved from the processed exposure. When 

504 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

505 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

506 Otherwise, the astrometric calibration is taken from the processed 

507 exposure. 

508 """ 

509 try: 

510 exposure = dataRef.get(self.calexpType, immediate=True) 

511 except dafPersist.NoResults as e: 

512 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

513 

514 if not bgSubtracted: 514 ↛ 515line 514 didn't jump to line 515, because the condition on line 514 was never true

515 background = dataRef.get("calexpBackground", immediate=True) 

516 mi = exposure.getMaskedImage() 

517 mi += background.getImage() 

518 del mi 

519 

520 if self.config.doApplyExternalPhotoCalib: 520 ↛ 521line 520 didn't jump to line 521, because the condition on line 520 was never true

521 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

522 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

523 photoCalib = dataRef.get(source) 

524 exposure.setPhotoCalib(photoCalib) 

525 else: 

526 photoCalib = exposure.getPhotoCalib() 

527 

528 if self.config.doApplyExternalSkyWcs: 528 ↛ 529line 528 didn't jump to line 529, because the condition on line 528 was never true

529 source = f"{self.config.externalSkyWcsName}_wcs" 

530 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

531 skyWcs = dataRef.get(source) 

532 exposure.setWcs(skyWcs) 

533 

534 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

535 includeScaleUncertainty=self.config.includeCalibVar) 

536 exposure.maskedImage /= photoCalib.getCalibrationMean() 

537 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

538 # exposure.setCalib(afwImage.Calib(1.0)) 

539 return exposure 

540 

541 @staticmethod 

542 def _prepareEmptyExposure(skyInfo): 

543 """Produce an empty exposure for a given patch""" 

544 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

545 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

546 .getPlaneBitMask("NO_DATA"), numpy.inf) 

547 return exp 

548 

549 def getWarpTypeList(self): 

550 """Return list of requested warp types per the config. 

551 """ 

552 warpTypeList = [] 

553 if self.config.makeDirect: 553 ↛ 555line 553 didn't jump to line 555, because the condition on line 553 was never false

554 warpTypeList.append("direct") 

555 if self.config.makePsfMatched: 555 ↛ 557line 555 didn't jump to line 557, because the condition on line 555 was never false

556 warpTypeList.append("psfMatched") 

557 return warpTypeList 

558 

559 def applySkyCorr(self, dataRef, calexp): 

560 """Apply correction to the sky background level 

561 

562 Sky corrections can be generated with the 'skyCorrection.py' 

563 executable in pipe_drivers. Because the sky model used by that 

564 code extends over the entire focal plane, this can produce 

565 better sky subtraction. 

566 

567 The calexp is updated in-place. 

568 

569 Parameters 

570 ---------- 

571 dataRef : `lsst.daf.persistence.ButlerDataRef` 

572 Data reference for calexp. 

573 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

574 Calibrated exposure. 

575 """ 

576 bg = dataRef.get("skyCorr") 

577 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

578 if isinstance(calexp, afwImage.Exposure): 

579 calexp = calexp.getMaskedImage() 

580 calexp -= bg.getImage() 

581 

582 

583class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

584 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

585 defaultTemplates={"coaddName": "deep", 

586 "skyWcsName": "jointcal", 

587 "photoCalibName": "fgcm", 

588 "calexpType": ""}): 

589 calExpList = connectionTypes.Input( 

590 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

591 name="{calexpType}calexp", 

592 storageClass="ExposureF", 

593 dimensions=("instrument", "visit", "detector"), 

594 multiple=True, 

595 deferLoad=True, 

596 ) 

597 backgroundList = connectionTypes.Input( 

598 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

599 name="calexpBackground", 

600 storageClass="Background", 

601 dimensions=("instrument", "visit", "detector"), 

602 multiple=True, 

603 ) 

604 skyCorrList = connectionTypes.Input( 

605 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

606 name="skyCorr", 

607 storageClass="Background", 

608 dimensions=("instrument", "visit", "detector"), 

609 multiple=True, 

610 ) 

611 skyMap = connectionTypes.Input( 

612 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

613 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

614 storageClass="SkyMap", 

615 dimensions=("skymap",), 

616 ) 

617 externalSkyWcsTractCatalog = connectionTypes.Input( 

618 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

619 "id for the catalog id, sorted on id for fast lookup."), 

620 name="{skyWcsName}SkyWcsCatalog", 

621 storageClass="ExposureCatalog", 

622 dimensions=("instrument", "visit", "tract"), 

623 ) 

624 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

625 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

626 "These catalogs use the detector id for the catalog id, sorted on id for " 

627 "fast lookup."), 

628 name="{skyWcsName}SkyWcsCatalog", 

629 storageClass="ExposureCatalog", 

630 dimensions=("instrument", "visit"), 

631 ) 

632 externalPhotoCalibTractCatalog = connectionTypes.Input( 

633 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

634 "detector id for the catalog id, sorted on id for fast lookup."), 

635 name="{photoCalibName}PhotoCalibCatalog", 

636 storageClass="ExposureCatalog", 

637 dimensions=("instrument", "visit", "tract"), 

638 ) 

639 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

640 doc=("Per-visit photometric calibrations computed globally (with no tract " 

641 "information). These catalogs use the detector id for the catalog id, " 

642 "sorted on id for fast lookup."), 

643 name="{photoCalibName}PhotoCalibCatalog", 

644 storageClass="ExposureCatalog", 

645 dimensions=("instrument", "visit"), 

646 ) 

647 direct = connectionTypes.Output( 

648 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

649 "calexps onto the skyMap patch geometry."), 

650 name="{coaddName}Coadd_directWarp", 

651 storageClass="ExposureF", 

652 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

653 ) 

654 psfMatched = connectionTypes.Output( 

655 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

656 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

657 name="{coaddName}Coadd_psfMatchedWarp", 

658 storageClass="ExposureF", 

659 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

660 ) 

661 # TODO DM-28769, have selectImages subtask indicate which connections they need: 

662 wcsList = connectionTypes.Input( 

663 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

664 name="{calexpType}calexp.wcs", 

665 storageClass="Wcs", 

666 dimensions=("instrument", "visit", "detector"), 

667 multiple=True, 

668 ) 

669 bboxList = connectionTypes.Input( 

670 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

671 name="{calexpType}calexp.bbox", 

672 storageClass="Box2I", 

673 dimensions=("instrument", "visit", "detector"), 

674 multiple=True, 

675 ) 

676 visitSummary = connectionTypes.Input( 

677 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask", 

678 name="{calexpType}visitSummary", 

679 storageClass="ExposureCatalog", 

680 dimensions=("instrument", "visit",), 

681 ) 

682 srcList = connectionTypes.Input( 

683 doc="Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability", 

684 name="src", 

685 storageClass="SourceCatalog", 

686 dimensions=("instrument", "visit", "detector"), 

687 multiple=True, 

688 ) 

689 

690 def __init__(self, *, config=None): 

691 super().__init__(config=config) 

692 if config.bgSubtracted: 

693 self.inputs.remove("backgroundList") 

694 if not config.doApplySkyCorr: 

695 self.inputs.remove("skyCorrList") 

696 if config.doApplyExternalSkyWcs: 

697 if config.useGlobalExternalSkyWcs: 

698 self.inputs.remove("externalSkyWcsTractCatalog") 

699 else: 

700 self.inputs.remove("externalSkyWcsGlobalCatalog") 

701 else: 

702 self.inputs.remove("externalSkyWcsTractCatalog") 

703 self.inputs.remove("externalSkyWcsGlobalCatalog") 

704 if config.doApplyExternalPhotoCalib: 

705 if config.useGlobalExternalPhotoCalib: 

706 self.inputs.remove("externalPhotoCalibTractCatalog") 

707 else: 

708 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

709 else: 

710 self.inputs.remove("externalPhotoCalibTractCatalog") 

711 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

712 if not config.makeDirect: 

713 self.outputs.remove("direct") 

714 if not config.makePsfMatched: 

715 self.outputs.remove("psfMatched") 

716 # TODO DM-28769: add connection per selectImages connections 

717 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

718 self.inputs.remove("visitSummary") 

719 self.inputs.remove("srcList") 

720 elif not config.select.doLegacyStarSelectionComputation: 

721 # Remove backwards-compatibility connections. 

722 self.inputs.remove("srcList") 

723 

724 

725class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

726 pipelineConnections=MakeWarpConnections): 

727 

728 def validate(self): 

729 super().validate() 

730 

731 

732class MakeWarpTask(MakeCoaddTempExpTask): 

733 """Warp and optionally PSF-Match calexps onto an a common projection 

734 """ 

735 ConfigClass = MakeWarpConfig 

736 _DefaultName = "makeWarp" 

737 

738 @utils.inheritDoc(pipeBase.PipelineTask) 

739 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

740 """ 

741 Notes 

742 ---- 

743 Construct warps for requested warp type for single epoch 

744 

745 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

746 calexps. This method is analogous to `runDataRef`. 

747 """ 

748 

749 # Ensure all input lists are in same detector order as the calExpList 

750 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

751 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

752 

753 # Read in all inputs. 

754 inputs = butlerQC.get(inputRefs) 

755 

756 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

757 # from the dictionary so we can pass it as kwargs later. 

758 skyMap = inputs.pop("skyMap") 

759 quantumDataId = butlerQC.quantum.dataId 

760 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

761 

762 # Construct list of input DataIds expected by `run` 

763 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

764 # Construct list of packed integer IDs expected by `run` 

765 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

766 

767 # Run the selector and filter out calexps that were not selected 

768 # primarily because they do not overlap the patch 

769 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

770 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

771 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

772 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

773 

774 # Read from disk only the selected calexps 

775 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']] 

776 

777 # Extract integer visitId requested by `run` 

778 visits = [dataId['visit'] for dataId in dataIdList] 

779 visitId = visits[0] 

780 

781 if self.config.doApplyExternalSkyWcs: 

782 if self.config.useGlobalExternalSkyWcs: 

783 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

784 else: 

785 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

786 else: 

787 externalSkyWcsCatalog = None 

788 

789 if self.config.doApplyExternalPhotoCalib: 

790 if self.config.useGlobalExternalPhotoCalib: 

791 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

792 else: 

793 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

794 else: 

795 externalPhotoCalibCatalog = None 

796 

797 completeIndices = self.prepareCalibratedExposures(**inputs, 

798 externalSkyWcsCatalog=externalSkyWcsCatalog, 

799 externalPhotoCalibCatalog=externalPhotoCalibCatalog) 

800 # Redo the input selection with inputs with complete wcs/photocalib info. 

801 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

802 

803 results = self.run(**inputs, visitId=visitId, 

804 ccdIdList=[ccdIdList[i] for i in goodIndices], 

805 dataIdList=[dataIdList[i] for i in goodIndices], 

806 skyInfo=skyInfo) 

807 if self.config.makeDirect and results.exposures["direct"] is not None: 

808 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

809 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

810 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

811 

812 def filterInputs(self, indices, inputs): 

813 """Return task inputs with their lists filtered by indices 

814 

815 Parameters 

816 ---------- 

817 indices : `list` of integers 

818 inputs : `dict` of `list` of input connections to be passed to run 

819 """ 

820 for key in inputs.keys(): 

821 # Only down-select on list inputs 

822 if isinstance(inputs[key], list): 

823 inputs[key] = [inputs[key][ind] for ind in indices] 

824 return inputs 

825 

826 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

827 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

828 **kwargs): 

829 """Calibrate and add backgrounds to input calExpList in place 

830 

831 Parameters 

832 ---------- 

833 calExpList : `list` of `lsst.afw.image.Exposure` 

834 Sequence of calexps to be modified in place 

835 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional 

836 Sequence of backgrounds to be added back in if bgSubtracted=False 

837 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional 

838 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

839 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

840 Exposure catalog with external skyWcs to be applied 

841 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

842 for the catalog id, sorted on id for fast lookup. 

843 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

844 Exposure catalog with external photoCalib to be applied 

845 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

846 id for the catalog id, sorted on id for fast lookup. 

847 

848 Returns 

849 ------- 

850 indices : `list` [`int`] 

851 Indices of calExpList and friends that have valid photoCalib/skyWcs 

852 """ 

853 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

854 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

855 

856 includeCalibVar = self.config.includeCalibVar 

857 

858 indices = [] 

859 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList, 

860 backgroundList, 

861 skyCorrList)): 

862 if not self.config.bgSubtracted: 

863 calexp.maskedImage += background.getImage() 

864 

865 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None: 

866 detectorId = calexp.getInfo().getDetector().getId() 

867 

868 # Find the external photoCalib 

869 if externalPhotoCalibCatalog is not None: