Coverage for python/lsst/pipe/tasks/makeCoaddTempExp.py: 46%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

338 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23import logging 

24 

25import lsst.pex.config as pexConfig 

26import lsst.daf.persistence as dafPersist 

27import lsst.afw.image as afwImage 

28import lsst.coadd.utils as coaddUtils 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as connectionTypes 

31import lsst.utils as utils 

32import lsst.geom 

33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

34from lsst.skymap import BaseSkyMap 

35from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

36from .warpAndPsfMatch import WarpAndPsfMatchTask 

37from .coaddHelpers import groupPatchExposures, getGroupDataRef 

38from collections.abc import Iterable 

39 

40__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

41 

42log = logging.getLogger(__name__.partition(".")[2]) 

43 

44 

45class MissingExposureError(Exception): 

46 """Raised when data cannot be retrieved for an exposure. 

47 When processing patches, sometimes one exposure is missing; this lets us 

48 distinguish bewteen that case, and other errors. 

49 """ 

50 pass 

51 

52 

53class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

54 """Config for MakeCoaddTempExpTask 

55 """ 

56 warpAndPsfMatch = pexConfig.ConfigurableField( 

57 target=WarpAndPsfMatchTask, 

58 doc="Task to warp and PSF-match calexp", 

59 ) 

60 doWrite = pexConfig.Field( 

61 doc="persist <coaddName>Coadd_<warpType>Warp", 

62 dtype=bool, 

63 default=True, 

64 ) 

65 bgSubtracted = pexConfig.Field( 

66 doc="Work with a background subtracted calexp?", 

67 dtype=bool, 

68 default=True, 

69 ) 

70 coaddPsf = pexConfig.ConfigField( 

71 doc="Configuration for CoaddPsf", 

72 dtype=CoaddPsfConfig, 

73 ) 

74 makeDirect = pexConfig.Field( 

75 doc="Make direct Warp/Coadds", 

76 dtype=bool, 

77 default=True, 

78 ) 

79 makePsfMatched = pexConfig.Field( 

80 doc="Make Psf-Matched Warp/Coadd?", 

81 dtype=bool, 

82 default=False, 

83 ) 

84 

85 doWriteEmptyWarps = pexConfig.Field( 

86 dtype=bool, 

87 default=False, 

88 doc="Write out warps even if they are empty" 

89 ) 

90 

91 hasFakes = pexConfig.Field( 

92 doc="Should be set to True if fake sources have been inserted into the input data.", 

93 dtype=bool, 

94 default=False, 

95 ) 

96 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

97 

98 def validate(self): 

99 CoaddBaseTask.ConfigClass.validate(self) 

100 if not self.makePsfMatched and not self.makeDirect: 

101 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

102 if self.doPsfMatch: 

103 # Backwards compatibility. 

104 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

105 self.makePsfMatched = True 

106 self.makeDirect = False 

107 

108 def setDefaults(self): 

109 CoaddBaseTask.ConfigClass.setDefaults(self) 

110 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

111 

112## \addtogroup LSST_task_documentation 

113## \{ 

114## \page MakeCoaddTempExpTask 

115## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

116## \copybrief MakeCoaddTempExpTask 

117## \} 

118 

119 

120class MakeCoaddTempExpTask(CoaddBaseTask): 

121 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 

122 

123 @anchor MakeCoaddTempExpTask_ 

124 

125 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

126 

127 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

128 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

129 - @ref pipe_tasks_makeCoaddTempExp_IO 

130 - @ref pipe_tasks_makeCoaddTempExp_Config 

131 - @ref pipe_tasks_makeCoaddTempExp_Debug 

132 - @ref pipe_tasks_makeCoaddTempExp_Example 

133 

134 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

135 

136 Warp and optionally PSF-Match calexps onto a common projection, by 

137 performing the following operations: 

138 - Group calexps by visit/run 

139 - For each visit, generate a Warp by calling method @ref makeTempExp. 

140 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 

141 on each visit 

142 

143 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

144 

145 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

146 

147 @copydoc \_\_init\_\_ 

148 

149 This task has one special keyword argument: passing reuse=True will cause 

150 the task to skip the creation of warps that are already present in the 

151 output repositories. 

152 

153 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

154 

155 This task is primarily designed to be run from the command line. 

156 

157 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

158 to process. 

159 

160 @copydoc run 

161 

162 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

163 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

164 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

165 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

166 

167 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

168 

169 See @ref MakeCoaddTempExpConfig and parameters inherited from 

170 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 

171 

172 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

173 

174 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

175 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

176 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

177 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

178 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

179 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

180 

181 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

182 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

183 In general, for a set of input images, this config should equal the FWHM of the visit 

184 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

185 of the other config options offer a reasonable starting point. 

186 The following list presents the most common problems that arise from a misconfigured 

187 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

188 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

189 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

190 Problem: Explanation. *Solution* 

191 

192 *Troublshooting PSF-Matching Configuration:* 

193 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

194 For example:_ 

195 

196 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

197 

198 Note that increasing the kernel size also increases runtime. 

199 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

200 for matching kernel. _Provide the matcher with more data by either increasing 

201 the spatial sampling by decreasing the spatial cell size,_ 

202 

203 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

204 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

205 

206 _or increasing the padding around the Science PSF, for example:_ 

207 

208 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

209 

210 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

211 matching kernel dimensions, thus increasing the number of pixels available to fit 

212 after convolving the PSF with the matching kernel. 

213 Optionally, for debugging the effects of padding, the level of padding may be manually 

214 controlled by setting turning off the automatic padding and setting the number 

215 of pixels by which to pad the PSF: 

216 

217 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

218 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

219 

220 - Deconvolution: Matching a large PSF to a smaller PSF produces 

221 a telltale noise pattern which looks like ripples or a brain. 

222 _Increase the size of the requested model PSF. For example:_ 

223 

224 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

225 

226 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

227 _Increase the width of the Gaussian basis functions. For example:_ 

228 

229 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

230 # from default [0.7, 1.5, 3.0] 

231 

232 

233 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

234 

235 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

236 

237 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

238 

239 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

240 into the larger Data Release Processing. 

241 Set up by running: 

242 

243 setup ci_hsc 

244 cd $CI_HSC_DIR 

245 # if not built already: 

246 python $(which scons) # this will take a while 

247 

248 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

249 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

250 output respository with the desired SkyMap. The command, 

251 

252 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

253 --id patch=5,4 tract=0 filter=HSC-I \ 

254 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

255 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

256 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

257 makePsfMatched=True modelPsf.defaultFwhm=11 

258 

259 writes a direct and PSF-Matched Warp to 

260 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

261 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

262 respectively. 

263 

264 @note PSF-Matching in this particular dataset would benefit from adding 

265 `--configfile ./matchingConfig.py` to 

266 the command line arguments where `matchingConfig.py` is defined by: 

267 

268 echo " 

269 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

270 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

271 

272 

273 Add the option `--help` to see more options. 

274 """ 

275 ConfigClass = MakeCoaddTempExpConfig 

276 _DefaultName = "makeCoaddTempExp" 

277 

278 def __init__(self, reuse=False, **kwargs): 

279 CoaddBaseTask.__init__(self, **kwargs) 

280 self.reuse = reuse 

281 self.makeSubtask("warpAndPsfMatch") 

282 if self.config.hasFakes: 282 ↛ 283line 282 didn't jump to line 283, because the condition on line 282 was never true

283 self.calexpType = "fakes_calexp" 

284 else: 

285 self.calexpType = "calexp" 

286 

287 @pipeBase.timeMethod 

288 def runDataRef(self, patchRef, selectDataList=[]): 

289 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 

290 

291 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

292 plus the camera-specific filter key (e.g. "filter" or "band") 

293 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 

294 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 

295 warps are requested. 

296 

297 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

298 

299 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

300 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

301 (assembleCoadd should determine zeropoint scaling without referring to it). 

302 """ 

303 skyInfo = self.getSkyInfo(patchRef) 

304 

305 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

306 if self.config.makePsfMatched and not self.config.makeDirect: 306 ↛ 307line 306 didn't jump to line 307, because the condition on line 306 was never true

307 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

308 else: 

309 primaryWarpDataset = self.getTempExpDatasetName("direct") 

310 

311 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

312 

313 if len(calExpRefList) == 0: 313 ↛ 314line 313 didn't jump to line 314, because the condition on line 313 was never true

314 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId) 

315 return None 

316 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

317 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

318 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

319 

320 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

321 primaryWarpDataset) 

322 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

323 

324 dataRefList = [] 

325 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

326 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

327 tempExpTuple, groupData.keys) 

328 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 328 ↛ 329line 328 didn't jump to line 329, because the condition on line 328 was never true

329 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

330 dataRefList.append(tempExpRef) 

331 continue 

332 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

333 

334 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

335 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

336 # of the visit in the list. 

337 try: 

338 visitId = int(tempExpRef.dataId["visit"]) 

339 except (KeyError, ValueError): 

340 visitId = i 

341 

342 calExpList = [] 

343 ccdIdList = [] 

344 dataIdList = [] 

345 

346 for calExpInd, calExpRef in enumerate(calexpRefList): 

347 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

348 calExpRef.dataId) 

349 try: 

350 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

351 except Exception: 

352 ccdId = calExpInd 

353 try: 

354 # We augment the dataRef here with the tract, which is harmless for loading things 

355 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

356 # which do. 

357 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

358 dataId=calExpRef.dataId, 

359 tract=skyInfo.tractInfo.getId()) 

360 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

361 except Exception as e: 

362 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

363 continue 

364 

365 if self.config.doApplySkyCorr: 365 ↛ 366line 365 didn't jump to line 366, because the condition on line 365 was never true

366 self.applySkyCorr(calExpRef, calExp) 

367 

368 calExpList.append(calExp) 

369 ccdIdList.append(ccdId) 

370 dataIdList.append(calExpRef.dataId) 

371 

372 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

373 

374 if any(exps.values()): 

375 dataRefList.append(tempExpRef) 

376 else: 

377 self.log.warning("Warp %s could not be created", tempExpRef.dataId) 

378 

379 if self.config.doWrite: 379 ↛ 325line 379 didn't jump to line 325, because the condition on line 379 was never false

380 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

381 if exposure is not None: 

382 self.log.info("Persisting %s", self.getTempExpDatasetName(warpType)) 

383 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

384 

385 return dataRefList 

386 

387 @pipeBase.timeMethod 

388 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

389 """Create a Warp from inputs 

390 

391 We iterate over the multiple calexps in a single exposure to construct 

392 the warp (previously called a coaddTempExp) of that exposure to the 

393 supplied tract/patch. 

394 

395 Pixels that receive no pixels are set to NAN; this is not correct 

396 (violates LSST algorithms group policy), but will be fixed up by 

397 interpolating after the coaddition. 

398 

399 @param calexpRefList: List of data references for calexps that (may) 

400 overlap the patch of interest 

401 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

402 information about the patch 

403 @param visitId: integer identifier for visit, for the table that will 

404 produce the CoaddPsf 

405 @return a pipeBase Struct containing: 

406 - exposures: a dictionary containing the warps requested: 

407 "direct": direct warp if config.makeDirect 

408 "psfMatched": PSF-matched warp if config.makePsfMatched 

409 """ 

410 warpTypeList = self.getWarpTypeList() 

411 

412 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

413 didSetMetadata = {warpType: False for warpType in warpTypeList} 

414 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

415 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

416 for warpType in warpTypeList} 

417 

418 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

419 if dataIdList is None: 419 ↛ 420line 419 didn't jump to line 420, because the condition on line 419 was never true

420 dataIdList = ccdIdList 

421 

422 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

423 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

424 calExpInd+1, len(calExpList), dataId) 

425 

426 try: 

427 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

428 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

429 makeDirect=self.config.makeDirect, 

430 makePsfMatched=self.config.makePsfMatched) 

431 except Exception as e: 

432 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

433 continue 

434 try: 

435 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

436 for warpType in warpTypeList: 

437 exposure = warpedAndMatched.getDict()[warpType] 

438 if exposure is None: 

439 continue 

440 coaddTempExp = coaddTempExps[warpType] 

441 if didSetMetadata[warpType]: 

442 mimg = exposure.getMaskedImage() 

443 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

444 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

445 del mimg 

446 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

447 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

448 totGoodPix[warpType] += numGoodPix[warpType] 

449 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

450 dataId, numGoodPix[warpType], 

451 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

452 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

453 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

454 coaddTempExp.setFilterLabel(exposure.getFilterLabel()) 

455 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

456 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

457 coaddTempExp.setPsf(exposure.getPsf()) 

458 didSetMetadata[warpType] = True 

459 

460 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

461 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

462 

463 except Exception as e: 

464 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

465 continue 

466 

467 for warpType in warpTypeList: 

468 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

469 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

470 

471 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

472 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

473 if warpType == "direct": 

474 coaddTempExps[warpType].setPsf( 

475 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

476 self.config.coaddPsf.makeControl())) 

477 else: 

478 if not self.config.doWriteEmptyWarps: 478 ↛ 467line 478 didn't jump to line 467, because the condition on line 478 was never false

479 # No good pixels. Exposure still empty 

480 coaddTempExps[warpType] = None 

481 # NoWorkFound is unnecessary as the downstream tasks will 

482 # adjust the quantum accordingly, and it prevents gen2 

483 # MakeCoaddTempExp from continuing to loop over visits. 

484 

485 result = pipeBase.Struct(exposures=coaddTempExps) 

486 return result 

487 

488 def getCalibratedExposure(self, dataRef, bgSubtracted): 

489 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

490 

491 @param[in] dataRef a sensor-level data reference 

492 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

493 calexp's background background model and add it to the calexp. 

494 @return calibrated exposure 

495 

496 @raises MissingExposureError If data for the exposure is not available. 

497 

498 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

499 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

500 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

501 retrieved from the processed exposure. When 

502 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

503 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

504 Otherwise, the astrometric calibration is taken from the processed 

505 exposure. 

506 """ 

507 try: 

508 exposure = dataRef.get(self.calexpType, immediate=True) 

509 except dafPersist.NoResults as e: 

510 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

511 

512 if not bgSubtracted: 512 ↛ 513line 512 didn't jump to line 513, because the condition on line 512 was never true

513 background = dataRef.get("calexpBackground", immediate=True) 

514 mi = exposure.getMaskedImage() 

515 mi += background.getImage() 

516 del mi 

517 

518 if self.config.doApplyExternalPhotoCalib: 518 ↛ 519line 518 didn't jump to line 519, because the condition on line 518 was never true

519 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

520 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

521 photoCalib = dataRef.get(source) 

522 exposure.setPhotoCalib(photoCalib) 

523 else: 

524 photoCalib = exposure.getPhotoCalib() 

525 

526 if self.config.doApplyExternalSkyWcs: 526 ↛ 527line 526 didn't jump to line 527, because the condition on line 526 was never true

527 source = f"{self.config.externalSkyWcsName}_wcs" 

528 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

529 skyWcs = dataRef.get(source) 

530 exposure.setWcs(skyWcs) 

531 

532 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

533 includeScaleUncertainty=self.config.includeCalibVar) 

534 exposure.maskedImage /= photoCalib.getCalibrationMean() 

535 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

536 # exposure.setCalib(afwImage.Calib(1.0)) 

537 return exposure 

538 

539 @staticmethod 

540 def _prepareEmptyExposure(skyInfo): 

541 """Produce an empty exposure for a given patch""" 

542 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

543 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

544 .getPlaneBitMask("NO_DATA"), numpy.inf) 

545 return exp 

546 

547 def getWarpTypeList(self): 

548 """Return list of requested warp types per the config. 

549 """ 

550 warpTypeList = [] 

551 if self.config.makeDirect: 551 ↛ 553line 551 didn't jump to line 553, because the condition on line 551 was never false

552 warpTypeList.append("direct") 

553 if self.config.makePsfMatched: 553 ↛ 555line 553 didn't jump to line 555, because the condition on line 553 was never false

554 warpTypeList.append("psfMatched") 

555 return warpTypeList 

556 

557 def applySkyCorr(self, dataRef, calexp): 

558 """Apply correction to the sky background level 

559 

560 Sky corrections can be generated with the 'skyCorrection.py' 

561 executable in pipe_drivers. Because the sky model used by that 

562 code extends over the entire focal plane, this can produce 

563 better sky subtraction. 

564 

565 The calexp is updated in-place. 

566 

567 Parameters 

568 ---------- 

569 dataRef : `lsst.daf.persistence.ButlerDataRef` 

570 Data reference for calexp. 

571 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

572 Calibrated exposure. 

573 """ 

574 bg = dataRef.get("skyCorr") 

575 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

576 if isinstance(calexp, afwImage.Exposure): 

577 calexp = calexp.getMaskedImage() 

578 calexp -= bg.getImage() 

579 

580 

581class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

582 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

583 defaultTemplates={"coaddName": "deep", 

584 "skyWcsName": "jointcal", 

585 "photoCalibName": "fgcm", 

586 "calexpType": ""}): 

587 calExpList = connectionTypes.Input( 

588 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

589 name="{calexpType}calexp", 

590 storageClass="ExposureF", 

591 dimensions=("instrument", "visit", "detector"), 

592 multiple=True, 

593 deferLoad=True, 

594 ) 

595 backgroundList = connectionTypes.Input( 

596 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

597 name="calexpBackground", 

598 storageClass="Background", 

599 dimensions=("instrument", "visit", "detector"), 

600 multiple=True, 

601 ) 

602 skyCorrList = connectionTypes.Input( 

603 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

604 name="skyCorr", 

605 storageClass="Background", 

606 dimensions=("instrument", "visit", "detector"), 

607 multiple=True, 

608 ) 

609 skyMap = connectionTypes.Input( 

610 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

611 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

612 storageClass="SkyMap", 

613 dimensions=("skymap",), 

614 ) 

615 externalSkyWcsTractCatalog = connectionTypes.Input( 

616 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

617 "id for the catalog id, sorted on id for fast lookup."), 

618 name="{skyWcsName}SkyWcsCatalog", 

619 storageClass="ExposureCatalog", 

620 dimensions=("instrument", "visit", "tract"), 

621 ) 

622 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

623 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

624 "These catalogs use the detector id for the catalog id, sorted on id for " 

625 "fast lookup."), 

626 name="{skyWcsName}SkyWcsCatalog", 

627 storageClass="ExposureCatalog", 

628 dimensions=("instrument", "visit"), 

629 ) 

630 externalPhotoCalibTractCatalog = connectionTypes.Input( 

631 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

632 "detector id for the catalog id, sorted on id for fast lookup."), 

633 name="{photoCalibName}PhotoCalibCatalog", 

634 storageClass="ExposureCatalog", 

635 dimensions=("instrument", "visit", "tract"), 

636 ) 

637 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

638 doc=("Per-visit photometric calibrations computed globally (with no tract " 

639 "information). These catalogs use the detector id for the catalog id, " 

640 "sorted on id for fast lookup."), 

641 name="{photoCalibName}PhotoCalibCatalog", 

642 storageClass="ExposureCatalog", 

643 dimensions=("instrument", "visit"), 

644 ) 

645 direct = connectionTypes.Output( 

646 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

647 "calexps onto the skyMap patch geometry."), 

648 name="{coaddName}Coadd_directWarp", 

649 storageClass="ExposureF", 

650 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

651 ) 

652 psfMatched = connectionTypes.Output( 

653 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

654 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

655 name="{coaddName}Coadd_psfMatchedWarp", 

656 storageClass="ExposureF", 

657 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

658 ) 

659 # TODO DM-28769, have selectImages subtask indicate which connections they need: 

660 wcsList = connectionTypes.Input( 

661 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

662 name="{calexpType}calexp.wcs", 

663 storageClass="Wcs", 

664 dimensions=("instrument", "visit", "detector"), 

665 multiple=True, 

666 ) 

667 bboxList = connectionTypes.Input( 

668 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

669 name="{calexpType}calexp.bbox", 

670 storageClass="Box2I", 

671 dimensions=("instrument", "visit", "detector"), 

672 multiple=True, 

673 ) 

674 visitSummary = connectionTypes.Input( 

675 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask", 

676 name="{calexpType}visitSummary", 

677 storageClass="ExposureCatalog", 

678 dimensions=("instrument", "visit",), 

679 ) 

680 

681 def __init__(self, *, config=None): 

682 super().__init__(config=config) 

683 if config.bgSubtracted: 

684 self.inputs.remove("backgroundList") 

685 if not config.doApplySkyCorr: 

686 self.inputs.remove("skyCorrList") 

687 if config.doApplyExternalSkyWcs: 

688 if config.useGlobalExternalSkyWcs: 

689 self.inputs.remove("externalSkyWcsTractCatalog") 

690 else: 

691 self.inputs.remove("externalSkyWcsGlobalCatalog") 

692 else: 

693 self.inputs.remove("externalSkyWcsTractCatalog") 

694 self.inputs.remove("externalSkyWcsGlobalCatalog") 

695 if config.doApplyExternalPhotoCalib: 

696 if config.useGlobalExternalPhotoCalib: 

697 self.inputs.remove("externalPhotoCalibTractCatalog") 

698 else: 

699 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

700 else: 

701 self.inputs.remove("externalPhotoCalibTractCatalog") 

702 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

703 if not config.makeDirect: 

704 self.outputs.remove("direct") 

705 if not config.makePsfMatched: 

706 self.outputs.remove("psfMatched") 

707 # TODO DM-28769: add connection per selectImages connections 

708 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

709 self.inputs.remove("visitSummary") 

710 

711 

712class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

713 pipelineConnections=MakeWarpConnections): 

714 

715 def validate(self): 

716 super().validate() 

717 

718 

719class MakeWarpTask(MakeCoaddTempExpTask): 

720 """Warp and optionally PSF-Match calexps onto an a common projection 

721 """ 

722 ConfigClass = MakeWarpConfig 

723 _DefaultName = "makeWarp" 

724 

725 @utils.inheritDoc(pipeBase.PipelineTask) 

726 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

727 """ 

728 Notes 

729 ---- 

730 Construct warps for requested warp type for single epoch 

731 

732 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

733 calexps. This method is analogous to `runDataRef`. 

734 """ 

735 

736 # Ensure all input lists are in same detector order as the calExpList 

737 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

738 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

739 

740 # Read in all inputs. 

741 inputs = butlerQC.get(inputRefs) 

742 

743 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

744 # from the dictionary so we can pass it as kwargs later. 

745 skyMap = inputs.pop("skyMap") 

746 quantumDataId = butlerQC.quantum.dataId 

747 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

748 

749 # Construct list of input DataIds expected by `run` 

750 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

751 # Construct list of packed integer IDs expected by `run` 

752 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

753 

754 # Run the selector and filter out calexps that were not selected 

755 # primarily because they do not overlap the patch 

756 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

757 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

758 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

759 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

760 

761 # Read from disk only the selected calexps 

762 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']] 

763 

764 # Extract integer visitId requested by `run` 

765 visits = [dataId['visit'] for dataId in dataIdList] 

766 visitId = visits[0] 

767 

768 if self.config.doApplyExternalSkyWcs: 

769 if self.config.useGlobalExternalSkyWcs: 

770 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

771 else: 

772 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

773 else: 

774 externalSkyWcsCatalog = None 

775 

776 if self.config.doApplyExternalPhotoCalib: 

777 if self.config.useGlobalExternalPhotoCalib: 

778 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

779 else: 

780 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

781 else: 

782 externalPhotoCalibCatalog = None 

783 

784 completeIndices = self.prepareCalibratedExposures(**inputs, 

785 externalSkyWcsCatalog=externalSkyWcsCatalog, 

786 externalPhotoCalibCatalog=externalPhotoCalibCatalog) 

787 # Redo the input selection with inputs with complete wcs/photocalib info. 

788 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

789 

790 results = self.run(**inputs, visitId=visitId, 

791 ccdIdList=[ccdIdList[i] for i in goodIndices], 

792 dataIdList=[dataIdList[i] for i in goodIndices], 

793 skyInfo=skyInfo) 

794 if self.config.makeDirect and results.exposures["direct"] is not None: 

795 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

796 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

797 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

798 

799 def filterInputs(self, indices, inputs): 

800 """Return task inputs with their lists filtered by indices 

801 

802 Parameters 

803 ---------- 

804 indices : `list` of integers 

805 inputs : `dict` of `list` of input connections to be passed to run 

806 """ 

807 for key in inputs.keys(): 

808 # Only down-select on list inputs 

809 if isinstance(inputs[key], list): 

810 inputs[key] = [inputs[key][ind] for ind in indices] 

811 return inputs 

812 

813 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

814 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

815 **kwargs): 

816 """Calibrate and add backgrounds to input calExpList in place 

817 

818 Parameters 

819 ---------- 

820 calExpList : `list` of `lsst.afw.image.Exposure` 

821 Sequence of calexps to be modified in place 

822 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional 

823 Sequence of backgrounds to be added back in if bgSubtracted=False 

824 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional 

825 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

826 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

827 Exposure catalog with external skyWcs to be applied 

828 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

829 for the catalog id, sorted on id for fast lookup. 

830 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

831 Exposure catalog with external photoCalib to be applied 

832 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

833 id for the catalog id, sorted on id for fast lookup. 

834 

835 Returns 

836 ------- 

837 indices : `list` [`int`] 

838 Indices of calExpList and friends that have valid photoCalib/skyWcs 

839 """ 

840 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

841 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

842 

843 includeCalibVar = self.config.includeCalibVar 

844 

845 indices = [] 

846 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList, 

847 backgroundList, 

848 skyCorrList)): 

849 mi = calexp.maskedImage 

850 if not self.config.bgSubtracted: 

851 mi += background.getImage() 

852 

853 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None: 

854 detectorId = calexp.getInfo().getDetector().getId() 

855 

856 # Find the external photoCalib 

857 if externalPhotoCalibCatalog is not None: 

858 row = externalPhotoCalibCatalog.find(detectorId) 

859 if row is None: 

860 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

861 "and will not be used in the warp.", detectorId) 

862 continue 

863 photoCalib = row.getPhotoCalib() 

864 if photoCalib is None: 

865 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

866 "and will not be used in the warp.", detectorId) 

867 continue 

868 calexp.setPhotoCalib(photoCalib) 

869 else: 

870 photoCalib = calexp.getPhotoCalib() 

871 if photoCalib is None: 

872 self.log.warning("Detector id %s has None for photoCalib in the calexp " 

873 "and will not be used in the warp.", detectorId) 

874 continue 

875 

876 # Find and apply external skyWcs 

877 if externalSkyWcsCatalog is not None: 

878 row = externalSkyWcsCatalog.find(detectorId) 

879 if row is None: 

880 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

881 "and will not be used in the warp.", detectorId) 

882 continue 

883 skyWcs = row.getWcs() 

884 if skyWcs is None: 

885 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

886 "and will not be used in the warp.", detectorId) 

887 continue 

888 calexp.setWcs(skyWcs) 

889 else: 

890 skyWcs = calexp.getWcs() 

891 if skyWcs is None: 

892 self.log.warning("Detector id %s has None for skyWcs in the calexp " 

893 "and will not be used in the warp.", detectorId) 

894 continue 

895 

896 # Calibrate the image 

897 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

898 includeScaleUncertainty=includeCalibVar) 

899 calexp.maskedImage /= photoCalib.getCalibrationMean() 

900 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

901 # exposure.setCalib(afwImage.Calib(1.0)) 

902 

903 # Apply skycorr 

904 if self.config.doApplySkyCorr: 

905 mi -= skyCorr.getImage() 

906 

907 indices.append(index) 

908 

909 return indices 

910 

911 

912def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

913 """Reorder inputRefs per outputSortKeyOrder 

914 

915 Any inputRefs which are lists will be resorted per specified key e.g., 

916 'detector.' Only iterables will be reordered, and values can be of type 

917 `lsst.pipe.base.connections.DeferredDatasetRef` or 

918 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

919 Returned lists of refs have the same length as the outputSortKeyOrder. 

920 If an outputSortKey not in the inputRef, then it will be padded with None. 

921 If an inputRef contains an inputSortKey that is not in the 

922 outputSortKeyOrder it will be removed. 

923 

924 Parameters 

925 ---------- 

926 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

927 Input references to be reordered and padded. 

928 outputSortKeyOrder : iterable 

929 Iterable of values to be compared with inputRef's dataId[dataIdKey] 

930 dataIdKey : `str` 

931 dataIdKey in the dataRefs to compare with the outputSortKeyOrder. 

932 

933 Returns: 

934 -------- 

935 inputRefs: `lsst.pipe.base.connections.QuantizedConnection` 

936 Quantized Connection with sorted DatasetRef values sorted if iterable. 

937 """ 

938 for connectionName, refs in inputRefs: 

939 if isinstance(refs, Iterable): 

940 if hasattr(refs[0], "dataId"): 

941 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

942 else: 

943 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

944 if inputSortKeyOrder != outputSortKeyOrder: 

945 setattr(inputRefs, connectionName, 

946 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

947 return inputRefs