Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23 

24import lsst.pex.config as pexConfig 

25import lsst.daf.persistence as dafPersist 

26import lsst.afw.image as afwImage 

27import lsst.coadd.utils as coaddUtils 

28import lsst.pipe.base as pipeBase 

29import lsst.pipe.base.connectionTypes as cT 

30import lsst.log as log 

31import lsst.utils as utils 

32from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

33from .coaddBase import CoaddBaseTask, makeSkyInfo 

34from .warpAndPsfMatch import WarpAndPsfMatchTask 

35from .coaddHelpers import groupPatchExposures, getGroupDataRef 

36 

37__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

38 

39 

40class MissingExposureError(Exception): 

41 """Raised when data cannot be retrieved for an exposure. 

42 When processing patches, sometimes one exposure is missing; this lets us 

43 distinguish bewteen that case, and other errors. 

44 """ 

45 pass 

46 

47 

48class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

49 """Config for MakeCoaddTempExpTask 

50 """ 

51 warpAndPsfMatch = pexConfig.ConfigurableField( 

52 target=WarpAndPsfMatchTask, 

53 doc="Task to warp and PSF-match calexp", 

54 ) 

55 doWrite = pexConfig.Field( 

56 doc="persist <coaddName>Coadd_<warpType>Warp", 

57 dtype=bool, 

58 default=True, 

59 ) 

60 bgSubtracted = pexConfig.Field( 

61 doc="Work with a background subtracted calexp?", 

62 dtype=bool, 

63 default=True, 

64 ) 

65 coaddPsf = pexConfig.ConfigField( 

66 doc="Configuration for CoaddPsf", 

67 dtype=CoaddPsfConfig, 

68 ) 

69 makeDirect = pexConfig.Field( 

70 doc="Make direct Warp/Coadds", 

71 dtype=bool, 

72 default=True, 

73 ) 

74 makePsfMatched = pexConfig.Field( 

75 doc="Make Psf-Matched Warp/Coadd?", 

76 dtype=bool, 

77 default=False, 

78 ) 

79 

80 doWriteEmptyWarps = pexConfig.Field( 

81 dtype=bool, 

82 default=False, 

83 doc="Write out warps even if they are empty" 

84 ) 

85 

86 hasFakes = pexConfig.Field( 

87 doc="Should be set to True if fake sources have been inserted into the input data.", 

88 dtype=bool, 

89 default=False, 

90 ) 

91 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

92 

93 def validate(self): 

94 CoaddBaseTask.ConfigClass.validate(self) 

95 if not self.makePsfMatched and not self.makeDirect: 

96 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

97 if self.doPsfMatch: 

98 # Backwards compatibility. 

99 log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

100 self.makePsfMatched = True 

101 self.makeDirect = False 

102 

103 def setDefaults(self): 

104 CoaddBaseTask.ConfigClass.setDefaults(self) 

105 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

106 

107## \addtogroup LSST_task_documentation 

108## \{ 

109## \page MakeCoaddTempExpTask 

110## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

111## \copybrief MakeCoaddTempExpTask 

112## \} 

113 

114 

115class MakeCoaddTempExpTask(CoaddBaseTask): 

116 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 

117 

118 @anchor MakeCoaddTempExpTask_ 

119 

120 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

121 

122 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

123 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

124 - @ref pipe_tasks_makeCoaddTempExp_IO 

125 - @ref pipe_tasks_makeCoaddTempExp_Config 

126 - @ref pipe_tasks_makeCoaddTempExp_Debug 

127 - @ref pipe_tasks_makeCoaddTempExp_Example 

128 

129 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

130 

131 Warp and optionally PSF-Match calexps onto a common projection, by 

132 performing the following operations: 

133 - Group calexps by visit/run 

134 - For each visit, generate a Warp by calling method @ref makeTempExp. 

135 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 

136 on each visit 

137 

138 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

139 

140 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

141 

142 @copydoc \_\_init\_\_ 

143 

144 This task has one special keyword argument: passing reuse=True will cause 

145 the task to skip the creation of warps that are already present in the 

146 output repositories. 

147 

148 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

149 

150 This task is primarily designed to be run from the command line. 

151 

152 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

153 to process. 

154 

155 @copydoc run 

156 

157 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

158 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

159 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

160 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

161 

162 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

163 

164 See @ref MakeCoaddTempExpConfig and parameters inherited from 

165 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 

166 

167 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

168 

169 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

170 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

171 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

172 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

173 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

174 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

175 

176 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

177 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

178 In general, for a set of input images, this config should equal the FWHM of the visit 

179 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

180 of the other config options offer a reasonable starting point. 

181 The following list presents the most common problems that arise from a misconfigured 

182 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

183 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

184 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

185 Problem: Explanation. *Solution* 

186 

187 *Troublshooting PSF-Matching Configuration:* 

188 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

189 For example:_ 

190 

191 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

192 

193 Note that increasing the kernel size also increases runtime. 

194 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

195 for matching kernel. _Provide the matcher with more data by either increasing 

196 the spatial sampling by decreasing the spatial cell size,_ 

197 

198 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

199 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

200 

201 _or increasing the padding around the Science PSF, for example:_ 

202 

203 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

204 

205 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

206 matching kernel dimensions, thus increasing the number of pixels available to fit 

207 after convolving the PSF with the matching kernel. 

208 Optionally, for debugging the effects of padding, the level of padding may be manually 

209 controlled by setting turning off the automatic padding and setting the number 

210 of pixels by which to pad the PSF: 

211 

212 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

213 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

214 

215 - Deconvolution: Matching a large PSF to a smaller PSF produces 

216 a telltale noise pattern which looks like ripples or a brain. 

217 _Increase the size of the requested model PSF. For example:_ 

218 

219 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

220 

221 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

222 _Increase the width of the Gaussian basis functions. For example:_ 

223 

224 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

225 # from default [0.7, 1.5, 3.0] 

226 

227 

228 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

229 

230 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

231 

232 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

233 

234 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

235 into the larger Data Release Processing. 

236 Set up by running: 

237 

238 setup ci_hsc 

239 cd $CI_HSC_DIR 

240 # if not built already: 

241 python $(which scons) # this will take a while 

242 

243 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

244 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

245 output respository with the desired SkyMap. The command, 

246 

247 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

248 --id patch=5,4 tract=0 filter=HSC-I \ 

249 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

250 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

251 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

252 makePsfMatched=True modelPsf.defaultFwhm=11 

253 

254 writes a direct and PSF-Matched Warp to 

255 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

256 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

257 respectively. 

258 

259 @note PSF-Matching in this particular dataset would benefit from adding 

260 `--configfile ./matchingConfig.py` to 

261 the command line arguments where `matchingConfig.py` is defined by: 

262 

263 echo " 

264 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

265 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

266 

267 

268 Add the option `--help` to see more options. 

269 """ 

270 ConfigClass = MakeCoaddTempExpConfig 

271 _DefaultName = "makeCoaddTempExp" 

272 

273 def __init__(self, reuse=False, **kwargs): 

274 CoaddBaseTask.__init__(self, **kwargs) 

275 self.reuse = reuse 

276 self.makeSubtask("warpAndPsfMatch") 

277 if self.config.hasFakes: 277 ↛ 278line 277 didn't jump to line 278, because the condition on line 277 was never true

278 self.calexpType = "fakes_calexp" 

279 else: 

280 self.calexpType = "calexp" 

281 

282 @pipeBase.timeMethod 

283 def runDataRef(self, patchRef, selectDataList=[]): 

284 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 

285 

286 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

287 plus the camera-specific filter key (e.g. "filter" or "band") 

288 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 

289 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 

290 warps are requested. 

291 

292 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

293 

294 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

295 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

296 (assembleCoadd should determine zeropoint scaling without referring to it). 

297 """ 

298 skyInfo = self.getSkyInfo(patchRef) 

299 

300 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

301 if self.config.makePsfMatched and not self.config.makeDirect: 301 ↛ 302line 301 didn't jump to line 302, because the condition on line 301 was never true

302 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

303 else: 

304 primaryWarpDataset = self.getTempExpDatasetName("direct") 

305 

306 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

307 

308 if len(calExpRefList) == 0: 308 ↛ 309line 308 didn't jump to line 309, because the condition on line 308 was never true

309 self.log.warn("No exposures to coadd for patch %s", patchRef.dataId) 

310 return None 

311 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

312 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

313 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

314 

315 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

316 primaryWarpDataset) 

317 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

318 

319 dataRefList = [] 

320 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

321 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

322 tempExpTuple, groupData.keys) 

323 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 323 ↛ 324line 323 didn't jump to line 324, because the condition on line 323 was never true

324 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

325 dataRefList.append(tempExpRef) 

326 continue 

327 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

328 

329 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

330 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

331 # of the visit in the list. 

332 try: 

333 visitId = int(tempExpRef.dataId["visit"]) 

334 except (KeyError, ValueError): 

335 visitId = i 

336 

337 calExpList = [] 

338 ccdIdList = [] 

339 dataIdList = [] 

340 

341 for calExpInd, calExpRef in enumerate(calexpRefList): 

342 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

343 calExpRef.dataId) 

344 try: 

345 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

346 except Exception: 

347 ccdId = calExpInd 

348 try: 

349 # We augment the dataRef here with the tract, which is harmless for loading things 

350 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

351 # which do. 

352 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

353 dataId=calExpRef.dataId, 

354 tract=skyInfo.tractInfo.getId()) 

355 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

356 except Exception as e: 

357 self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

358 continue 

359 

360 if self.config.doApplySkyCorr: 360 ↛ 361line 360 didn't jump to line 361, because the condition on line 360 was never true

361 self.applySkyCorr(calExpRef, calExp) 

362 

363 calExpList.append(calExp) 

364 ccdIdList.append(ccdId) 

365 dataIdList.append(calExpRef.dataId) 

366 

367 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

368 

369 if any(exps.values()): 

370 dataRefList.append(tempExpRef) 

371 else: 

372 self.log.warn("Warp %s could not be created", tempExpRef.dataId) 

373 

374 if self.config.doWrite: 374 ↛ 320line 374 didn't jump to line 320, because the condition on line 374 was never false

375 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

376 if exposure is not None: 

377 self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType)) 

378 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

379 

380 return dataRefList 

381 

382 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None): 

383 """Create a Warp from inputs 

384 

385 We iterate over the multiple calexps in a single exposure to construct 

386 the warp (previously called a coaddTempExp) of that exposure to the 

387 supplied tract/patch. 

388 

389 Pixels that receive no pixels are set to NAN; this is not correct 

390 (violates LSST algorithms group policy), but will be fixed up by 

391 interpolating after the coaddition. 

392 

393 @param calexpRefList: List of data references for calexps that (may) 

394 overlap the patch of interest 

395 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

396 information about the patch 

397 @param visitId: integer identifier for visit, for the table that will 

398 produce the CoaddPsf 

399 @return a pipeBase Struct containing: 

400 - exposures: a dictionary containing the warps requested: 

401 "direct": direct warp if config.makeDirect 

402 "psfMatched": PSF-matched warp if config.makePsfMatched 

403 """ 

404 warpTypeList = self.getWarpTypeList() 

405 

406 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

407 didSetMetadata = {warpType: False for warpType in warpTypeList} 

408 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

409 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

410 for warpType in warpTypeList} 

411 

412 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

413 if dataIdList is None: 413 ↛ 414line 413 didn't jump to line 414, because the condition on line 413 was never true

414 dataIdList = ccdIdList 

415 

416 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

417 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

418 calExpInd+1, len(calExpList), dataId) 

419 

420 try: 

421 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

422 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

423 makeDirect=self.config.makeDirect, 

424 makePsfMatched=self.config.makePsfMatched) 

425 except Exception as e: 

426 self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

427 continue 

428 try: 

429 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

430 for warpType in warpTypeList: 

431 exposure = warpedAndMatched.getDict()[warpType] 

432 if exposure is None: 

433 continue 

434 coaddTempExp = coaddTempExps[warpType] 

435 if didSetMetadata[warpType]: 

436 mimg = exposure.getMaskedImage() 

437 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

438 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

439 del mimg 

440 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

441 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

442 totGoodPix[warpType] += numGoodPix[warpType] 

443 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

444 dataId, numGoodPix[warpType], 

445 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

446 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

447 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

448 coaddTempExp.setFilter(exposure.getFilter()) 

449 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

450 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

451 coaddTempExp.setPsf(exposure.getPsf()) 

452 didSetMetadata[warpType] = True 

453 

454 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

455 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

456 

457 except Exception as e: 

458 self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e) 

459 continue 

460 

461 for warpType in warpTypeList: 

462 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

463 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

464 

465 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

466 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

467 if warpType == "direct": 

468 coaddTempExps[warpType].setPsf( 

469 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

470 self.config.coaddPsf.makeControl())) 

471 else: 

472 if not self.config.doWriteEmptyWarps: 472 ↛ 461line 472 didn't jump to line 461, because the condition on line 472 was never false

473 # No good pixels. Exposure still empty 

474 coaddTempExps[warpType] = None 

475 

476 result = pipeBase.Struct(exposures=coaddTempExps) 

477 return result 

478 

479 def getCalibratedExposure(self, dataRef, bgSubtracted): 

480 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

481 

482 @param[in] dataRef a sensor-level data reference 

483 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

484 calexp's background background model and add it to the calexp. 

485 @return calibrated exposure 

486 

487 @raises MissingExposureError If data for the exposure is not available. 

488 

489 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

490 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

491 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

492 retrieved from the processed exposure. When 

493 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

494 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

495 Otherwise, the astrometric calibration is taken from the processed 

496 exposure. 

497 """ 

498 try: 

499 exposure = dataRef.get(self.calexpType, immediate=True) 

500 except dafPersist.NoResults as e: 

501 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

502 

503 if not bgSubtracted: 503 ↛ 504line 503 didn't jump to line 504, because the condition on line 503 was never true

504 background = dataRef.get("calexpBackground", immediate=True) 

505 mi = exposure.getMaskedImage() 

506 mi += background.getImage() 

507 del mi 

508 

509 if self.config.doApplyExternalPhotoCalib: 

510 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

511 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

512 photoCalib = dataRef.get(source) 

513 exposure.setPhotoCalib(photoCalib) 

514 else: 

515 photoCalib = exposure.getPhotoCalib() 

516 

517 if self.config.doApplyExternalSkyWcs: 

518 source = f"{self.config.externalSkyWcsName}_wcs" 

519 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

520 skyWcs = dataRef.get(source) 

521 exposure.setWcs(skyWcs) 

522 

523 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

524 includeScaleUncertainty=self.config.includeCalibVar) 

525 exposure.maskedImage /= photoCalib.getCalibrationMean() 

526 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

527 # exposure.setCalib(afwImage.Calib(1.0)) 

528 return exposure 

529 

530 @staticmethod 

531 def _prepareEmptyExposure(skyInfo): 

532 """Produce an empty exposure for a given patch""" 

533 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

534 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

535 .getPlaneBitMask("NO_DATA"), numpy.inf) 

536 return exp 

537 

538 def getWarpTypeList(self): 

539 """Return list of requested warp types per the config. 

540 """ 

541 warpTypeList = [] 

542 if self.config.makeDirect: 542 ↛ 544line 542 didn't jump to line 544, because the condition on line 542 was never false

543 warpTypeList.append("direct") 

544 if self.config.makePsfMatched: 544 ↛ 546line 544 didn't jump to line 546, because the condition on line 544 was never false

545 warpTypeList.append("psfMatched") 

546 return warpTypeList 

547 

548 def applySkyCorr(self, dataRef, calexp): 

549 """Apply correction to the sky background level 

550 

551 Sky corrections can be generated with the 'skyCorrection.py' 

552 executable in pipe_drivers. Because the sky model used by that 

553 code extends over the entire focal plane, this can produce 

554 better sky subtraction. 

555 

556 The calexp is updated in-place. 

557 

558 Parameters 

559 ---------- 

560 dataRef : `lsst.daf.persistence.ButlerDataRef` 

561 Data reference for calexp. 

562 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

563 Calibrated exposure. 

564 """ 

565 bg = dataRef.get("skyCorr") 

566 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

567 if isinstance(calexp, afwImage.Exposure): 

568 calexp = calexp.getMaskedImage() 

569 calexp -= bg.getImage() 

570 

571 

572class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

573 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

574 defaultTemplates={"coaddName": "deep"}): 

575 calExpList = cT.Input( 

576 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

577 name="calexp", 

578 storageClass="ExposureF", 

579 dimensions=("instrument", "visit", "detector"), 

580 multiple=True, 

581 ) 

582 backgroundList = cT.Input( 

583 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

584 name="calexpBackground", 

585 storageClass="Background", 

586 dimensions=("instrument", "visit", "detector"), 

587 multiple=True, 

588 ) 

589 skyCorrList = cT.Input( 

590 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

591 name="skyCorr", 

592 storageClass="Background", 

593 dimensions=("instrument", "visit", "detector"), 

594 multiple=True, 

595 ) 

596 skyMap = cT.Input( 

597 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

598 name="{coaddName}Coadd_skyMap", 

599 storageClass="SkyMap", 

600 dimensions=("skymap",), 

601 ) 

602 direct = cT.Output( 

603 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

604 "calexps onto the skyMap patch geometry."), 

605 name="{coaddName}Coadd_directWarp", 

606 storageClass="ExposureF", 

607 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

608 ) 

609 psfMatched = cT.Output( 

610 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

611 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

612 name="{coaddName}Coadd_psfMatchedWarp", 

613 storageClass="ExposureF", 

614 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

615 ) 

616 

617 def __init__(self, *, config=None): 

618 super().__init__(config=config) 

619 if config.bgSubtracted: 

620 self.inputs.remove("backgroundList") 

621 if not config.doApplySkyCorr: 

622 self.inputs.remove("skyCorrList") 

623 if not config.makeDirect: 

624 self.outputs.remove("direct") 

625 if not config.makePsfMatched: 

626 self.outputs.remove("psfMatched") 

627 

628 

629class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

630 pipelineConnections=MakeWarpConnections): 

631 

632 def validate(self): 

633 super().validate() 

634 # TODO: Remove this constraint after DM-17062 

635 if self.doApplyExternalPhotoCalib: 

636 raise RuntimeError("Gen3 MakeWarpTask cannot apply external PhotoCalib results. " 

637 "Please set doApplyExternalPhotoCalib=False.") 

638 if self.doApplyExternalSkyWcs: 

639 raise RuntimeError("Gen3 MakeWarpTask cannot apply external SkyWcs results. " 

640 "Please set doApplyExternalSkyWcs=False.") 

641 

642 

643class MakeWarpTask(MakeCoaddTempExpTask, pipeBase.PipelineTask): 

644 """Warp and optionally PSF-Match calexps onto an a common projection 

645 

646 First Draft of a Gen3 compatible MakeWarpTask which 

647 currently does not handle doApplyExternalPhotoCalib=True or 

648 doApplyExternalSkyWcs=True. 

649 """ 

650 ConfigClass = MakeWarpConfig 

651 _DefaultName = "makeWarp" 

652 

653 @utils.inheritDoc(pipeBase.PipelineTask) 

654 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

655 """ 

656 Notes 

657 ---- 

658 Construct warps for requested warp type for single epoch 

659 

660 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

661 calexps. This method is analogous to `runDataRef`. 

662 """ 

663 # Read in all inputs. 

664 inputs = butlerQC.get(inputRefs) 

665 

666 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

667 # from the dictionary so we can pass it as kwargs later. 

668 skyMap = inputs.pop("skyMap") 

669 quantumDataId = butlerQC.quantum.dataId 

670 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

671 

672 # Construct list of input DataIds expected by `run` 

673 dataIdList = [ref.dataId for ref in inputRefs.calExpList] 

674 

675 # Construct list of packed integer IDs expected by `run` 

676 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

677 

678 # Extract integer visitId requested by `run` 

679 visits = [dataId['visit'] for dataId in dataIdList] 

680 assert(all(visits[0] == visit for visit in visits)) 

681 visitId = visits[0] 

682 

683 self.prepareCalibratedExposures(**inputs) 

684 results = self.run(**inputs, visitId=visitId, ccdIdList=ccdIdList, dataIdList=dataIdList, 

685 skyInfo=skyInfo) 

686 if self.config.makeDirect: 

687 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

688 if self.config.makePsfMatched: 

689 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

690 

691 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None): 

692 """Calibrate and add backgrounds to input calExpList in place 

693 

694 TODO DM-17062: apply jointcal/meas_mosaic here 

695 

696 Parameters 

697 ---------- 

698 calExpList : `list` of `lsst.afw.image.Exposure` 

699 Sequence of calexps to be modified in place 

700 backgroundList : `list` of `lsst.afw.math.backgroundList` 

701 Sequence of backgrounds to be added back in if bgSubtracted=False 

702 skyCorrList : `list` of `lsst.afw.math.backgroundList` 

703 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

704 """ 

705 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

706 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

707 for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList): 

708 mi = calexp.maskedImage 

709 if not self.config.bgSubtracted: 

710 mi += background.getImage() 

711 if self.config.doApplySkyCorr: 

712 mi -= skyCorr.getImage()