Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23 

24import lsst.pex.config as pexConfig 

25import lsst.daf.persistence as dafPersist 

26import lsst.afw.image as afwImage 

27import lsst.coadd.utils as coaddUtils 

28import lsst.pipe.base as pipeBase 

29import lsst.pipe.base.connectionTypes as cT 

30import lsst.log as log 

31import lsst.utils as utils 

32from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

33from lsst.skymap import BaseSkyMap 

34from .coaddBase import CoaddBaseTask, makeSkyInfo 

35from .warpAndPsfMatch import WarpAndPsfMatchTask 

36from .coaddHelpers import groupPatchExposures, getGroupDataRef 

37 

38__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

39 

40 

41class MissingExposureError(Exception): 

42 """Raised when data cannot be retrieved for an exposure. 

43 When processing patches, sometimes one exposure is missing; this lets us 

44 distinguish bewteen that case, and other errors. 

45 """ 

46 pass 

47 

48 

49class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

50 """Config for MakeCoaddTempExpTask 

51 """ 

52 warpAndPsfMatch = pexConfig.ConfigurableField( 

53 target=WarpAndPsfMatchTask, 

54 doc="Task to warp and PSF-match calexp", 

55 ) 

56 doWrite = pexConfig.Field( 

57 doc="persist <coaddName>Coadd_<warpType>Warp", 

58 dtype=bool, 

59 default=True, 

60 ) 

61 bgSubtracted = pexConfig.Field( 

62 doc="Work with a background subtracted calexp?", 

63 dtype=bool, 

64 default=True, 

65 ) 

66 coaddPsf = pexConfig.ConfigField( 

67 doc="Configuration for CoaddPsf", 

68 dtype=CoaddPsfConfig, 

69 ) 

70 makeDirect = pexConfig.Field( 

71 doc="Make direct Warp/Coadds", 

72 dtype=bool, 

73 default=True, 

74 ) 

75 makePsfMatched = pexConfig.Field( 

76 doc="Make Psf-Matched Warp/Coadd?", 

77 dtype=bool, 

78 default=False, 

79 ) 

80 

81 doWriteEmptyWarps = pexConfig.Field( 

82 dtype=bool, 

83 default=False, 

84 doc="Write out warps even if they are empty" 

85 ) 

86 

87 hasFakes = pexConfig.Field( 

88 doc="Should be set to True if fake sources have been inserted into the input data.", 

89 dtype=bool, 

90 default=False, 

91 ) 

92 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

93 

94 def validate(self): 

95 CoaddBaseTask.ConfigClass.validate(self) 

96 if not self.makePsfMatched and not self.makeDirect: 

97 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

98 if self.doPsfMatch: 

99 # Backwards compatibility. 

100 log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

101 self.makePsfMatched = True 

102 self.makeDirect = False 

103 

104 def setDefaults(self): 

105 CoaddBaseTask.ConfigClass.setDefaults(self) 

106 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

107 

108## \addtogroup LSST_task_documentation 

109## \{ 

110## \page MakeCoaddTempExpTask 

111## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

112## \copybrief MakeCoaddTempExpTask 

113## \} 

114 

115 

116class MakeCoaddTempExpTask(CoaddBaseTask): 

117 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 

118 

119 @anchor MakeCoaddTempExpTask_ 

120 

121 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

122 

123 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

124 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

125 - @ref pipe_tasks_makeCoaddTempExp_IO 

126 - @ref pipe_tasks_makeCoaddTempExp_Config 

127 - @ref pipe_tasks_makeCoaddTempExp_Debug 

128 - @ref pipe_tasks_makeCoaddTempExp_Example 

129 

130 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

131 

132 Warp and optionally PSF-Match calexps onto a common projection, by 

133 performing the following operations: 

134 - Group calexps by visit/run 

135 - For each visit, generate a Warp by calling method @ref makeTempExp. 

136 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 

137 on each visit 

138 

139 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

140 

141 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

142 

143 @copydoc \_\_init\_\_ 

144 

145 This task has one special keyword argument: passing reuse=True will cause 

146 the task to skip the creation of warps that are already present in the 

147 output repositories. 

148 

149 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

150 

151 This task is primarily designed to be run from the command line. 

152 

153 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

154 to process. 

155 

156 @copydoc run 

157 

158 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

159 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

160 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

161 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

162 

163 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

164 

165 See @ref MakeCoaddTempExpConfig and parameters inherited from 

166 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 

167 

168 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

169 

170 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

171 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

172 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

173 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

174 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

175 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

176 

177 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

178 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

179 In general, for a set of input images, this config should equal the FWHM of the visit 

180 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

181 of the other config options offer a reasonable starting point. 

182 The following list presents the most common problems that arise from a misconfigured 

183 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

184 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

185 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

186 Problem: Explanation. *Solution* 

187 

188 *Troublshooting PSF-Matching Configuration:* 

189 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

190 For example:_ 

191 

192 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

193 

194 Note that increasing the kernel size also increases runtime. 

195 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

196 for matching kernel. _Provide the matcher with more data by either increasing 

197 the spatial sampling by decreasing the spatial cell size,_ 

198 

199 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

200 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

201 

202 _or increasing the padding around the Science PSF, for example:_ 

203 

204 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

205 

206 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

207 matching kernel dimensions, thus increasing the number of pixels available to fit 

208 after convolving the PSF with the matching kernel. 

209 Optionally, for debugging the effects of padding, the level of padding may be manually 

210 controlled by setting turning off the automatic padding and setting the number 

211 of pixels by which to pad the PSF: 

212 

213 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

214 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

215 

216 - Deconvolution: Matching a large PSF to a smaller PSF produces 

217 a telltale noise pattern which looks like ripples or a brain. 

218 _Increase the size of the requested model PSF. For example:_ 

219 

220 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

221 

222 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

223 _Increase the width of the Gaussian basis functions. For example:_ 

224 

225 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

226 # from default [0.7, 1.5, 3.0] 

227 

228 

229 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

230 

231 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

232 

233 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

234 

235 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

236 into the larger Data Release Processing. 

237 Set up by running: 

238 

239 setup ci_hsc 

240 cd $CI_HSC_DIR 

241 # if not built already: 

242 python $(which scons) # this will take a while 

243 

244 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

245 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

246 output respository with the desired SkyMap. The command, 

247 

248 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

249 --id patch=5,4 tract=0 filter=HSC-I \ 

250 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

251 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

252 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

253 makePsfMatched=True modelPsf.defaultFwhm=11 

254 

255 writes a direct and PSF-Matched Warp to 

256 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

257 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

258 respectively. 

259 

260 @note PSF-Matching in this particular dataset would benefit from adding 

261 `--configfile ./matchingConfig.py` to 

262 the command line arguments where `matchingConfig.py` is defined by: 

263 

264 echo " 

265 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

266 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

267 

268 

269 Add the option `--help` to see more options. 

270 """ 

271 ConfigClass = MakeCoaddTempExpConfig 

272 _DefaultName = "makeCoaddTempExp" 

273 

274 def __init__(self, reuse=False, **kwargs): 

275 CoaddBaseTask.__init__(self, **kwargs) 

276 self.reuse = reuse 

277 self.makeSubtask("warpAndPsfMatch") 

278 if self.config.hasFakes: 278 ↛ 279line 278 didn't jump to line 279, because the condition on line 278 was never true

279 self.calexpType = "fakes_calexp" 

280 else: 

281 self.calexpType = "calexp" 

282 

283 @pipeBase.timeMethod 

284 def runDataRef(self, patchRef, selectDataList=[]): 

285 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 

286 

287 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

288 plus the camera-specific filter key (e.g. "filter" or "band") 

289 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 

290 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 

291 warps are requested. 

292 

293 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

294 

295 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

296 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

297 (assembleCoadd should determine zeropoint scaling without referring to it). 

298 """ 

299 skyInfo = self.getSkyInfo(patchRef) 

300 

301 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

302 if self.config.makePsfMatched and not self.config.makeDirect: 302 ↛ 303line 302 didn't jump to line 303, because the condition on line 302 was never true

303 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

304 else: 

305 primaryWarpDataset = self.getTempExpDatasetName("direct") 

306 

307 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

308 

309 if len(calExpRefList) == 0: 309 ↛ 310line 309 didn't jump to line 310, because the condition on line 309 was never true

310 self.log.warn("No exposures to coadd for patch %s", patchRef.dataId) 

311 return None 

312 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

313 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

314 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

315 

316 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

317 primaryWarpDataset) 

318 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

319 

320 dataRefList = [] 

321 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

322 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

323 tempExpTuple, groupData.keys) 

324 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 324 ↛ 325line 324 didn't jump to line 325, because the condition on line 324 was never true

325 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

326 dataRefList.append(tempExpRef) 

327 continue 

328 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

329 

330 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

331 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

332 # of the visit in the list. 

333 try: 

334 visitId = int(tempExpRef.dataId["visit"]) 

335 except (KeyError, ValueError): 

336 visitId = i 

337 

338 calExpList = [] 

339 ccdIdList = [] 

340 dataIdList = [] 

341 

342 for calExpInd, calExpRef in enumerate(calexpRefList): 

343 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

344 calExpRef.dataId) 

345 try: 

346 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

347 except Exception: 

348 ccdId = calExpInd 

349 try: 

350 # We augment the dataRef here with the tract, which is harmless for loading things 

351 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

352 # which do. 

353 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

354 dataId=calExpRef.dataId, 

355 tract=skyInfo.tractInfo.getId()) 

356 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

357 except Exception as e: 

358 self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

359 continue 

360 

361 if self.config.doApplySkyCorr: 361 ↛ 362line 361 didn't jump to line 362, because the condition on line 361 was never true

362 self.applySkyCorr(calExpRef, calExp) 

363 

364 calExpList.append(calExp) 

365 ccdIdList.append(ccdId) 

366 dataIdList.append(calExpRef.dataId) 

367 

368 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

369 

370 if any(exps.values()): 

371 dataRefList.append(tempExpRef) 

372 else: 

373 self.log.warn("Warp %s could not be created", tempExpRef.dataId) 

374 

375 if self.config.doWrite: 375 ↛ 321line 375 didn't jump to line 321, because the condition on line 375 was never false

376 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

377 if exposure is not None: 

378 self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType)) 

379 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

380 

381 return dataRefList 

382 

383 @pipeBase.timeMethod 

384 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None): 

385 """Create a Warp from inputs 

386 

387 We iterate over the multiple calexps in a single exposure to construct 

388 the warp (previously called a coaddTempExp) of that exposure to the 

389 supplied tract/patch. 

390 

391 Pixels that receive no pixels are set to NAN; this is not correct 

392 (violates LSST algorithms group policy), but will be fixed up by 

393 interpolating after the coaddition. 

394 

395 @param calexpRefList: List of data references for calexps that (may) 

396 overlap the patch of interest 

397 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

398 information about the patch 

399 @param visitId: integer identifier for visit, for the table that will 

400 produce the CoaddPsf 

401 @return a pipeBase Struct containing: 

402 - exposures: a dictionary containing the warps requested: 

403 "direct": direct warp if config.makeDirect 

404 "psfMatched": PSF-matched warp if config.makePsfMatched 

405 """ 

406 warpTypeList = self.getWarpTypeList() 

407 

408 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

409 didSetMetadata = {warpType: False for warpType in warpTypeList} 

410 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

411 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

412 for warpType in warpTypeList} 

413 

414 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

415 if dataIdList is None: 415 ↛ 416line 415 didn't jump to line 416, because the condition on line 415 was never true

416 dataIdList = ccdIdList 

417 

418 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

419 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

420 calExpInd+1, len(calExpList), dataId) 

421 

422 try: 

423 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

424 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

425 makeDirect=self.config.makeDirect, 

426 makePsfMatched=self.config.makePsfMatched) 

427 except Exception as e: 

428 self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

429 continue 

430 try: 

431 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

432 for warpType in warpTypeList: 

433 exposure = warpedAndMatched.getDict()[warpType] 

434 if exposure is None: 

435 continue 

436 coaddTempExp = coaddTempExps[warpType] 

437 if didSetMetadata[warpType]: 

438 mimg = exposure.getMaskedImage() 

439 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

440 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

441 del mimg 

442 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

443 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

444 totGoodPix[warpType] += numGoodPix[warpType] 

445 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

446 dataId, numGoodPix[warpType], 

447 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

448 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

449 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

450 coaddTempExp.setFilter(exposure.getFilter()) 

451 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

452 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

453 coaddTempExp.setPsf(exposure.getPsf()) 

454 didSetMetadata[warpType] = True 

455 

456 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

457 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

458 

459 except Exception as e: 

460 self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e) 

461 continue 

462 

463 for warpType in warpTypeList: 

464 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

465 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

466 

467 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

468 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

469 if warpType == "direct": 

470 coaddTempExps[warpType].setPsf( 

471 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

472 self.config.coaddPsf.makeControl())) 

473 else: 

474 if not self.config.doWriteEmptyWarps: 474 ↛ 463line 474 didn't jump to line 463, because the condition on line 474 was never false

475 # No good pixels. Exposure still empty 

476 coaddTempExps[warpType] = None 

477 

478 result = pipeBase.Struct(exposures=coaddTempExps) 

479 return result 

480 

481 def getCalibratedExposure(self, dataRef, bgSubtracted): 

482 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

483 

484 @param[in] dataRef a sensor-level data reference 

485 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

486 calexp's background background model and add it to the calexp. 

487 @return calibrated exposure 

488 

489 @raises MissingExposureError If data for the exposure is not available. 

490 

491 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

492 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

493 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

494 retrieved from the processed exposure. When 

495 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

496 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

497 Otherwise, the astrometric calibration is taken from the processed 

498 exposure. 

499 """ 

500 try: 

501 exposure = dataRef.get(self.calexpType, immediate=True) 

502 except dafPersist.NoResults as e: 

503 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

504 

505 if not bgSubtracted: 505 ↛ 506line 505 didn't jump to line 506, because the condition on line 505 was never true

506 background = dataRef.get("calexpBackground", immediate=True) 

507 mi = exposure.getMaskedImage() 

508 mi += background.getImage() 

509 del mi 

510 

511 if self.config.doApplyExternalPhotoCalib: 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true

512 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

513 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

514 photoCalib = dataRef.get(source) 

515 exposure.setPhotoCalib(photoCalib) 

516 else: 

517 photoCalib = exposure.getPhotoCalib() 

518 

519 if self.config.doApplyExternalSkyWcs: 519 ↛ 520line 519 didn't jump to line 520, because the condition on line 519 was never true

520 source = f"{self.config.externalSkyWcsName}_wcs" 

521 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

522 skyWcs = dataRef.get(source) 

523 exposure.setWcs(skyWcs) 

524 

525 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

526 includeScaleUncertainty=self.config.includeCalibVar) 

527 exposure.maskedImage /= photoCalib.getCalibrationMean() 

528 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

529 # exposure.setCalib(afwImage.Calib(1.0)) 

530 return exposure 

531 

532 @staticmethod 

533 def _prepareEmptyExposure(skyInfo): 

534 """Produce an empty exposure for a given patch""" 

535 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

536 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

537 .getPlaneBitMask("NO_DATA"), numpy.inf) 

538 return exp 

539 

540 def getWarpTypeList(self): 

541 """Return list of requested warp types per the config. 

542 """ 

543 warpTypeList = [] 

544 if self.config.makeDirect: 544 ↛ 546line 544 didn't jump to line 546, because the condition on line 544 was never false

545 warpTypeList.append("direct") 

546 if self.config.makePsfMatched: 546 ↛ 548line 546 didn't jump to line 548, because the condition on line 546 was never false

547 warpTypeList.append("psfMatched") 

548 return warpTypeList 

549 

550 def applySkyCorr(self, dataRef, calexp): 

551 """Apply correction to the sky background level 

552 

553 Sky corrections can be generated with the 'skyCorrection.py' 

554 executable in pipe_drivers. Because the sky model used by that 

555 code extends over the entire focal plane, this can produce 

556 better sky subtraction. 

557 

558 The calexp is updated in-place. 

559 

560 Parameters 

561 ---------- 

562 dataRef : `lsst.daf.persistence.ButlerDataRef` 

563 Data reference for calexp. 

564 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

565 Calibrated exposure. 

566 """ 

567 bg = dataRef.get("skyCorr") 

568 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

569 if isinstance(calexp, afwImage.Exposure): 

570 calexp = calexp.getMaskedImage() 

571 calexp -= bg.getImage() 

572 

573 

574class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

575 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

576 defaultTemplates={"coaddName": "deep"}): 

577 calExpList = cT.Input( 

578 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

579 name="calexp", 

580 storageClass="ExposureF", 

581 dimensions=("instrument", "visit", "detector"), 

582 multiple=True, 

583 ) 

584 backgroundList = cT.Input( 

585 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

586 name="calexpBackground", 

587 storageClass="Background", 

588 dimensions=("instrument", "visit", "detector"), 

589 multiple=True, 

590 ) 

591 skyCorrList = cT.Input( 

592 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

593 name="skyCorr", 

594 storageClass="Background", 

595 dimensions=("instrument", "visit", "detector"), 

596 multiple=True, 

597 ) 

598 skyMap = cT.Input( 

599 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

600 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

601 storageClass="SkyMap", 

602 dimensions=("skymap",), 

603 ) 

604 direct = cT.Output( 

605 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

606 "calexps onto the skyMap patch geometry."), 

607 name="{coaddName}Coadd_directWarp", 

608 storageClass="ExposureF", 

609 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

610 ) 

611 psfMatched = cT.Output( 

612 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

613 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

614 name="{coaddName}Coadd_psfMatchedWarp", 

615 storageClass="ExposureF", 

616 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

617 ) 

618 

619 def __init__(self, *, config=None): 

620 super().__init__(config=config) 

621 if config.bgSubtracted: 

622 self.inputs.remove("backgroundList") 

623 if not config.doApplySkyCorr: 

624 self.inputs.remove("skyCorrList") 

625 if not config.makeDirect: 

626 self.outputs.remove("direct") 

627 if not config.makePsfMatched: 

628 self.outputs.remove("psfMatched") 

629 

630 

631class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

632 pipelineConnections=MakeWarpConnections): 

633 

634 def validate(self): 

635 super().validate() 

636 # TODO: Remove this constraint after DM-17062 

637 if self.doApplyExternalPhotoCalib: 

638 raise RuntimeError("Gen3 MakeWarpTask cannot apply external PhotoCalib results. " 

639 "Please set doApplyExternalPhotoCalib=False.") 

640 if self.doApplyExternalSkyWcs: 

641 raise RuntimeError("Gen3 MakeWarpTask cannot apply external SkyWcs results. " 

642 "Please set doApplyExternalSkyWcs=False.") 

643 

644 

645class MakeWarpTask(MakeCoaddTempExpTask, pipeBase.PipelineTask): 

646 """Warp and optionally PSF-Match calexps onto an a common projection 

647 

648 First Draft of a Gen3 compatible MakeWarpTask which 

649 currently does not handle doApplyExternalPhotoCalib=True or 

650 doApplyExternalSkyWcs=True. 

651 """ 

652 ConfigClass = MakeWarpConfig 

653 _DefaultName = "makeWarp" 

654 

655 @utils.inheritDoc(pipeBase.PipelineTask) 

656 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

657 """ 

658 Notes 

659 ---- 

660 Construct warps for requested warp type for single epoch 

661 

662 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

663 calexps. This method is analogous to `runDataRef`. 

664 """ 

665 # Read in all inputs. 

666 inputs = butlerQC.get(inputRefs) 

667 

668 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

669 # from the dictionary so we can pass it as kwargs later. 

670 skyMap = inputs.pop("skyMap") 

671 quantumDataId = butlerQC.quantum.dataId 

672 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

673 

674 # Construct list of input DataIds expected by `run` 

675 dataIdList = [ref.dataId for ref in inputRefs.calExpList] 

676 

677 # Construct list of packed integer IDs expected by `run` 

678 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

679 

680 # Extract integer visitId requested by `run` 

681 visits = [dataId['visit'] for dataId in dataIdList] 

682 assert(all(visits[0] == visit for visit in visits)) 

683 visitId = visits[0] 

684 

685 self.prepareCalibratedExposures(**inputs) 

686 results = self.run(**inputs, visitId=visitId, ccdIdList=ccdIdList, dataIdList=dataIdList, 

687 skyInfo=skyInfo) 

688 if self.config.makeDirect: 

689 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

690 if self.config.makePsfMatched: 

691 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

692 

693 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None): 

694 """Calibrate and add backgrounds to input calExpList in place 

695 

696 TODO DM-17062: apply jointcal/meas_mosaic here 

697 

698 Parameters 

699 ---------- 

700 calExpList : `list` of `lsst.afw.image.Exposure` 

701 Sequence of calexps to be modified in place 

702 backgroundList : `list` of `lsst.afw.math.backgroundList` 

703 Sequence of backgrounds to be added back in if bgSubtracted=False 

704 skyCorrList : `list` of `lsst.afw.math.backgroundList` 

705 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

706 """ 

707 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

708 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

709 for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList): 

710 mi = calexp.maskedImage 

711 if not self.config.bgSubtracted: 

712 mi += background.getImage() 

713 if self.config.doApplySkyCorr: 

714 mi -= skyCorr.getImage()