Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23 

24import lsst.pex.config as pexConfig 

25import lsst.daf.persistence as dafPersist 

26import lsst.afw.image as afwImage 

27import lsst.coadd.utils as coaddUtils 

28import lsst.pipe.base as pipeBase 

29import lsst.pipe.base.connectionTypes as connectionTypes 

30import lsst.log as log 

31import lsst.utils as utils 

32import lsst.geom 

33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

34from lsst.skymap import BaseSkyMap 

35from .coaddBase import CoaddBaseTask, makeSkyInfo 

36from .warpAndPsfMatch import WarpAndPsfMatchTask 

37from .coaddHelpers import groupPatchExposures, getGroupDataRef 

38 

39__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

40 

41 

42class MissingExposureError(Exception): 

43 """Raised when data cannot be retrieved for an exposure. 

44 When processing patches, sometimes one exposure is missing; this lets us 

45 distinguish bewteen that case, and other errors. 

46 """ 

47 pass 

48 

49 

50class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

51 """Config for MakeCoaddTempExpTask 

52 """ 

53 warpAndPsfMatch = pexConfig.ConfigurableField( 

54 target=WarpAndPsfMatchTask, 

55 doc="Task to warp and PSF-match calexp", 

56 ) 

57 doWrite = pexConfig.Field( 

58 doc="persist <coaddName>Coadd_<warpType>Warp", 

59 dtype=bool, 

60 default=True, 

61 ) 

62 bgSubtracted = pexConfig.Field( 

63 doc="Work with a background subtracted calexp?", 

64 dtype=bool, 

65 default=True, 

66 ) 

67 coaddPsf = pexConfig.ConfigField( 

68 doc="Configuration for CoaddPsf", 

69 dtype=CoaddPsfConfig, 

70 ) 

71 makeDirect = pexConfig.Field( 

72 doc="Make direct Warp/Coadds", 

73 dtype=bool, 

74 default=True, 

75 ) 

76 makePsfMatched = pexConfig.Field( 

77 doc="Make Psf-Matched Warp/Coadd?", 

78 dtype=bool, 

79 default=False, 

80 ) 

81 

82 doWriteEmptyWarps = pexConfig.Field( 

83 dtype=bool, 

84 default=False, 

85 doc="Write out warps even if they are empty" 

86 ) 

87 

88 hasFakes = pexConfig.Field( 

89 doc="Should be set to True if fake sources have been inserted into the input data.", 

90 dtype=bool, 

91 default=False, 

92 ) 

93 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

94 

95 def validate(self): 

96 CoaddBaseTask.ConfigClass.validate(self) 

97 if not self.makePsfMatched and not self.makeDirect: 

98 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

99 if self.doPsfMatch: 

100 # Backwards compatibility. 

101 log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

102 self.makePsfMatched = True 

103 self.makeDirect = False 

104 

105 def setDefaults(self): 

106 CoaddBaseTask.ConfigClass.setDefaults(self) 

107 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

108 

109## \addtogroup LSST_task_documentation 

110## \{ 

111## \page MakeCoaddTempExpTask 

112## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

113## \copybrief MakeCoaddTempExpTask 

114## \} 

115 

116 

117class MakeCoaddTempExpTask(CoaddBaseTask): 

118 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 

119 

120 @anchor MakeCoaddTempExpTask_ 

121 

122 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

123 

124 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

125 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

126 - @ref pipe_tasks_makeCoaddTempExp_IO 

127 - @ref pipe_tasks_makeCoaddTempExp_Config 

128 - @ref pipe_tasks_makeCoaddTempExp_Debug 

129 - @ref pipe_tasks_makeCoaddTempExp_Example 

130 

131 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

132 

133 Warp and optionally PSF-Match calexps onto a common projection, by 

134 performing the following operations: 

135 - Group calexps by visit/run 

136 - For each visit, generate a Warp by calling method @ref makeTempExp. 

137 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 

138 on each visit 

139 

140 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

141 

142 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

143 

144 @copydoc \_\_init\_\_ 

145 

146 This task has one special keyword argument: passing reuse=True will cause 

147 the task to skip the creation of warps that are already present in the 

148 output repositories. 

149 

150 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

151 

152 This task is primarily designed to be run from the command line. 

153 

154 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

155 to process. 

156 

157 @copydoc run 

158 

159 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

160 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

161 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

162 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

163 

164 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

165 

166 See @ref MakeCoaddTempExpConfig and parameters inherited from 

167 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 

168 

169 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

170 

171 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

172 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

173 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

174 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

175 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

176 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

177 

178 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

179 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

180 In general, for a set of input images, this config should equal the FWHM of the visit 

181 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

182 of the other config options offer a reasonable starting point. 

183 The following list presents the most common problems that arise from a misconfigured 

184 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

185 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

186 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

187 Problem: Explanation. *Solution* 

188 

189 *Troublshooting PSF-Matching Configuration:* 

190 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

191 For example:_ 

192 

193 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

194 

195 Note that increasing the kernel size also increases runtime. 

196 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

197 for matching kernel. _Provide the matcher with more data by either increasing 

198 the spatial sampling by decreasing the spatial cell size,_ 

199 

200 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

201 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

202 

203 _or increasing the padding around the Science PSF, for example:_ 

204 

205 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

206 

207 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

208 matching kernel dimensions, thus increasing the number of pixels available to fit 

209 after convolving the PSF with the matching kernel. 

210 Optionally, for debugging the effects of padding, the level of padding may be manually 

211 controlled by setting turning off the automatic padding and setting the number 

212 of pixels by which to pad the PSF: 

213 

214 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

215 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

216 

217 - Deconvolution: Matching a large PSF to a smaller PSF produces 

218 a telltale noise pattern which looks like ripples or a brain. 

219 _Increase the size of the requested model PSF. For example:_ 

220 

221 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

222 

223 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

224 _Increase the width of the Gaussian basis functions. For example:_ 

225 

226 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

227 # from default [0.7, 1.5, 3.0] 

228 

229 

230 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

231 

232 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

233 

234 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

235 

236 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

237 into the larger Data Release Processing. 

238 Set up by running: 

239 

240 setup ci_hsc 

241 cd $CI_HSC_DIR 

242 # if not built already: 

243 python $(which scons) # this will take a while 

244 

245 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

246 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

247 output respository with the desired SkyMap. The command, 

248 

249 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

250 --id patch=5,4 tract=0 filter=HSC-I \ 

251 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

252 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

253 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

254 makePsfMatched=True modelPsf.defaultFwhm=11 

255 

256 writes a direct and PSF-Matched Warp to 

257 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

258 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

259 respectively. 

260 

261 @note PSF-Matching in this particular dataset would benefit from adding 

262 `--configfile ./matchingConfig.py` to 

263 the command line arguments where `matchingConfig.py` is defined by: 

264 

265 echo " 

266 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

267 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

268 

269 

270 Add the option `--help` to see more options. 

271 """ 

272 ConfigClass = MakeCoaddTempExpConfig 

273 _DefaultName = "makeCoaddTempExp" 

274 

275 def __init__(self, reuse=False, **kwargs): 

276 CoaddBaseTask.__init__(self, **kwargs) 

277 self.reuse = reuse 

278 self.makeSubtask("warpAndPsfMatch") 

279 if self.config.hasFakes: 279 ↛ 280line 279 didn't jump to line 280, because the condition on line 279 was never true

280 self.calexpType = "fakes_calexp" 

281 else: 

282 self.calexpType = "calexp" 

283 

284 @pipeBase.timeMethod 

285 def runDataRef(self, patchRef, selectDataList=[]): 

286 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 

287 

288 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

289 plus the camera-specific filter key (e.g. "filter" or "band") 

290 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 

291 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 

292 warps are requested. 

293 

294 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

295 

296 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

297 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

298 (assembleCoadd should determine zeropoint scaling without referring to it). 

299 """ 

300 skyInfo = self.getSkyInfo(patchRef) 

301 

302 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

303 if self.config.makePsfMatched and not self.config.makeDirect: 303 ↛ 304line 303 didn't jump to line 304, because the condition on line 303 was never true

304 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

305 else: 

306 primaryWarpDataset = self.getTempExpDatasetName("direct") 

307 

308 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

309 

310 if len(calExpRefList) == 0: 310 ↛ 311line 310 didn't jump to line 311, because the condition on line 310 was never true

311 self.log.warn("No exposures to coadd for patch %s", patchRef.dataId) 

312 return None 

313 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

314 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

315 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

316 

317 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

318 primaryWarpDataset) 

319 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

320 

321 dataRefList = [] 

322 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

323 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

324 tempExpTuple, groupData.keys) 

325 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 325 ↛ 326line 325 didn't jump to line 326, because the condition on line 325 was never true

326 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

327 dataRefList.append(tempExpRef) 

328 continue 

329 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

330 

331 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

332 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

333 # of the visit in the list. 

334 try: 

335 visitId = int(tempExpRef.dataId["visit"]) 

336 except (KeyError, ValueError): 

337 visitId = i 

338 

339 calExpList = [] 

340 ccdIdList = [] 

341 dataIdList = [] 

342 

343 for calExpInd, calExpRef in enumerate(calexpRefList): 

344 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

345 calExpRef.dataId) 

346 try: 

347 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

348 except Exception: 

349 ccdId = calExpInd 

350 try: 

351 # We augment the dataRef here with the tract, which is harmless for loading things 

352 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

353 # which do. 

354 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

355 dataId=calExpRef.dataId, 

356 tract=skyInfo.tractInfo.getId()) 

357 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

358 except Exception as e: 

359 self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

360 continue 

361 

362 if self.config.doApplySkyCorr: 362 ↛ 363line 362 didn't jump to line 363, because the condition on line 362 was never true

363 self.applySkyCorr(calExpRef, calExp) 

364 

365 calExpList.append(calExp) 

366 ccdIdList.append(ccdId) 

367 dataIdList.append(calExpRef.dataId) 

368 

369 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

370 

371 if any(exps.values()): 

372 dataRefList.append(tempExpRef) 

373 else: 

374 self.log.warn("Warp %s could not be created", tempExpRef.dataId) 

375 

376 if self.config.doWrite: 376 ↛ 322line 376 didn't jump to line 322, because the condition on line 376 was never false

377 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

378 if exposure is not None: 

379 self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType)) 

380 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

381 

382 return dataRefList 

383 

384 @pipeBase.timeMethod 

385 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

386 """Create a Warp from inputs 

387 

388 We iterate over the multiple calexps in a single exposure to construct 

389 the warp (previously called a coaddTempExp) of that exposure to the 

390 supplied tract/patch. 

391 

392 Pixels that receive no pixels are set to NAN; this is not correct 

393 (violates LSST algorithms group policy), but will be fixed up by 

394 interpolating after the coaddition. 

395 

396 @param calexpRefList: List of data references for calexps that (may) 

397 overlap the patch of interest 

398 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

399 information about the patch 

400 @param visitId: integer identifier for visit, for the table that will 

401 produce the CoaddPsf 

402 @return a pipeBase Struct containing: 

403 - exposures: a dictionary containing the warps requested: 

404 "direct": direct warp if config.makeDirect 

405 "psfMatched": PSF-matched warp if config.makePsfMatched 

406 """ 

407 warpTypeList = self.getWarpTypeList() 

408 

409 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

410 didSetMetadata = {warpType: False for warpType in warpTypeList} 

411 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

412 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

413 for warpType in warpTypeList} 

414 

415 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

416 if dataIdList is None: 416 ↛ 417line 416 didn't jump to line 417, because the condition on line 416 was never true

417 dataIdList = ccdIdList 

418 

419 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

420 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

421 calExpInd+1, len(calExpList), dataId) 

422 

423 try: 

424 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

425 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

426 makeDirect=self.config.makeDirect, 

427 makePsfMatched=self.config.makePsfMatched) 

428 except Exception as e: 

429 self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

430 continue 

431 try: 

432 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

433 for warpType in warpTypeList: 

434 exposure = warpedAndMatched.getDict()[warpType] 

435 if exposure is None: 

436 continue 

437 coaddTempExp = coaddTempExps[warpType] 

438 if didSetMetadata[warpType]: 

439 mimg = exposure.getMaskedImage() 

440 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

441 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

442 del mimg 

443 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

444 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

445 totGoodPix[warpType] += numGoodPix[warpType] 

446 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

447 dataId, numGoodPix[warpType], 

448 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

449 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

450 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

451 coaddTempExp.setFilterLabel(exposure.getFilterLabel()) 

452 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

453 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

454 coaddTempExp.setPsf(exposure.getPsf()) 

455 didSetMetadata[warpType] = True 

456 

457 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

458 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

459 

460 except Exception as e: 

461 self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e) 

462 continue 

463 

464 for warpType in warpTypeList: 

465 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

466 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

467 

468 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

469 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

470 if warpType == "direct": 

471 coaddTempExps[warpType].setPsf( 

472 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

473 self.config.coaddPsf.makeControl())) 

474 else: 

475 if not self.config.doWriteEmptyWarps: 475 ↛ 464line 475 didn't jump to line 464, because the condition on line 475 was never false

476 # No good pixels. Exposure still empty 

477 coaddTempExps[warpType] = None 

478 

479 result = pipeBase.Struct(exposures=coaddTempExps) 

480 return result 

481 

482 def getCalibratedExposure(self, dataRef, bgSubtracted): 

483 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

484 

485 @param[in] dataRef a sensor-level data reference 

486 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

487 calexp's background background model and add it to the calexp. 

488 @return calibrated exposure 

489 

490 @raises MissingExposureError If data for the exposure is not available. 

491 

492 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

493 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

494 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

495 retrieved from the processed exposure. When 

496 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

497 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

498 Otherwise, the astrometric calibration is taken from the processed 

499 exposure. 

500 """ 

501 try: 

502 exposure = dataRef.get(self.calexpType, immediate=True) 

503 except dafPersist.NoResults as e: 

504 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

505 

506 if not bgSubtracted: 506 ↛ 507line 506 didn't jump to line 507, because the condition on line 506 was never true

507 background = dataRef.get("calexpBackground", immediate=True) 

508 mi = exposure.getMaskedImage() 

509 mi += background.getImage() 

510 del mi 

511 

512 if self.config.doApplyExternalPhotoCalib: 512 ↛ 513line 512 didn't jump to line 513, because the condition on line 512 was never true

513 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

514 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

515 photoCalib = dataRef.get(source) 

516 exposure.setPhotoCalib(photoCalib) 

517 else: 

518 photoCalib = exposure.getPhotoCalib() 

519 

520 if self.config.doApplyExternalSkyWcs: 520 ↛ 521line 520 didn't jump to line 521, because the condition on line 520 was never true

521 source = f"{self.config.externalSkyWcsName}_wcs" 

522 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

523 skyWcs = dataRef.get(source) 

524 exposure.setWcs(skyWcs) 

525 

526 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

527 includeScaleUncertainty=self.config.includeCalibVar) 

528 exposure.maskedImage /= photoCalib.getCalibrationMean() 

529 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

530 # exposure.setCalib(afwImage.Calib(1.0)) 

531 return exposure 

532 

533 @staticmethod 

534 def _prepareEmptyExposure(skyInfo): 

535 """Produce an empty exposure for a given patch""" 

536 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

537 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

538 .getPlaneBitMask("NO_DATA"), numpy.inf) 

539 return exp 

540 

541 def getWarpTypeList(self): 

542 """Return list of requested warp types per the config. 

543 """ 

544 warpTypeList = [] 

545 if self.config.makeDirect: 545 ↛ 547line 545 didn't jump to line 547, because the condition on line 545 was never false

546 warpTypeList.append("direct") 

547 if self.config.makePsfMatched: 547 ↛ 549line 547 didn't jump to line 549, because the condition on line 547 was never false

548 warpTypeList.append("psfMatched") 

549 return warpTypeList 

550 

551 def applySkyCorr(self, dataRef, calexp): 

552 """Apply correction to the sky background level 

553 

554 Sky corrections can be generated with the 'skyCorrection.py' 

555 executable in pipe_drivers. Because the sky model used by that 

556 code extends over the entire focal plane, this can produce 

557 better sky subtraction. 

558 

559 The calexp is updated in-place. 

560 

561 Parameters 

562 ---------- 

563 dataRef : `lsst.daf.persistence.ButlerDataRef` 

564 Data reference for calexp. 

565 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

566 Calibrated exposure. 

567 """ 

568 bg = dataRef.get("skyCorr") 

569 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

570 if isinstance(calexp, afwImage.Exposure): 

571 calexp = calexp.getMaskedImage() 

572 calexp -= bg.getImage() 

573 

574 

575class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

576 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

577 defaultTemplates={"coaddName": "deep", 

578 "skyWcsName": "jointcal", 

579 "photoCalibName": "fgcmcal"}): 

580 calExpList = connectionTypes.Input( 

581 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

582 name="calexp", 

583 storageClass="ExposureF", 

584 dimensions=("instrument", "visit", "detector"), 

585 multiple=True, 

586 deferLoad=True, 

587 ) 

588 backgroundList = connectionTypes.Input( 

589 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

590 name="calexpBackground", 

591 storageClass="Background", 

592 dimensions=("instrument", "visit", "detector"), 

593 multiple=True, 

594 ) 

595 skyCorrList = connectionTypes.Input( 

596 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

597 name="skyCorr", 

598 storageClass="Background", 

599 dimensions=("instrument", "visit", "detector"), 

600 multiple=True, 

601 ) 

602 skyMap = connectionTypes.Input( 

603 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

604 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

605 storageClass="SkyMap", 

606 dimensions=("skymap",), 

607 ) 

608 externalSkyWcsTractCatalog = connectionTypes.Input( 

609 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

610 "id for the catalog id, sorted on id for fast lookup."), 

611 name="{skyWcsName}SkyWcsCatalog", 

612 storageClass="ExposureCatalog", 

613 dimensions=("instrument", "visit", "tract"), 

614 ) 

615 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

616 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

617 "These catalogs use the detector id for the catalog id, sorted on id for " 

618 "fast lookup."), 

619 name="{skyWcsName}SkyWcsCatalog", 

620 storageClass="ExposureCatalog", 

621 dimensions=("instrument", "visit"), 

622 ) 

623 externalPhotoCalibTractCatalog = connectionTypes.Input( 

624 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

625 "detector id for the catalog id, sorted on id for fast lookup."), 

626 name="{photoCalibName}PhotoCalibCatalog", 

627 storageClass="ExposureCatalog", 

628 dimensions=("instrument", "visit", "tract"), 

629 ) 

630 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

631 doc=("Per-visit photometric calibrations computed globally (with no tract " 

632 "information). These catalogs use the detector id for the catalog id, " 

633 "sorted on id for fast lookup."), 

634 name="{photoCalibName}PhotoCalibCatalog", 

635 storageClass="ExposureCatalog", 

636 dimensions=("instrument", "visit"), 

637 ) 

638 direct = connectionTypes.Output( 

639 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

640 "calexps onto the skyMap patch geometry."), 

641 name="{coaddName}Coadd_directWarp", 

642 storageClass="ExposureF", 

643 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

644 ) 

645 psfMatched = connectionTypes.Output( 

646 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

647 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

648 name="{coaddName}Coadd_psfMatchedWarp", 

649 storageClass="ExposureF", 

650 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

651 ) 

652 # TODO DM-28769, have selectImages subtask indicate which connections they need: 

653 wcsList = connectionTypes.Input( 

654 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

655 name="calexp.wcs", 

656 storageClass="Wcs", 

657 dimensions=("instrument", "visit", "detector"), 

658 multiple=True, 

659 ) 

660 bboxList = connectionTypes.Input( 

661 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

662 name="calexp.bbox", 

663 storageClass="Box2I", 

664 dimensions=("instrument", "visit", "detector"), 

665 multiple=True, 

666 ) 

667 srcList = connectionTypes.Input( 

668 doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability", 

669 name="src", 

670 storageClass="SourceCatalog", 

671 dimensions=("instrument", "visit", "detector"), 

672 multiple=True, 

673 ) 

674 psfList = connectionTypes.Input( 

675 doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing", 

676 name="calexp.psf", 

677 storageClass="Psf", 

678 dimensions=("instrument", "visit", "detector"), 

679 multiple=True, 

680 ) 

681 

682 def __init__(self, *, config=None): 

683 super().__init__(config=config) 

684 if config.bgSubtracted: 

685 self.inputs.remove("backgroundList") 

686 if not config.doApplySkyCorr: 

687 self.inputs.remove("skyCorrList") 

688 if config.doApplyExternalSkyWcs: 

689 if config.useGlobalExternalSkyWcs: 

690 self.inputs.remove("externalSkyWcsTractCatalog") 

691 else: 

692 self.inputs.remove("externalSkyWcsGlobalCatalog") 

693 else: 

694 self.inputs.remove("externalSkyWcsTractCatalog") 

695 self.inputs.remove("externalSkyWcsGlobalCatalog") 

696 if config.doApplyExternalPhotoCalib: 

697 if config.useGlobalExternalPhotoCalib: 

698 self.inputs.remove("externalPhotoCalibTractCatalog") 

699 else: 

700 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

701 else: 

702 self.inputs.remove("externalPhotoCalibTractCatalog") 

703 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

704 if not config.makeDirect: 

705 self.outputs.remove("direct") 

706 if not config.makePsfMatched: 

707 self.outputs.remove("psfMatched") 

708 # TODO DM-28769: add connection per selectImages connections 

709 # instead of removing if not PsfWcsSelectImagesTask here: 

710 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

711 self.inputs.remove("srcList") 

712 if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask: 

713 self.inputs.remove("psfList") 

714 

715 

716class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

717 pipelineConnections=MakeWarpConnections): 

718 

719 def validate(self): 

720 super().validate() 

721 

722 

723class MakeWarpTask(MakeCoaddTempExpTask): 

724 """Warp and optionally PSF-Match calexps onto an a common projection 

725 """ 

726 ConfigClass = MakeWarpConfig 

727 _DefaultName = "makeWarp" 

728 

729 @utils.inheritDoc(pipeBase.PipelineTask) 

730 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

731 """ 

732 Notes 

733 ---- 

734 Construct warps for requested warp type for single epoch 

735 

736 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

737 calexps. This method is analogous to `runDataRef`. 

738 """ 

739 # Read in all inputs. 

740 inputs = butlerQC.get(inputRefs) 

741 

742 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

743 # from the dictionary so we can pass it as kwargs later. 

744 skyMap = inputs.pop("skyMap") 

745 quantumDataId = butlerQC.quantum.dataId 

746 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

747 

748 # Construct list of input DataIds expected by `run` 

749 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

750 # Construct list of packed integer IDs expected by `run` 

751 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

752 

753 # Run the selector and filter out calexps that were not selected 

754 # primarily because they do not overlap the patch 

755 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

756 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

757 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

758 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

759 

760 # Read from disk only the selected calexps 

761 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']] 

762 

763 # Extract integer visitId requested by `run` 

764 visits = [dataId['visit'] for dataId in dataIdList] 

765 visitId = visits[0] 

766 

767 if self.config.doApplyExternalSkyWcs: 

768 if self.config.useGlobalExternalSkyWcs: 

769 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

770 else: 

771 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

772 else: 

773 externalSkyWcsCatalog = None 

774 

775 if self.config.doApplyExternalPhotoCalib: 

776 if self.config.useGlobalExternalPhotoCalib: 

777 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

778 else: 

779 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

780 else: 

781 externalPhotoCalibCatalog = None 

782 

783 self.prepareCalibratedExposures(**inputs, externalSkyWcsCatalog=externalSkyWcsCatalog, 

784 externalPhotoCalibCatalog=externalPhotoCalibCatalog) 

785 

786 results = self.run(**inputs, visitId=visitId, 

787 ccdIdList=[ccdIdList[i] for i in goodIndices], 

788 dataIdList=[dataIdList[i] for i in goodIndices], 

789 skyInfo=skyInfo) 

790 if self.config.makeDirect: 

791 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

792 if self.config.makePsfMatched: 

793 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

794 

795 def filterInputs(self, indices, inputs): 

796 """Return task inputs with their lists filtered by indices 

797 

798 Parameters 

799 ---------- 

800 indices : `list` of integers 

801 inputs : `dict` of `list` of input connections to be passed to run 

802 """ 

803 for key in inputs.keys(): 

804 # Only down-select on list inputs 

805 if isinstance(inputs[key], list): 

806 inputs[key] = [inputs[key][ind] for ind in indices] 

807 return inputs 

808 

809 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

810 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

811 **kwargs): 

812 """Calibrate and add backgrounds to input calExpList in place 

813 

814 Parameters 

815 ---------- 

816 calExpList : `list` of `lsst.afw.image.Exposure` 

817 Sequence of calexps to be modified in place 

818 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional 

819 Sequence of backgrounds to be added back in if bgSubtracted=False 

820 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional 

821 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

822 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

823 Exposure catalog with external skyWcs to be applied 

824 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

825 for the catalog id, sorted on id for fast lookup. 

826 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

827 Exposure catalog with external photoCalib to be applied 

828 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

829 id for the catalog id, sorted on id for fast lookup. 

830 """ 

831 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

832 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

833 

834 includeCalibVar = self.config.includeCalibVar 

835 

836 for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList): 

837 mi = calexp.maskedImage 

838 if not self.config.bgSubtracted: 

839 mi += background.getImage() 

840 

841 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None: 

842 detectorId = calexp.getInfo().getDetector().getId() 

843 

844 # Find the external photoCalib 

845 if externalPhotoCalibCatalog is not None: 

846 row = externalPhotoCalibCatalog.find(detectorId) 

847 if row is None: 

848 raise RuntimeError(f"Detector id {detectorId} not found in " 

849 f"externalPhotoCalibCatalog.") 

850 photoCalib = row.getPhotoCalib() 

851 if photoCalib is None: 

852 raise RuntimeError(f"Detector id {detectorId} has None for photoCalib " 

853 f"in externalPhotoCalibCatalog.") 

854 else: 

855 photoCalib = calexp.getPhotoCalib() 

856 

857 # Find and apply external skyWcs 

858 if externalSkyWcsCatalog is not None: 

859 row = externalSkyWcsCatalog.find(detectorId) 

860 if row is None: 

861 raise RuntimeError(f"Detector id {detectorId} not found in externalSkyWcsCatalog.") 

862 skyWcs = row.getWcs() 

863 if skyWcs is None: 

864 raise RuntimeError(f"Detector id {detectorId} has None for WCS " 

865 f" in externalSkyWcsCatalog.") 

866 calexp.setWcs(skyWcs) 

867 

868 # Calibrate the image 

869 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

870 includeScaleUncertainty=includeCalibVar) 

871 calexp.maskedImage /= photoCalib.getCalibrationMean() 

872 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

873 # exposure.setCalib(afwImage.Calib(1.0)) 

874 

875 # Apply skycorr 

876 if self.config.doApplySkyCorr: 

877 mi -= skyCorr.getImage()