Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23import logging 

24 

25import lsst.pex.config as pexConfig 

26import lsst.daf.persistence as dafPersist 

27import lsst.afw.image as afwImage 

28import lsst.coadd.utils as coaddUtils 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as connectionTypes 

31import lsst.utils as utils 

32import lsst.geom 

33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

34from lsst.skymap import BaseSkyMap 

35from lsst.utils.timer import timeMethod 

36from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

37from .warpAndPsfMatch import WarpAndPsfMatchTask 

38from .coaddHelpers import groupPatchExposures, getGroupDataRef 

39from collections.abc import Iterable 

40 

41__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

42 

43log = logging.getLogger(__name__.partition(".")[2]) 

44 

45 

46class MissingExposureError(Exception): 

47 """Raised when data cannot be retrieved for an exposure. 

48 When processing patches, sometimes one exposure is missing; this lets us 

49 distinguish bewteen that case, and other errors. 

50 """ 

51 pass 

52 

53 

54class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

55 """Config for MakeCoaddTempExpTask 

56 """ 

57 warpAndPsfMatch = pexConfig.ConfigurableField( 

58 target=WarpAndPsfMatchTask, 

59 doc="Task to warp and PSF-match calexp", 

60 ) 

61 doWrite = pexConfig.Field( 

62 doc="persist <coaddName>Coadd_<warpType>Warp", 

63 dtype=bool, 

64 default=True, 

65 ) 

66 bgSubtracted = pexConfig.Field( 

67 doc="Work with a background subtracted calexp?", 

68 dtype=bool, 

69 default=True, 

70 ) 

71 coaddPsf = pexConfig.ConfigField( 

72 doc="Configuration for CoaddPsf", 

73 dtype=CoaddPsfConfig, 

74 ) 

75 makeDirect = pexConfig.Field( 

76 doc="Make direct Warp/Coadds", 

77 dtype=bool, 

78 default=True, 

79 ) 

80 makePsfMatched = pexConfig.Field( 

81 doc="Make Psf-Matched Warp/Coadd?", 

82 dtype=bool, 

83 default=False, 

84 ) 

85 

86 doWriteEmptyWarps = pexConfig.Field( 

87 dtype=bool, 

88 default=False, 

89 doc="Write out warps even if they are empty" 

90 ) 

91 

92 hasFakes = pexConfig.Field( 

93 doc="Should be set to True if fake sources have been inserted into the input data.", 

94 dtype=bool, 

95 default=False, 

96 ) 

97 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

98 

99 def validate(self): 

100 CoaddBaseTask.ConfigClass.validate(self) 

101 if not self.makePsfMatched and not self.makeDirect: 

102 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

103 if self.doPsfMatch: 

104 # Backwards compatibility. 

105 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

106 self.makePsfMatched = True 

107 self.makeDirect = False 

108 

109 def setDefaults(self): 

110 CoaddBaseTask.ConfigClass.setDefaults(self) 

111 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

112 

113## \addtogroup LSST_task_documentation 

114## \{ 

115## \page MakeCoaddTempExpTask 

116## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

117## \copybrief MakeCoaddTempExpTask 

118## \} 

119 

120 

121class MakeCoaddTempExpTask(CoaddBaseTask): 

122 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 

123 

124 @anchor MakeCoaddTempExpTask_ 

125 

126 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

127 

128 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

129 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

130 - @ref pipe_tasks_makeCoaddTempExp_IO 

131 - @ref pipe_tasks_makeCoaddTempExp_Config 

132 - @ref pipe_tasks_makeCoaddTempExp_Debug 

133 - @ref pipe_tasks_makeCoaddTempExp_Example 

134 

135 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

136 

137 Warp and optionally PSF-Match calexps onto a common projection, by 

138 performing the following operations: 

139 - Group calexps by visit/run 

140 - For each visit, generate a Warp by calling method @ref makeTempExp. 

141 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 

142 on each visit 

143 

144 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

145 

146 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

147 

148 @copydoc \_\_init\_\_ 

149 

150 This task has one special keyword argument: passing reuse=True will cause 

151 the task to skip the creation of warps that are already present in the 

152 output repositories. 

153 

154 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

155 

156 This task is primarily designed to be run from the command line. 

157 

158 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

159 to process. 

160 

161 @copydoc run 

162 

163 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

164 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

165 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

166 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

167 

168 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

169 

170 See @ref MakeCoaddTempExpConfig and parameters inherited from 

171 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 

172 

173 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

174 

175 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

176 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

177 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

178 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

179 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

180 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

181 

182 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

183 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

184 In general, for a set of input images, this config should equal the FWHM of the visit 

185 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

186 of the other config options offer a reasonable starting point. 

187 The following list presents the most common problems that arise from a misconfigured 

188 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

189 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

190 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

191 Problem: Explanation. *Solution* 

192 

193 *Troublshooting PSF-Matching Configuration:* 

194 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

195 For example:_ 

196 

197 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

198 

199 Note that increasing the kernel size also increases runtime. 

200 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

201 for matching kernel. _Provide the matcher with more data by either increasing 

202 the spatial sampling by decreasing the spatial cell size,_ 

203 

204 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

205 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

206 

207 _or increasing the padding around the Science PSF, for example:_ 

208 

209 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

210 

211 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

212 matching kernel dimensions, thus increasing the number of pixels available to fit 

213 after convolving the PSF with the matching kernel. 

214 Optionally, for debugging the effects of padding, the level of padding may be manually 

215 controlled by setting turning off the automatic padding and setting the number 

216 of pixels by which to pad the PSF: 

217 

218 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

219 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

220 

221 - Deconvolution: Matching a large PSF to a smaller PSF produces 

222 a telltale noise pattern which looks like ripples or a brain. 

223 _Increase the size of the requested model PSF. For example:_ 

224 

225 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

226 

227 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

228 _Increase the width of the Gaussian basis functions. For example:_ 

229 

230 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

231 # from default [0.7, 1.5, 3.0] 

232 

233 

234 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

235 

236 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

237 

238 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

239 

240 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

241 into the larger Data Release Processing. 

242 Set up by running: 

243 

244 setup ci_hsc 

245 cd $CI_HSC_DIR 

246 # if not built already: 

247 python $(which scons) # this will take a while 

248 

249 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

250 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

251 output respository with the desired SkyMap. The command, 

252 

253 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

254 --id patch=5,4 tract=0 filter=HSC-I \ 

255 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

256 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

257 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

258 makePsfMatched=True modelPsf.defaultFwhm=11 

259 

260 writes a direct and PSF-Matched Warp to 

261 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

262 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

263 respectively. 

264 

265 @note PSF-Matching in this particular dataset would benefit from adding 

266 `--configfile ./matchingConfig.py` to 

267 the command line arguments where `matchingConfig.py` is defined by: 

268 

269 echo " 

270 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

271 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

272 

273 

274 Add the option `--help` to see more options. 

275 """ 

276 ConfigClass = MakeCoaddTempExpConfig 

277 _DefaultName = "makeCoaddTempExp" 

278 

279 def __init__(self, reuse=False, **kwargs): 

280 CoaddBaseTask.__init__(self, **kwargs) 

281 self.reuse = reuse 

282 self.makeSubtask("warpAndPsfMatch") 

283 if self.config.hasFakes: 283 ↛ 284line 283 didn't jump to line 284, because the condition on line 283 was never true

284 self.calexpType = "fakes_calexp" 

285 else: 

286 self.calexpType = "calexp" 

287 

288 @timeMethod 

289 def runDataRef(self, patchRef, selectDataList=[]): 

290 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 

291 

292 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

293 plus the camera-specific filter key (e.g. "filter" or "band") 

294 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 

295 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 

296 warps are requested. 

297 

298 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

299 

300 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

301 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

302 (assembleCoadd should determine zeropoint scaling without referring to it). 

303 """ 

304 skyInfo = self.getSkyInfo(patchRef) 

305 

306 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

307 if self.config.makePsfMatched and not self.config.makeDirect: 307 ↛ 308line 307 didn't jump to line 308, because the condition on line 307 was never true

308 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

309 else: 

310 primaryWarpDataset = self.getTempExpDatasetName("direct") 

311 

312 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

313 

314 if len(calExpRefList) == 0: 314 ↛ 315line 314 didn't jump to line 315, because the condition on line 314 was never true

315 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId) 

316 return None 

317 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

318 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

319 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

320 

321 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

322 primaryWarpDataset) 

323 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

324 

325 dataRefList = [] 

326 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

327 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

328 tempExpTuple, groupData.keys) 

329 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 329 ↛ 330line 329 didn't jump to line 330, because the condition on line 329 was never true

330 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

331 dataRefList.append(tempExpRef) 

332 continue 

333 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

334 

335 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

336 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

337 # of the visit in the list. 

338 try: 

339 visitId = int(tempExpRef.dataId["visit"]) 

340 except (KeyError, ValueError): 

341 visitId = i 

342 

343 calExpList = [] 

344 ccdIdList = [] 

345 dataIdList = [] 

346 

347 for calExpInd, calExpRef in enumerate(calexpRefList): 

348 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

349 calExpRef.dataId) 

350 try: 

351 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

352 except Exception: 

353 ccdId = calExpInd 

354 try: 

355 # We augment the dataRef here with the tract, which is harmless for loading things 

356 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

357 # which do. 

358 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

359 dataId=calExpRef.dataId, 

360 tract=skyInfo.tractInfo.getId()) 

361 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

362 except Exception as e: 

363 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

364 continue 

365 

366 if self.config.doApplySkyCorr: 366 ↛ 367line 366 didn't jump to line 367, because the condition on line 366 was never true

367 self.applySkyCorr(calExpRef, calExp) 

368 

369 calExpList.append(calExp) 

370 ccdIdList.append(ccdId) 

371 dataIdList.append(calExpRef.dataId) 

372 

373 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

374 

375 if any(exps.values()): 

376 dataRefList.append(tempExpRef) 

377 else: 

378 self.log.warning("Warp %s could not be created", tempExpRef.dataId) 

379 

380 if self.config.doWrite: 380 ↛ 326line 380 didn't jump to line 326, because the condition on line 380 was never false

381 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

382 if exposure is not None: 

383 self.log.info("Persisting %s", self.getTempExpDatasetName(warpType)) 

384 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

385 

386 return dataRefList 

387 

388 @timeMethod 

389 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

390 """Create a Warp from inputs 

391 

392 We iterate over the multiple calexps in a single exposure to construct 

393 the warp (previously called a coaddTempExp) of that exposure to the 

394 supplied tract/patch. 

395 

396 Pixels that receive no pixels are set to NAN; this is not correct 

397 (violates LSST algorithms group policy), but will be fixed up by 

398 interpolating after the coaddition. 

399 

400 @param calexpRefList: List of data references for calexps that (may) 

401 overlap the patch of interest 

402 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

403 information about the patch 

404 @param visitId: integer identifier for visit, for the table that will 

405 produce the CoaddPsf 

406 @return a pipeBase Struct containing: 

407 - exposures: a dictionary containing the warps requested: 

408 "direct": direct warp if config.makeDirect 

409 "psfMatched": PSF-matched warp if config.makePsfMatched 

410 """ 

411 warpTypeList = self.getWarpTypeList() 

412 

413 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

414 didSetMetadata = {warpType: False for warpType in warpTypeList} 

415 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

416 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

417 for warpType in warpTypeList} 

418 

419 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

420 if dataIdList is None: 420 ↛ 421line 420 didn't jump to line 421, because the condition on line 420 was never true

421 dataIdList = ccdIdList 

422 

423 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

424 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

425 calExpInd+1, len(calExpList), dataId) 

426 

427 try: 

428 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

429 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

430 makeDirect=self.config.makeDirect, 

431 makePsfMatched=self.config.makePsfMatched) 

432 except Exception as e: 

433 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

434 continue 

435 try: 

436 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

437 for warpType in warpTypeList: 

438 exposure = warpedAndMatched.getDict()[warpType] 

439 if exposure is None: 

440 continue 

441 coaddTempExp = coaddTempExps[warpType] 

442 if didSetMetadata[warpType]: 

443 mimg = exposure.getMaskedImage() 

444 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

445 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

446 del mimg 

447 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

448 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

449 totGoodPix[warpType] += numGoodPix[warpType] 

450 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

451 dataId, numGoodPix[warpType], 

452 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

453 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

454 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

455 coaddTempExp.setFilterLabel(exposure.getFilterLabel()) 

456 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

457 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

458 coaddTempExp.setPsf(exposure.getPsf()) 

459 didSetMetadata[warpType] = True 

460 

461 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

462 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

463 

464 except Exception as e: 

465 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

466 continue 

467 

468 for warpType in warpTypeList: 

469 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

470 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

471 

472 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

473 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

474 if warpType == "direct": 

475 coaddTempExps[warpType].setPsf( 

476 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

477 self.config.coaddPsf.makeControl())) 

478 else: 

479 if not self.config.doWriteEmptyWarps: 479 ↛ 468line 479 didn't jump to line 468, because the condition on line 479 was never false

480 # No good pixels. Exposure still empty 

481 coaddTempExps[warpType] = None 

482 # NoWorkFound is unnecessary as the downstream tasks will 

483 # adjust the quantum accordingly, and it prevents gen2 

484 # MakeCoaddTempExp from continuing to loop over visits. 

485 

486 result = pipeBase.Struct(exposures=coaddTempExps) 

487 return result 

488 

489 def getCalibratedExposure(self, dataRef, bgSubtracted): 

490 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

491 

492 @param[in] dataRef a sensor-level data reference 

493 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

494 calexp's background background model and add it to the calexp. 

495 @return calibrated exposure 

496 

497 @raises MissingExposureError If data for the exposure is not available. 

498 

499 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

500 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

501 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

502 retrieved from the processed exposure. When 

503 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

504 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

505 Otherwise, the astrometric calibration is taken from the processed 

506 exposure. 

507 """ 

508 try: 

509 exposure = dataRef.get(self.calexpType, immediate=True) 

510 except dafPersist.NoResults as e: 

511 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

512 

513 if not bgSubtracted: 513 ↛ 514line 513 didn't jump to line 514, because the condition on line 513 was never true

514 background = dataRef.get("calexpBackground", immediate=True) 

515 mi = exposure.getMaskedImage() 

516 mi += background.getImage() 

517 del mi 

518 

519 if self.config.doApplyExternalPhotoCalib: 519 ↛ 520line 519 didn't jump to line 520, because the condition on line 519 was never true

520 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

521 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

522 photoCalib = dataRef.get(source) 

523 exposure.setPhotoCalib(photoCalib) 

524 else: 

525 photoCalib = exposure.getPhotoCalib() 

526 

527 if self.config.doApplyExternalSkyWcs: 527 ↛ 528line 527 didn't jump to line 528, because the condition on line 527 was never true

528 source = f"{self.config.externalSkyWcsName}_wcs" 

529 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

530 skyWcs = dataRef.get(source) 

531 exposure.setWcs(skyWcs) 

532 

533 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

534 includeScaleUncertainty=self.config.includeCalibVar) 

535 exposure.maskedImage /= photoCalib.getCalibrationMean() 

536 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

537 # exposure.setCalib(afwImage.Calib(1.0)) 

538 return exposure 

539 

540 @staticmethod 

541 def _prepareEmptyExposure(skyInfo): 

542 """Produce an empty exposure for a given patch""" 

543 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

544 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

545 .getPlaneBitMask("NO_DATA"), numpy.inf) 

546 return exp 

547 

548 def getWarpTypeList(self): 

549 """Return list of requested warp types per the config. 

550 """ 

551 warpTypeList = [] 

552 if self.config.makeDirect: 552 ↛ 554line 552 didn't jump to line 554, because the condition on line 552 was never false

553 warpTypeList.append("direct") 

554 if self.config.makePsfMatched: 554 ↛ 556line 554 didn't jump to line 556, because the condition on line 554 was never false

555 warpTypeList.append("psfMatched") 

556 return warpTypeList 

557 

558 def applySkyCorr(self, dataRef, calexp): 

559 """Apply correction to the sky background level 

560 

561 Sky corrections can be generated with the 'skyCorrection.py' 

562 executable in pipe_drivers. Because the sky model used by that 

563 code extends over the entire focal plane, this can produce 

564 better sky subtraction. 

565 

566 The calexp is updated in-place. 

567 

568 Parameters 

569 ---------- 

570 dataRef : `lsst.daf.persistence.ButlerDataRef` 

571 Data reference for calexp. 

572 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

573 Calibrated exposure. 

574 """ 

575 bg = dataRef.get("skyCorr") 

576 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

577 if isinstance(calexp, afwImage.Exposure): 

578 calexp = calexp.getMaskedImage() 

579 calexp -= bg.getImage() 

580 

581 

582class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

583 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

584 defaultTemplates={"coaddName": "deep", 

585 "skyWcsName": "jointcal", 

586 "photoCalibName": "fgcm", 

587 "calexpType": ""}): 

588 calExpList = connectionTypes.Input( 

589 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

590 name="{calexpType}calexp", 

591 storageClass="ExposureF", 

592 dimensions=("instrument", "visit", "detector"), 

593 multiple=True, 

594 deferLoad=True, 

595 ) 

596 backgroundList = connectionTypes.Input( 

597 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

598 name="calexpBackground", 

599 storageClass="Background", 

600 dimensions=("instrument", "visit", "detector"), 

601 multiple=True, 

602 ) 

603 skyCorrList = connectionTypes.Input( 

604 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

605 name="skyCorr", 

606 storageClass="Background", 

607 dimensions=("instrument", "visit", "detector"), 

608 multiple=True, 

609 ) 

610 skyMap = connectionTypes.Input( 

611 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

612 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

613 storageClass="SkyMap", 

614 dimensions=("skymap",), 

615 ) 

616 externalSkyWcsTractCatalog = connectionTypes.Input( 

617 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

618 "id for the catalog id, sorted on id for fast lookup."), 

619 name="{skyWcsName}SkyWcsCatalog", 

620 storageClass="ExposureCatalog", 

621 dimensions=("instrument", "visit", "tract"), 

622 ) 

623 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

624 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

625 "These catalogs use the detector id for the catalog id, sorted on id for " 

626 "fast lookup."), 

627 name="{skyWcsName}SkyWcsCatalog", 

628 storageClass="ExposureCatalog", 

629 dimensions=("instrument", "visit"), 

630 ) 

631 externalPhotoCalibTractCatalog = connectionTypes.Input( 

632 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

633 "detector id for the catalog id, sorted on id for fast lookup."), 

634 name="{photoCalibName}PhotoCalibCatalog", 

635 storageClass="ExposureCatalog", 

636 dimensions=("instrument", "visit", "tract"), 

637 ) 

638 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

639 doc=("Per-visit photometric calibrations computed globally (with no tract " 

640 "information). These catalogs use the detector id for the catalog id, " 

641 "sorted on id for fast lookup."), 

642 name="{photoCalibName}PhotoCalibCatalog", 

643 storageClass="ExposureCatalog", 

644 dimensions=("instrument", "visit"), 

645 ) 

646 direct = connectionTypes.Output( 

647 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

648 "calexps onto the skyMap patch geometry."), 

649 name="{coaddName}Coadd_directWarp", 

650 storageClass="ExposureF", 

651 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

652 ) 

653 psfMatched = connectionTypes.Output( 

654 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

655 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

656 name="{coaddName}Coadd_psfMatchedWarp", 

657 storageClass="ExposureF", 

658 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

659 ) 

660 # TODO DM-28769, have selectImages subtask indicate which connections they need: 

661 wcsList = connectionTypes.Input( 

662 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

663 name="{calexpType}calexp.wcs", 

664 storageClass="Wcs", 

665 dimensions=("instrument", "visit", "detector"), 

666 multiple=True, 

667 ) 

668 bboxList = connectionTypes.Input( 

669 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

670 name="{calexpType}calexp.bbox", 

671 storageClass="Box2I", 

672 dimensions=("instrument", "visit", "detector"), 

673 multiple=True, 

674 ) 

675 srcList = connectionTypes.Input( 

676 doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability", 

677 name="src", 

678 storageClass="SourceCatalog", 

679 dimensions=("instrument", "visit", "detector"), 

680 multiple=True, 

681 ) 

682 psfList = connectionTypes.Input( 

683 doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing", 

684 name="{calexpType}calexp.psf", 

685 storageClass="Psf", 

686 dimensions=("instrument", "visit", "detector"), 

687 multiple=True, 

688 ) 

689 

690 def __init__(self, *, config=None): 

691 super().__init__(config=config) 

692 if config.bgSubtracted: 

693 self.inputs.remove("backgroundList") 

694 if not config.doApplySkyCorr: 

695 self.inputs.remove("skyCorrList") 

696 if config.doApplyExternalSkyWcs: 

697 if config.useGlobalExternalSkyWcs: 

698 self.inputs.remove("externalSkyWcsTractCatalog") 

699 else: 

700 self.inputs.remove("externalSkyWcsGlobalCatalog") 

701 else: 

702 self.inputs.remove("externalSkyWcsTractCatalog") 

703 self.inputs.remove("externalSkyWcsGlobalCatalog") 

704 if config.doApplyExternalPhotoCalib: 

705 if config.useGlobalExternalPhotoCalib: 

706 self.inputs.remove("externalPhotoCalibTractCatalog") 

707 else: 

708 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

709 else: 

710 self.inputs.remove("externalPhotoCalibTractCatalog") 

711 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

712 if not config.makeDirect: 

713 self.outputs.remove("direct") 

714 if not config.makePsfMatched: 

715 self.outputs.remove("psfMatched") 

716 # TODO DM-28769: add connection per selectImages connections 

717 # instead of removing if not PsfWcsSelectImagesTask here: 

718 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

719 self.inputs.remove("srcList") 

720 if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask: 

721 self.inputs.remove("psfList") 

722 

723 

724class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

725 pipelineConnections=MakeWarpConnections): 

726 

727 def validate(self): 

728 super().validate() 

729 

730 

731class MakeWarpTask(MakeCoaddTempExpTask): 

732 """Warp and optionally PSF-Match calexps onto an a common projection 

733 """ 

734 ConfigClass = MakeWarpConfig 

735 _DefaultName = "makeWarp" 

736 

737 @utils.inheritDoc(pipeBase.PipelineTask) 

738 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

739 """ 

740 Notes 

741 ---- 

742 Construct warps for requested warp type for single epoch 

743 

744 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

745 calexps. This method is analogous to `runDataRef`. 

746 """ 

747 

748 # Ensure all input lists are in same detector order as the calExpList 

749 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

750 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

751 

752 # Read in all inputs. 

753 inputs = butlerQC.get(inputRefs) 

754 

755 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

756 # from the dictionary so we can pass it as kwargs later. 

757 skyMap = inputs.pop("skyMap") 

758 quantumDataId = butlerQC.quantum.dataId 

759 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

760 

761 # Construct list of input DataIds expected by `run` 

762 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

763 # Construct list of packed integer IDs expected by `run` 

764 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

765 

766 # Run the selector and filter out calexps that were not selected 

767 # primarily because they do not overlap the patch 

768 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

769 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

770 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

771 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

772 

773 # Read from disk only the selected calexps 

774 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']] 

775 

776 # Extract integer visitId requested by `run` 

777 visits = [dataId['visit'] for dataId in dataIdList] 

778 visitId = visits[0] 

779 

780 if self.config.doApplyExternalSkyWcs: 

781 if self.config.useGlobalExternalSkyWcs: 

782 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

783 else: 

784 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

785 else: 

786 externalSkyWcsCatalog = None 

787 

788 if self.config.doApplyExternalPhotoCalib: 

789 if self.config.useGlobalExternalPhotoCalib: 

790 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

791 else: 

792 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

793 else: 

794 externalPhotoCalibCatalog = None 

795 

796 completeIndices = self.prepareCalibratedExposures(**inputs, 

797 externalSkyWcsCatalog=externalSkyWcsCatalog, 

798 externalPhotoCalibCatalog=externalPhotoCalibCatalog) 

799 # Redo the input selection with inputs with complete wcs/photocalib info. 

800 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

801 

802 results = self.run(**inputs, visitId=visitId, 

803 ccdIdList=[ccdIdList[i] for i in goodIndices], 

804 dataIdList=[dataIdList[i] for i in goodIndices], 

805 skyInfo=skyInfo) 

806 if self.config.makeDirect and results.exposures["direct"] is not None: 

807 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

808 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

809 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

810 

811 def filterInputs(self, indices, inputs): 

812 """Return task inputs with their lists filtered by indices 

813 

814 Parameters 

815 ---------- 

816 indices : `list` of integers 

817 inputs : `dict` of `list` of input connections to be passed to run 

818 """ 

819 for key in inputs.keys(): 

820 # Only down-select on list inputs 

821 if isinstance(inputs[key], list): 

822 inputs[key] = [inputs[key][ind] for ind in indices] 

823 return inputs 

824 

825 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

826 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

827 **kwargs): 

828 """Calibrate and add backgrounds to input calExpList in place 

829 

830 Parameters 

831 ---------- 

832 calExpList : `list` of `lsst.afw.image.Exposure` 

833 Sequence of calexps to be modified in place 

834 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional 

835 Sequence of backgrounds to be added back in if bgSubtracted=False 

836 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional 

837 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

838 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

839 Exposure catalog with external skyWcs to be applied 

840 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

841 for the catalog id, sorted on id for fast lookup. 

842 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

843 Exposure catalog with external photoCalib to be applied 

844 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

845 id for the catalog id, sorted on id for fast lookup. 

846 

847 Returns 

848 ------- 

849 indices : `list` [`int`] 

850 Indices of calExpList and friends that have valid photoCalib/skyWcs 

851 """ 

852 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

853 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

854 

855 includeCalibVar = self.config.includeCalibVar 

856 

857 indices = [] 

858 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList, 

859 backgroundList, 

860 skyCorrList)): 

861 mi = calexp.maskedImage 

862 if not self.config.bgSubtracted: 

863 mi += background.getImage() 

864 

865 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None: 

866 detectorId = calexp.getInfo().getDetector().getId() 

867 

868 # Find the external photoCalib 

869 if externalPhotoCalibCatalog is not None: 

870 row = externalPhotoCalibCatalog.find(detectorId) 

871 if row is None: 

872 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

873 "and will not be used in the warp.", detectorId) 

874 continue 

875 photoCalib = row.getPhotoCalib() 

876 if photoCalib is None: 

877 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

878 "and will not be used in the warp.", detectorId) 

879 continue 

880 calexp.setPhotoCalib(photoCalib) 

881 else: 

882 photoCalib = calexp.getPhotoCalib() 

883 if photoCalib is None: 

884 self.log.warning("Detector id %s has None for photoCalib in the calexp " 

885 "and will not be used in the warp.", detectorId) 

886 continue 

887 

888 # Find and apply external skyWcs 

889 if externalSkyWcsCatalog is not None: 

890 row = externalSkyWcsCatalog.find(detectorId) 

891 if row is None: 

892 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

893 "and will not be used in the warp.", detectorId) 

894 continue 

895 skyWcs = row.getWcs() 

896 if skyWcs is None: 

897 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

898 "and will not be used in the warp.", detectorId) 

899 continue 

900 calexp.setWcs(skyWcs) 

901 else: 

902 skyWcs = calexp.getWcs() 

903 if skyWcs is None: 

904 self.log.warning("Detector id %s has None for skyWcs in the calexp " 

905 "and will not be used in the warp.", detectorId) 

906 continue 

907 

908 # Calibrate the image 

909 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

910 includeScaleUncertainty=includeCalibVar) 

911 calexp.maskedImage /= photoCalib.getCalibrationMean() 

912 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

913 # exposure.setCalib(afwImage.Calib(1.0)) 

914 

915 # Apply skycorr 

916 if self.config.doApplySkyCorr: 

917 mi -= skyCorr.getImage() 

918 

919 indices.append(index) 

920 

921 return indices 

922 

923 

924def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

925 """Reorder inputRefs per outputSortKeyOrder 

926 

927 Any inputRefs which are lists will be resorted per specified key e.g., 

928 'detector.' Only iterables will be reordered, and values can be of type 

929 `lsst.pipe.base.connections.DeferredDatasetRef` or 

930 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

931 Returned lists of refs have the same length as the outputSortKeyOrder. 

932 If an outputSortKey not in the inputRef, then it will be padded with None. 

933 If an inputRef contains an inputSortKey that is not in the 

934 outputSortKeyOrder it will be removed. 

935 

936 Parameters 

937 ---------- 

938 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

939 Input references to be reordered and padded. 

940 outputSortKeyOrder : iterable 

941 Iterable of values to be compared with inputRef's dataId[dataIdKey] 

942 dataIdKey : `str` 

943 dataIdKey in the dataRefs to compare with the outputSortKeyOrder. 

944 

945 Returns: 

946 -------- 

947 inputRefs: `lsst.pipe.base.connections.QuantizedConnection` 

948 Quantized Connection with sorted DatasetRef values sorted if iterable. 

949 """ 

950 for connectionName, refs in inputRefs: 

951 if isinstance(refs, Iterable): 

952 if hasattr(refs[0], "dataId"): 

953 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

954 else: 

955 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

956 if inputSortKeyOrder != outputSortKeyOrder: 

957 setattr(inputRefs, connectionName, 

958 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

959 return inputRefs