Coverage for python/lsst/pipe/tasks/makeCoaddTempExp.py: 16%

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23import logging 

24 

25import lsst.pex.config as pexConfig 

26import lsst.daf.persistence as dafPersist 

27import lsst.afw.image as afwImage 

28import lsst.coadd.utils as coaddUtils 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as connectionTypes 

31import lsst.utils as utils 

32import lsst.geom 

33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

34from lsst.skymap import BaseSkyMap 

35from lsst.utils.timer import timeMethod 

36from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

37from .selectImages import PsfWcsSelectImagesTask 

38from .warpAndPsfMatch import WarpAndPsfMatchTask 

39from .coaddHelpers import groupPatchExposures, getGroupDataRef 

40from collections.abc import Iterable 

41 

42__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

43 

44log = logging.getLogger(__name__) 

45 

46 

47class MissingExposureError(Exception): 

48 """Raised when data cannot be retrieved for an exposure. 

49 When processing patches, sometimes one exposure is missing; this lets us 

50 distinguish bewteen that case, and other errors. 

51 """ 

52 pass 

53 

54 

55class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

56 """Config for MakeCoaddTempExpTask 

57 """ 

58 warpAndPsfMatch = pexConfig.ConfigurableField( 

59 target=WarpAndPsfMatchTask, 

60 doc="Task to warp and PSF-match calexp", 

61 ) 

62 doWrite = pexConfig.Field( 

63 doc="persist <coaddName>Coadd_<warpType>Warp", 

64 dtype=bool, 

65 default=True, 

66 ) 

67 bgSubtracted = pexConfig.Field( 

68 doc="Work with a background subtracted calexp?", 

69 dtype=bool, 

70 default=True, 

71 ) 

72 coaddPsf = pexConfig.ConfigField( 

73 doc="Configuration for CoaddPsf", 

74 dtype=CoaddPsfConfig, 

75 ) 

76 makeDirect = pexConfig.Field( 

77 doc="Make direct Warp/Coadds", 

78 dtype=bool, 

79 default=True, 

80 ) 

81 makePsfMatched = pexConfig.Field( 

82 doc="Make Psf-Matched Warp/Coadd?", 

83 dtype=bool, 

84 default=False, 

85 ) 

86 

87 doWriteEmptyWarps = pexConfig.Field( 

88 dtype=bool, 

89 default=False, 

90 doc="Write out warps even if they are empty" 

91 ) 

92 

93 hasFakes = pexConfig.Field( 

94 doc="Should be set to True if fake sources have been inserted into the input data.", 

95 dtype=bool, 

96 default=False, 

97 ) 

98 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

99 

100 doApplyFinalizedPsf = pexConfig.Field( 

101 doc="Whether to apply finalized psf models and aperture correction map.", 

102 dtype=bool, 

103 default=False, 

104 ) 

105 

106 def validate(self): 

107 CoaddBaseTask.ConfigClass.validate(self) 

108 if not self.makePsfMatched and not self.makeDirect: 

109 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

110 if self.doPsfMatch: 

111 # Backwards compatibility. 

112 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

113 self.makePsfMatched = True 

114 self.makeDirect = False 

115 

116 def setDefaults(self): 

117 CoaddBaseTask.ConfigClass.setDefaults(self) 

118 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

119 self.select.retarget(PsfWcsSelectImagesTask) 

120 

121## \addtogroup LSST_task_documentation 

122## \{ 

123## \page page_MakeCoaddTempExpTask MakeCoaddTempExpTask 

124## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

125## \copybrief MakeCoaddTempExpTask 

126## \} 

127 

128 

129class MakeCoaddTempExpTask(CoaddBaseTask): 

130 r"""! 

131 Warp and optionally PSF-Match calexps onto an a common projection. 

132 

133 @anchor MakeCoaddTempExpTask_ 

134 

135 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

136 

137 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

138 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

139 - @ref pipe_tasks_makeCoaddTempExp_IO 

140 - @ref pipe_tasks_makeCoaddTempExp_Config 

141 - @ref pipe_tasks_makeCoaddTempExp_Debug 

142 - @ref pipe_tasks_makeCoaddTempExp_Example 

143 

144 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

145 

146 Warp and optionally PSF-Match calexps onto a common projection, by 

147 performing the following operations: 

148 - Group calexps by visit/run 

149 - For each visit, generate a Warp by calling method @ref run. 

150 `run` loops over the visit's calexps calling 

151 @ref warpAndPsfMatch::WarpAndPsfMatchTask "WarpAndPsfMatchTask" 

152 on each visit 

153 

154 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

155 

156 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

157 

158 @copydoc \_\_init\_\_ 

159 

160 This task has one special keyword argument: passing reuse=True will cause 

161 the task to skip the creation of warps that are already present in the 

162 output repositories. 

163 

164 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

165 

166 This task is primarily designed to be run from the command line. 

167 

168 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

169 to process. 

170 

171 @copydoc run 

172 

173 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

174 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

175 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

176 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

177 

178 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

179 

180 See @ref MakeCoaddTempExpConfig and parameters inherited from 

181 @link coaddBase::CoaddBaseConfig CoaddBaseConfig @endlink 

182 

183 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

184 

185 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

186 @link ip::diffim::modelPsfMatch::ModelPsfMatchTask ModelPsfMatchTask @endlink 

187 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

188 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

189 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

190 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

191 

192 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

193 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

194 In general, for a set of input images, this config should equal the FWHM of the visit 

195 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

196 of the other config options offer a reasonable starting point. 

197 The following list presents the most common problems that arise from a misconfigured 

198 @link ip::diffim::modelPsfMatch::ModelPsfMatchTask ModelPsfMatchTask @endlink 

199 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

200 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

201 Problem: Explanation. *Solution* 

202 

203 *Troublshooting PSF-Matching Configuration:* 

204 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

205 For example:_ 

206 

207 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

208 

209 Note that increasing the kernel size also increases runtime. 

210 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

211 for matching kernel. _Provide the matcher with more data by either increasing 

212 the spatial sampling by decreasing the spatial cell size,_ 

213 

214 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

215 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

216 

217 _or increasing the padding around the Science PSF, for example:_ 

218 

219 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

220 

221 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

222 matching kernel dimensions, thus increasing the number of pixels available to fit 

223 after convolving the PSF with the matching kernel. 

224 Optionally, for debugging the effects of padding, the level of padding may be manually 

225 controlled by setting turning off the automatic padding and setting the number 

226 of pixels by which to pad the PSF: 

227 

228 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

229 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

230 

231 - Deconvolution: Matching a large PSF to a smaller PSF produces 

232 a telltale noise pattern which looks like ripples or a brain. 

233 _Increase the size of the requested model PSF. For example:_ 

234 

235 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

236 

237 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

238 _Increase the width of the Gaussian basis functions. For example:_ 

239 

240 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

241 # from default [0.7, 1.5, 3.0] 

242 

243 

244 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

245 

246 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

247 

248 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

249 

250 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

251 into the larger Data Release Processing. 

252 Set up by running: 

253 

254 setup ci_hsc 

255 cd $CI_HSC_DIR 

256 # if not built already: 

257 python $(which scons) # this will take a while 

258 

259 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

260 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

261 output respository with the desired SkyMap. The command, 

262 

263 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

264 --id patch=5,4 tract=0 filter=HSC-I \ 

265 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

266 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

267 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

268 makePsfMatched=True modelPsf.defaultFwhm=11 

269 

270 writes a direct and PSF-Matched Warp to 

271 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

272 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

273 respectively. 

274 

275 @note PSF-Matching in this particular dataset would benefit from adding 

276 `--configfile ./matchingConfig.py` to 

277 the command line arguments where `matchingConfig.py` is defined by: 

278 

279 echo " 

280 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

281 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

282 

283 

284 Add the option `--help` to see more options. 

285 """ 

286 ConfigClass = MakeCoaddTempExpConfig 

287 _DefaultName = "makeCoaddTempExp" 

288 

289 def __init__(self, reuse=False, **kwargs): 

290 CoaddBaseTask.__init__(self, **kwargs) 

291 self.reuse = reuse 

292 self.makeSubtask("warpAndPsfMatch") 

293 if self.config.hasFakes: 

294 self.calexpType = "fakes_calexp" 

295 else: 

296 self.calexpType = "calexp" 

297 

298 @timeMethod 

299 def runDataRef(self, patchRef, selectDataList=[]): 

300 """! 

301 Produce @<coaddName>Coadd_@<warpType>Warp images by warping and optionally PSF-matching. 

302 

303 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

304 plus the camera-specific filter key (e.g. "filter" or "band") 

305 @param[in] selectDataList list of @ref selectImages::SelectStruct "SelectStruct" 

306 to consider for selection 

307 @return: dataRefList: a list of data references for the new @<coaddName>Coadd_directWarps 

308 if direct or both warp types are requested and @<coaddName>Coadd_psfMatchedWarps 

309 if only psfMatched 

310 warps are requested. 

311 

312 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

313 

314 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

315 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

316 (assembleCoadd should determine zeropoint scaling without referring to it). 

317 """ 

318 skyInfo = self.getSkyInfo(patchRef) 

319 

320 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

321 if self.config.makePsfMatched and not self.config.makeDirect: 

322 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

323 else: 

324 primaryWarpDataset = self.getTempExpDatasetName("direct") 

325 

326 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

327 

328 if len(calExpRefList) == 0: 

329 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId) 

330 return None 

331 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

332 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

333 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

334 

335 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

336 primaryWarpDataset) 

337 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

338 

339 dataRefList = [] 

340 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

341 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

342 tempExpTuple, groupData.keys) 

343 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 

344 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

345 dataRefList.append(tempExpRef) 

346 continue 

347 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

348 

349 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

350 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

351 # of the visit in the list. 

352 try: 

353 visitId = int(tempExpRef.dataId["visit"]) 

354 except (KeyError, ValueError): 

355 visitId = i 

356 

357 calExpList = [] 

358 ccdIdList = [] 

359 dataIdList = [] 

360 

361 for calExpInd, calExpRef in enumerate(calexpRefList): 

362 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

363 calExpRef.dataId) 

364 try: 

365 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

366 except Exception: 

367 ccdId = calExpInd 

368 try: 

369 # We augment the dataRef here with the tract, which is harmless for loading things 

370 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

371 # which do. 

372 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

373 dataId=calExpRef.dataId, 

374 tract=skyInfo.tractInfo.getId()) 

375 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

376 except Exception as e: 

377 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

378 continue 

379 

380 if self.config.doApplySkyCorr: 

381 self.applySkyCorr(calExpRef, calExp) 

382 

383 calExpList.append(calExp) 

384 ccdIdList.append(ccdId) 

385 dataIdList.append(calExpRef.dataId) 

386 

387 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

388 

389 if any(exps.values()): 

390 dataRefList.append(tempExpRef) 

391 else: 

392 self.log.warning("Warp %s could not be created", tempExpRef.dataId) 

393 

394 if self.config.doWrite: 

395 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

396 if exposure is not None: 

397 self.log.info("Persisting %s", self.getTempExpDatasetName(warpType)) 

398 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

399 

400 return dataRefList 

401 

402 @timeMethod 

403 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

404 """Create a Warp from inputs 

405 

406 We iterate over the multiple calexps in a single exposure to construct 

407 the warp (previously called a coaddTempExp) of that exposure to the 

408 supplied tract/patch. 

409 

410 Pixels that receive no pixels are set to NAN; this is not correct 

411 (violates LSST algorithms group policy), but will be fixed up by 

412 interpolating after the coaddition. 

413 

414 @param calexpRefList: List of data references for calexps that (may) 

415 overlap the patch of interest 

416 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

417 information about the patch 

418 @param visitId: integer identifier for visit, for the table that will 

419 produce the CoaddPsf 

420 @return a pipeBase Struct containing: 

421 - exposures: a dictionary containing the warps requested: 

422 "direct": direct warp if config.makeDirect 

423 "psfMatched": PSF-matched warp if config.makePsfMatched 

424 """ 

425 warpTypeList = self.getWarpTypeList() 

426 

427 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

428 didSetMetadata = {warpType: False for warpType in warpTypeList} 

429 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

430 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

431 for warpType in warpTypeList} 

432 

433 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

434 if dataIdList is None: 

435 dataIdList = ccdIdList 

436 

437 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

438 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

439 calExpInd+1, len(calExpList), dataId) 

440 

441 try: 

442 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

443 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

444 makeDirect=self.config.makeDirect, 

445 makePsfMatched=self.config.makePsfMatched) 

446 except Exception as e: 

447 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

448 continue 

449 try: 

450 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

451 for warpType in warpTypeList: 

452 exposure = warpedAndMatched.getDict()[warpType] 

453 if exposure is None: 

454 continue 

455 coaddTempExp = coaddTempExps[warpType] 

456 if didSetMetadata[warpType]: 

457 mimg = exposure.getMaskedImage() 

458 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

459 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

460 del mimg 

461 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

462 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

463 totGoodPix[warpType] += numGoodPix[warpType] 

464 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

465 dataId, numGoodPix[warpType], 

466 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

467 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

468 coaddTempExp.info.id = exposure.info.id 

469 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

470 coaddTempExp.setFilterLabel(exposure.getFilterLabel()) 

471 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

472 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

473 coaddTempExp.setPsf(exposure.getPsf()) 

474 didSetMetadata[warpType] = True 

475 

476 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

477 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

478 

479 except Exception as e: 

480 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

481 continue 

482 

483 for warpType in warpTypeList: 

484 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

485 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

486 

487 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

488 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

489 if warpType == "direct": 

490 coaddTempExps[warpType].setPsf( 

491 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

492 self.config.coaddPsf.makeControl())) 

493 else: 

494 if not self.config.doWriteEmptyWarps: 

495 # No good pixels. Exposure still empty 

496 coaddTempExps[warpType] = None 

497 # NoWorkFound is unnecessary as the downstream tasks will 

498 # adjust the quantum accordingly, and it prevents gen2 

499 # MakeCoaddTempExp from continuing to loop over visits. 

500 

501 result = pipeBase.Struct(exposures=coaddTempExps) 

502 return result 

503 

504 def getCalibratedExposure(self, dataRef, bgSubtracted): 

505 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

506 

507 @param[in] dataRef a sensor-level data reference 

508 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

509 calexp's background background model and add it to the calexp. 

510 @return calibrated exposure 

511 

512 @raises MissingExposureError If data for the exposure is not available. 

513 

514 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

515 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

516 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

517 retrieved from the processed exposure. When 

518 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

519 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

520 Otherwise, the astrometric calibration is taken from the processed 

521 exposure. 

522 """ 

523 try: 

524 exposure = dataRef.get(self.calexpType, immediate=True) 

525 except dafPersist.NoResults as e: 

526 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

527 

528 if not bgSubtracted: 

529 background = dataRef.get("calexpBackground", immediate=True) 

530 mi = exposure.getMaskedImage() 

531 mi += background.getImage() 

532 del mi 

533 

534 if self.config.doApplyExternalPhotoCalib: 

535 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

536 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

537 photoCalib = dataRef.get(source) 

538 exposure.setPhotoCalib(photoCalib) 

539 else: 

540 photoCalib = exposure.getPhotoCalib() 

541 

542 if self.config.doApplyExternalSkyWcs: 

543 source = f"{self.config.externalSkyWcsName}_wcs" 

544 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

545 skyWcs = dataRef.get(source) 

546 exposure.setWcs(skyWcs) 

547 

548 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

549 includeScaleUncertainty=self.config.includeCalibVar) 

550 exposure.maskedImage /= photoCalib.getCalibrationMean() 

551 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

552 # exposure.setCalib(afwImage.Calib(1.0)) 

553 return exposure 

554 

555 @staticmethod 

556 def _prepareEmptyExposure(skyInfo): 

557 """Produce an empty exposure for a given patch""" 

558 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

559 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

560 .getPlaneBitMask("NO_DATA"), numpy.inf) 

561 return exp 

562 

563 def getWarpTypeList(self): 

564 """Return list of requested warp types per the config. 

565 """ 

566 warpTypeList = [] 

567 if self.config.makeDirect: 

568 warpTypeList.append("direct") 

569 if self.config.makePsfMatched: 

570 warpTypeList.append("psfMatched") 

571 return warpTypeList 

572 

573 def applySkyCorr(self, dataRef, calexp): 

574 """Apply correction to the sky background level 

575 

576 Sky corrections can be generated with the 'skyCorrection.py' 

577 executable in pipe_drivers. Because the sky model used by that 

578 code extends over the entire focal plane, this can produce 

579 better sky subtraction. 

580 

581 The calexp is updated in-place. 

582 

583 Parameters 

584 ---------- 

585 dataRef : `lsst.daf.persistence.ButlerDataRef` 

586 Data reference for calexp. 

587 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

588 Calibrated exposure. 

589 """ 

590 bg = dataRef.get("skyCorr") 

591 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

592 if isinstance(calexp, afwImage.Exposure): 

593 calexp = calexp.getMaskedImage() 

594 calexp -= bg.getImage() 

595 

596 

597class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

598 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

599 defaultTemplates={"coaddName": "deep", 

600 "skyWcsName": "jointcal", 

601 "photoCalibName": "fgcm", 

602 "calexpType": ""}): 

603 calExpList = connectionTypes.Input( 

604 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

605 name="{calexpType}calexp", 

606 storageClass="ExposureF", 

607 dimensions=("instrument", "visit", "detector"), 

608 multiple=True, 

609 deferLoad=True, 

610 ) 

611 backgroundList = connectionTypes.Input( 

612 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

613 name="calexpBackground", 

614 storageClass="Background", 

615 dimensions=("instrument", "visit", "detector"), 

616 multiple=True, 

617 ) 

618 skyCorrList = connectionTypes.Input( 

619 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

620 name="skyCorr", 

621 storageClass="Background", 

622 dimensions=("instrument", "visit", "detector"), 

623 multiple=True, 

624 ) 

625 skyMap = connectionTypes.Input( 

626 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

627 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

628 storageClass="SkyMap", 

629 dimensions=("skymap",), 

630 ) 

631 externalSkyWcsTractCatalog = connectionTypes.Input( 

632 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

633 "id for the catalog id, sorted on id for fast lookup."), 

634 name="{skyWcsName}SkyWcsCatalog", 

635 storageClass="ExposureCatalog", 

636 dimensions=("instrument", "visit", "tract"), 

637 ) 

638 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

639 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

640 "These catalogs use the detector id for the catalog id, sorted on id for " 

641 "fast lookup."), 

642 name="{skyWcsName}SkyWcsCatalog", 

643 storageClass="ExposureCatalog", 

644 dimensions=("instrument", "visit"), 

645 ) 

646 externalPhotoCalibTractCatalog = connectionTypes.Input( 

647 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

648 "detector id for the catalog id, sorted on id for fast lookup."), 

649 name="{photoCalibName}PhotoCalibCatalog", 

650 storageClass="ExposureCatalog", 

651 dimensions=("instrument", "visit", "tract"), 

652 ) 

653 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

654 doc=("Per-visit photometric calibrations computed globally (with no tract " 

655 "information). These catalogs use the detector id for the catalog id, " 

656 "sorted on id for fast lookup."), 

657 name="{photoCalibName}PhotoCalibCatalog", 

658 storageClass="ExposureCatalog", 

659 dimensions=("instrument", "visit"), 

660 ) 

661 finalizedPsfApCorrCatalog = connectionTypes.Input( 

662 doc=("Per-visit finalized psf models and aperture correction maps. " 

663 "These catalogs use the detector id for the catalog id, " 

664 "sorted on id for fast lookup."), 

665 name="finalized_psf_ap_corr_catalog", 

666 storageClass="ExposureCatalog", 

667 dimensions=("instrument", "visit"), 

668 ) 

669 direct = connectionTypes.Output( 

670 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

671 "calexps onto the skyMap patch geometry."), 

672 name="{coaddName}Coadd_directWarp", 

673 storageClass="ExposureF", 

674 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

675 ) 

676 psfMatched = connectionTypes.Output( 

677 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

678 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

679 name="{coaddName}Coadd_psfMatchedWarp", 

680 storageClass="ExposureF", 

681 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

682 ) 

683 # TODO DM-28769, have selectImages subtask indicate which connections they need: 

684 wcsList = connectionTypes.Input( 

685 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

686 name="{calexpType}calexp.wcs", 

687 storageClass="Wcs", 

688 dimensions=("instrument", "visit", "detector"), 

689 multiple=True, 

690 ) 

691 bboxList = connectionTypes.Input( 

692 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

693 name="{calexpType}calexp.bbox", 

694 storageClass="Box2I", 

695 dimensions=("instrument", "visit", "detector"), 

696 multiple=True, 

697 ) 

698 visitSummary = connectionTypes.Input( 

699 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask", 

700 name="{calexpType}visitSummary", 

701 storageClass="ExposureCatalog", 

702 dimensions=("instrument", "visit",), 

703 ) 

704 srcList = connectionTypes.Input( 

705 doc="Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability", 

706 name="src", 

707 storageClass="SourceCatalog", 

708 dimensions=("instrument", "visit", "detector"), 

709 multiple=True, 

710 ) 

711 

712 def __init__(self, *, config=None): 

713 super().__init__(config=config) 

714 if config.bgSubtracted: 

715 self.inputs.remove("backgroundList") 

716 if not config.doApplySkyCorr: 

717 self.inputs.remove("skyCorrList") 

718 if config.doApplyExternalSkyWcs: 

719 if config.useGlobalExternalSkyWcs: 

720 self.inputs.remove("externalSkyWcsTractCatalog") 

721 else: 

722 self.inputs.remove("externalSkyWcsGlobalCatalog") 

723 else: 

724 self.inputs.remove("externalSkyWcsTractCatalog") 

725 self.inputs.remove("externalSkyWcsGlobalCatalog") 

726 if config.doApplyExternalPhotoCalib: 

727 if config.useGlobalExternalPhotoCalib: 

728 self.inputs.remove("externalPhotoCalibTractCatalog") 

729 else: 

730 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

731 else: 

732 self.inputs.remove("externalPhotoCalibTractCatalog") 

733 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

734 if not config.doApplyFinalizedPsf: 

735 self.inputs.remove("finalizedPsfApCorrCatalog") 

736 if not config.makeDirect: 

737 self.outputs.remove("direct") 

738 if not config.makePsfMatched: 

739 self.outputs.remove("psfMatched") 

740 # TODO DM-28769: add connection per selectImages connections 

741 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

742 self.inputs.remove("visitSummary") 

743 self.inputs.remove("srcList") 

744 elif not config.select.doLegacyStarSelectionComputation: 

745 # Remove backwards-compatibility connections. 

746 self.inputs.remove("srcList") 

747 

748 

749class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

750 pipelineConnections=MakeWarpConnections): 

751 

752 def validate(self): 

753 super().validate() 

754 

755 

756class MakeWarpTask(MakeCoaddTempExpTask): 

757 """Warp and optionally PSF-Match calexps onto an a common projection 

758 """ 

759 ConfigClass = MakeWarpConfig 

760 _DefaultName = "makeWarp" 

761 

762 @utils.inheritDoc(pipeBase.PipelineTask) 

763 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

764 """ 

765 Notes 

766 ---- 

767 Construct warps for requested warp type for single epoch 

768 

769 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

770 calexps. This method is analogous to `runDataRef`. 

771 """ 

772 # Obtain the list of input detectors from calExpList. Sort them by 

773 # detector order (to ensure reproducibility). Then ensure all input 

774 # lists are in the same sorted detector order. 

775 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

776 detectorOrder.sort() 

777 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

778 

779 # Read in all inputs. 

780 inputs = butlerQC.get(inputRefs) 

781 

782 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

783 # from the dictionary so we can pass it as kwargs later. 

784 skyMap = inputs.pop("skyMap") 

785 quantumDataId = butlerQC.quantum.dataId 

786 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

787 

788 # Construct list of input DataIds expected by `run` 

789 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

790 # Construct list of packed integer IDs expected by `run` 

791 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

792 

793 # Run the selector and filter out calexps that were not selected 

794 # primarily because they do not overlap the patch 

795 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

796 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

797 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

798 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

799 

800 # Read from disk only the selected calexps 

801 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']] 

802 

803 # Extract integer visitId requested by `run` 

804 visits = [dataId['visit'] for dataId in dataIdList] 

805 visitId = visits[0] 

806 

807 if self.config.doApplyExternalSkyWcs: 

808 if self.config.useGlobalExternalSkyWcs: 

809 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

810 else: 

811 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

812 else: 

813 externalSkyWcsCatalog = None 

814 

815 if self.config.doApplyExternalPhotoCalib: 

816 if self.config.useGlobalExternalPhotoCalib: 

817 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

818 else: 

819 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

820 else: 

821 externalPhotoCalibCatalog = None 

822 

823 if self.config.doApplyFinalizedPsf: 

824 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog") 

825 else: 

826 finalizedPsfApCorrCatalog = None 

827 

828 completeIndices = self.prepareCalibratedExposures(**inputs, 

829 externalSkyWcsCatalog=externalSkyWcsCatalog, 

830 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

831 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

832 # Redo the input selection with inputs with complete wcs/photocalib info. 

833 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

834 

835 results = self.run(**inputs, visitId=visitId, 

836 ccdIdList=[ccdIdList[i] for i in goodIndices], 

837 dataIdList=[dataIdList[i] for i in goodIndices], 

838 skyInfo=skyInfo) 

839 if self.config.makeDirect and results.exposures["direct"] is not None: 

840 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

841 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

842 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

843 

844 def filterInputs(self, indices, inputs): 

845 """Return task inputs with their lists filtered by indices 

846 

847 Parameters 

848 ---------- 

849 indices : `list` of integers 

850 inputs : `dict` of `list` of input connections to be passed to run 

851 """ 

852 for key in inputs.keys(): 

853 # Only down-select on list inputs 

854 if isinstance(inputs[key], list): 

855 inputs[key] = [inputs[key][ind] for ind in indices] 

856 return inputs 

857 

858 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

859 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

860 finalizedPsfApCorrCatalog=None, 

861 **kwargs): 

862 """Calibrate and add backgrounds to input calExpList in place 

863 

864 Parameters 

865 ---------- 

866 calExpList : `list` of `lsst.afw.image.Exposure` 

867 Sequence of calexps to be modified in place 

868 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional 

869 Sequence of backgrounds to be added back in if bgSubtracted=False 

870 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional 

871 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

872 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

873 Exposure catalog with external skyWcs to be applied 

874 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

875 for the catalog id, sorted on id for fast lookup. 

876 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional