Coverage for python/lsst/pipe/tasks/makeCoaddTempExp.py: 16%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

367 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23import logging 

24 

25import lsst.pex.config as pexConfig 

26import lsst.daf.persistence as dafPersist 

27import lsst.afw.image as afwImage 

28import lsst.coadd.utils as coaddUtils 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as connectionTypes 

31import lsst.utils as utils 

32import lsst.geom 

33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

34from lsst.skymap import BaseSkyMap 

35from lsst.utils.timer import timeMethod 

36from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

37from .selectImages import PsfWcsSelectImagesTask 

38from .warpAndPsfMatch import WarpAndPsfMatchTask 

39from .coaddHelpers import groupPatchExposures, getGroupDataRef 

40from collections.abc import Iterable 

41 

42__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

43 

44log = logging.getLogger(__name__) 

45 

46 

47class MissingExposureError(Exception): 

48 """Raised when data cannot be retrieved for an exposure. 

49 When processing patches, sometimes one exposure is missing; this lets us 

50 distinguish bewteen that case, and other errors. 

51 """ 

52 pass 

53 

54 

55class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

56 """Config for MakeCoaddTempExpTask 

57 """ 

58 warpAndPsfMatch = pexConfig.ConfigurableField( 

59 target=WarpAndPsfMatchTask, 

60 doc="Task to warp and PSF-match calexp", 

61 ) 

62 doWrite = pexConfig.Field( 

63 doc="persist <coaddName>Coadd_<warpType>Warp", 

64 dtype=bool, 

65 default=True, 

66 ) 

67 bgSubtracted = pexConfig.Field( 

68 doc="Work with a background subtracted calexp?", 

69 dtype=bool, 

70 default=True, 

71 ) 

72 coaddPsf = pexConfig.ConfigField( 

73 doc="Configuration for CoaddPsf", 

74 dtype=CoaddPsfConfig, 

75 ) 

76 makeDirect = pexConfig.Field( 

77 doc="Make direct Warp/Coadds", 

78 dtype=bool, 

79 default=True, 

80 ) 

81 makePsfMatched = pexConfig.Field( 

82 doc="Make Psf-Matched Warp/Coadd?", 

83 dtype=bool, 

84 default=False, 

85 ) 

86 

87 doWriteEmptyWarps = pexConfig.Field( 

88 dtype=bool, 

89 default=False, 

90 doc="Write out warps even if they are empty" 

91 ) 

92 

93 hasFakes = pexConfig.Field( 

94 doc="Should be set to True if fake sources have been inserted into the input data.", 

95 dtype=bool, 

96 default=False, 

97 ) 

98 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

99 

100 doApplyFinalizedPsf = pexConfig.Field( 

101 doc="Whether to apply finalized psf models and aperture correction map.", 

102 dtype=bool, 

103 default=False, 

104 ) 

105 

106 def validate(self): 

107 CoaddBaseTask.ConfigClass.validate(self) 

108 if not self.makePsfMatched and not self.makeDirect: 

109 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

110 if self.doPsfMatch: 

111 # Backwards compatibility. 

112 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

113 self.makePsfMatched = True 

114 self.makeDirect = False 

115 

116 def setDefaults(self): 

117 CoaddBaseTask.ConfigClass.setDefaults(self) 

118 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

119 self.select.retarget(PsfWcsSelectImagesTask) 

120 

121## \addtogroup LSST_task_documentation 

122## \{ 

123## \page MakeCoaddTempExpTask 

124## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

125## \copybrief MakeCoaddTempExpTask 

126## \} 

127 

128 

129class MakeCoaddTempExpTask(CoaddBaseTask): 

130 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 

131 

132 @anchor MakeCoaddTempExpTask_ 

133 

134 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

135 

136 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

137 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

138 - @ref pipe_tasks_makeCoaddTempExp_IO 

139 - @ref pipe_tasks_makeCoaddTempExp_Config 

140 - @ref pipe_tasks_makeCoaddTempExp_Debug 

141 - @ref pipe_tasks_makeCoaddTempExp_Example 

142 

143 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

144 

145 Warp and optionally PSF-Match calexps onto a common projection, by 

146 performing the following operations: 

147 - Group calexps by visit/run 

148 - For each visit, generate a Warp by calling method @ref makeTempExp. 

149 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 

150 on each visit 

151 

152 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

153 

154 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

155 

156 @copydoc \_\_init\_\_ 

157 

158 This task has one special keyword argument: passing reuse=True will cause 

159 the task to skip the creation of warps that are already present in the 

160 output repositories. 

161 

162 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

163 

164 This task is primarily designed to be run from the command line. 

165 

166 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

167 to process. 

168 

169 @copydoc run 

170 

171 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

172 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

173 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

174 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

175 

176 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

177 

178 See @ref MakeCoaddTempExpConfig and parameters inherited from 

179 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 

180 

181 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

182 

183 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

184 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

185 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

186 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

187 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

188 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

189 

190 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

191 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

192 In general, for a set of input images, this config should equal the FWHM of the visit 

193 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

194 of the other config options offer a reasonable starting point. 

195 The following list presents the most common problems that arise from a misconfigured 

196 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

197 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

198 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

199 Problem: Explanation. *Solution* 

200 

201 *Troublshooting PSF-Matching Configuration:* 

202 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

203 For example:_ 

204 

205 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

206 

207 Note that increasing the kernel size also increases runtime. 

208 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

209 for matching kernel. _Provide the matcher with more data by either increasing 

210 the spatial sampling by decreasing the spatial cell size,_ 

211 

212 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

213 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

214 

215 _or increasing the padding around the Science PSF, for example:_ 

216 

217 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

218 

219 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

220 matching kernel dimensions, thus increasing the number of pixels available to fit 

221 after convolving the PSF with the matching kernel. 

222 Optionally, for debugging the effects of padding, the level of padding may be manually 

223 controlled by setting turning off the automatic padding and setting the number 

224 of pixels by which to pad the PSF: 

225 

226 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

227 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

228 

229 - Deconvolution: Matching a large PSF to a smaller PSF produces 

230 a telltale noise pattern which looks like ripples or a brain. 

231 _Increase the size of the requested model PSF. For example:_ 

232 

233 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

234 

235 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

236 _Increase the width of the Gaussian basis functions. For example:_ 

237 

238 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

239 # from default [0.7, 1.5, 3.0] 

240 

241 

242 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

243 

244 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

245 

246 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

247 

248 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

249 into the larger Data Release Processing. 

250 Set up by running: 

251 

252 setup ci_hsc 

253 cd $CI_HSC_DIR 

254 # if not built already: 

255 python $(which scons) # this will take a while 

256 

257 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

258 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

259 output respository with the desired SkyMap. The command, 

260 

261 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

262 --id patch=5,4 tract=0 filter=HSC-I \ 

263 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

264 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

265 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

266 makePsfMatched=True modelPsf.defaultFwhm=11 

267 

268 writes a direct and PSF-Matched Warp to 

269 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

270 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

271 respectively. 

272 

273 @note PSF-Matching in this particular dataset would benefit from adding 

274 `--configfile ./matchingConfig.py` to 

275 the command line arguments where `matchingConfig.py` is defined by: 

276 

277 echo " 

278 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

279 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

280 

281 

282 Add the option `--help` to see more options. 

283 """ 

284 ConfigClass = MakeCoaddTempExpConfig 

285 _DefaultName = "makeCoaddTempExp" 

286 

287 def __init__(self, reuse=False, **kwargs): 

288 CoaddBaseTask.__init__(self, **kwargs) 

289 self.reuse = reuse 

290 self.makeSubtask("warpAndPsfMatch") 

291 if self.config.hasFakes: 

292 self.calexpType = "fakes_calexp" 

293 else: 

294 self.calexpType = "calexp" 

295 

296 @timeMethod 

297 def runDataRef(self, patchRef, selectDataList=[]): 

298 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 

299 

300 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

301 plus the camera-specific filter key (e.g. "filter" or "band") 

302 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 

303 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 

304 warps are requested. 

305 

306 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

307 

308 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

309 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

310 (assembleCoadd should determine zeropoint scaling without referring to it). 

311 """ 

312 skyInfo = self.getSkyInfo(patchRef) 

313 

314 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

315 if self.config.makePsfMatched and not self.config.makeDirect: 

316 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

317 else: 

318 primaryWarpDataset = self.getTempExpDatasetName("direct") 

319 

320 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

321 

322 if len(calExpRefList) == 0: 

323 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId) 

324 return None 

325 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

326 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

327 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

328 

329 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

330 primaryWarpDataset) 

331 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

332 

333 dataRefList = [] 

334 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

335 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

336 tempExpTuple, groupData.keys) 

337 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 

338 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

339 dataRefList.append(tempExpRef) 

340 continue 

341 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

342 

343 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

344 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

345 # of the visit in the list. 

346 try: 

347 visitId = int(tempExpRef.dataId["visit"]) 

348 except (KeyError, ValueError): 

349 visitId = i 

350 

351 calExpList = [] 

352 ccdIdList = [] 

353 dataIdList = [] 

354 

355 for calExpInd, calExpRef in enumerate(calexpRefList): 

356 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

357 calExpRef.dataId) 

358 try: 

359 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

360 except Exception: 

361 ccdId = calExpInd 

362 try: 

363 # We augment the dataRef here with the tract, which is harmless for loading things 

364 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

365 # which do. 

366 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

367 dataId=calExpRef.dataId, 

368 tract=skyInfo.tractInfo.getId()) 

369 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

370 except Exception as e: 

371 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

372 continue 

373 

374 if self.config.doApplySkyCorr: 

375 self.applySkyCorr(calExpRef, calExp) 

376 

377 calExpList.append(calExp) 

378 ccdIdList.append(ccdId) 

379 dataIdList.append(calExpRef.dataId) 

380 

381 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

382 

383 if any(exps.values()): 

384 dataRefList.append(tempExpRef) 

385 else: 

386 self.log.warning("Warp %s could not be created", tempExpRef.dataId) 

387 

388 if self.config.doWrite: 

389 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

390 if exposure is not None: 

391 self.log.info("Persisting %s", self.getTempExpDatasetName(warpType)) 

392 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

393 

394 return dataRefList 

395 

396 @timeMethod 

397 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

398 """Create a Warp from inputs 

399 

400 We iterate over the multiple calexps in a single exposure to construct 

401 the warp (previously called a coaddTempExp) of that exposure to the 

402 supplied tract/patch. 

403 

404 Pixels that receive no pixels are set to NAN; this is not correct 

405 (violates LSST algorithms group policy), but will be fixed up by 

406 interpolating after the coaddition. 

407 

408 @param calexpRefList: List of data references for calexps that (may) 

409 overlap the patch of interest 

410 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

411 information about the patch 

412 @param visitId: integer identifier for visit, for the table that will 

413 produce the CoaddPsf 

414 @return a pipeBase Struct containing: 

415 - exposures: a dictionary containing the warps requested: 

416 "direct": direct warp if config.makeDirect 

417 "psfMatched": PSF-matched warp if config.makePsfMatched 

418 """ 

419 warpTypeList = self.getWarpTypeList() 

420 

421 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

422 didSetMetadata = {warpType: False for warpType in warpTypeList} 

423 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

424 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

425 for warpType in warpTypeList} 

426 

427 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

428 if dataIdList is None: 

429 dataIdList = ccdIdList 

430 

431 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

432 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

433 calExpInd+1, len(calExpList), dataId) 

434 

435 try: 

436 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

437 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

438 makeDirect=self.config.makeDirect, 

439 makePsfMatched=self.config.makePsfMatched) 

440 except Exception as e: 

441 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

442 continue 

443 try: 

444 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

445 for warpType in warpTypeList: 

446 exposure = warpedAndMatched.getDict()[warpType] 

447 if exposure is None: 

448 continue 

449 coaddTempExp = coaddTempExps[warpType] 

450 if didSetMetadata[warpType]: 

451 mimg = exposure.getMaskedImage() 

452 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

453 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

454 del mimg 

455 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

456 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

457 totGoodPix[warpType] += numGoodPix[warpType] 

458 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

459 dataId, numGoodPix[warpType], 

460 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

461 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

462 coaddTempExp.info.id = exposure.info.id 

463 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

464 coaddTempExp.setFilterLabel(exposure.getFilterLabel()) 

465 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

466 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

467 coaddTempExp.setPsf(exposure.getPsf()) 

468 didSetMetadata[warpType] = True 

469 

470 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

471 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

472 

473 except Exception as e: 

474 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

475 continue 

476 

477 for warpType in warpTypeList: 

478 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

479 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

480 

481 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

482 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

483 if warpType == "direct": 

484 coaddTempExps[warpType].setPsf( 

485 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

486 self.config.coaddPsf.makeControl())) 

487 else: 

488 if not self.config.doWriteEmptyWarps: 

489 # No good pixels. Exposure still empty 

490 coaddTempExps[warpType] = None 

491 # NoWorkFound is unnecessary as the downstream tasks will 

492 # adjust the quantum accordingly, and it prevents gen2 

493 # MakeCoaddTempExp from continuing to loop over visits. 

494 

495 result = pipeBase.Struct(exposures=coaddTempExps) 

496 return result 

497 

498 def getCalibratedExposure(self, dataRef, bgSubtracted): 

499 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

500 

501 @param[in] dataRef a sensor-level data reference 

502 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

503 calexp's background background model and add it to the calexp. 

504 @return calibrated exposure 

505 

506 @raises MissingExposureError If data for the exposure is not available. 

507 

508 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

509 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

510 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

511 retrieved from the processed exposure. When 

512 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

513 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

514 Otherwise, the astrometric calibration is taken from the processed 

515 exposure. 

516 """ 

517 try: 

518 exposure = dataRef.get(self.calexpType, immediate=True) 

519 except dafPersist.NoResults as e: 

520 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

521 

522 if not bgSubtracted: 

523 background = dataRef.get("calexpBackground", immediate=True) 

524 mi = exposure.getMaskedImage() 

525 mi += background.getImage() 

526 del mi 

527 

528 if self.config.doApplyExternalPhotoCalib: 

529 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

530 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

531 photoCalib = dataRef.get(source) 

532 exposure.setPhotoCalib(photoCalib) 

533 else: 

534 photoCalib = exposure.getPhotoCalib() 

535 

536 if self.config.doApplyExternalSkyWcs: 

537 source = f"{self.config.externalSkyWcsName}_wcs" 

538 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

539 skyWcs = dataRef.get(source) 

540 exposure.setWcs(skyWcs) 

541 

542 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

543 includeScaleUncertainty=self.config.includeCalibVar) 

544 exposure.maskedImage /= photoCalib.getCalibrationMean() 

545 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

546 # exposure.setCalib(afwImage.Calib(1.0)) 

547 return exposure 

548 

549 @staticmethod 

550 def _prepareEmptyExposure(skyInfo): 

551 """Produce an empty exposure for a given patch""" 

552 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

553 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

554 .getPlaneBitMask("NO_DATA"), numpy.inf) 

555 return exp 

556 

557 def getWarpTypeList(self): 

558 """Return list of requested warp types per the config. 

559 """ 

560 warpTypeList = [] 

561 if self.config.makeDirect: 

562 warpTypeList.append("direct") 

563 if self.config.makePsfMatched: 

564 warpTypeList.append("psfMatched") 

565 return warpTypeList 

566 

567 def applySkyCorr(self, dataRef, calexp): 

568 """Apply correction to the sky background level 

569 

570 Sky corrections can be generated with the 'skyCorrection.py' 

571 executable in pipe_drivers. Because the sky model used by that 

572 code extends over the entire focal plane, this can produce 

573 better sky subtraction. 

574 

575 The calexp is updated in-place. 

576 

577 Parameters 

578 ---------- 

579 dataRef : `lsst.daf.persistence.ButlerDataRef` 

580 Data reference for calexp. 

581 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

582 Calibrated exposure. 

583 """ 

584 bg = dataRef.get("skyCorr") 

585 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

586 if isinstance(calexp, afwImage.Exposure): 

587 calexp = calexp.getMaskedImage() 

588 calexp -= bg.getImage() 

589 

590 

591class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

592 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

593 defaultTemplates={"coaddName": "deep", 

594 "skyWcsName": "jointcal", 

595 "photoCalibName": "fgcm", 

596 "calexpType": ""}): 

597 calExpList = connectionTypes.Input( 

598 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

599 name="{calexpType}calexp", 

600 storageClass="ExposureF", 

601 dimensions=("instrument", "visit", "detector"), 

602 multiple=True, 

603 deferLoad=True, 

604 ) 

605 backgroundList = connectionTypes.Input( 

606 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

607 name="calexpBackground", 

608 storageClass="Background", 

609 dimensions=("instrument", "visit", "detector"), 

610 multiple=True, 

611 ) 

612 skyCorrList = connectionTypes.Input( 

613 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

614 name="skyCorr", 

615 storageClass="Background", 

616 dimensions=("instrument", "visit", "detector"), 

617 multiple=True, 

618 ) 

619 skyMap = connectionTypes.Input( 

620 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

621 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

622 storageClass="SkyMap", 

623 dimensions=("skymap",), 

624 ) 

625 externalSkyWcsTractCatalog = connectionTypes.Input( 

626 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

627 "id for the catalog id, sorted on id for fast lookup."), 

628 name="{skyWcsName}SkyWcsCatalog", 

629 storageClass="ExposureCatalog", 

630 dimensions=("instrument", "visit", "tract"), 

631 ) 

632 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

633 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

634 "These catalogs use the detector id for the catalog id, sorted on id for " 

635 "fast lookup."), 

636 name="{skyWcsName}SkyWcsCatalog", 

637 storageClass="ExposureCatalog", 

638 dimensions=("instrument", "visit"), 

639 ) 

640 externalPhotoCalibTractCatalog = connectionTypes.Input( 

641 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

642 "detector id for the catalog id, sorted on id for fast lookup."), 

643 name="{photoCalibName}PhotoCalibCatalog", 

644 storageClass="ExposureCatalog", 

645 dimensions=("instrument", "visit", "tract"), 

646 ) 

647 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

648 doc=("Per-visit photometric calibrations computed globally (with no tract " 

649 "information). These catalogs use the detector id for the catalog id, " 

650 "sorted on id for fast lookup."), 

651 name="{photoCalibName}PhotoCalibCatalog", 

652 storageClass="ExposureCatalog", 

653 dimensions=("instrument", "visit"), 

654 ) 

655 finalizedPsfApCorrCatalog = connectionTypes.Input( 

656 doc=("Per-visit finalized psf models and aperture correction maps. " 

657 "These catalogs use the detector id for the catalog id, " 

658 "sorted on id for fast lookup."), 

659 name="finalized_psf_ap_corr_catalog", 

660 storageClass="ExposureCatalog", 

661 dimensions=("instrument", "visit"), 

662 ) 

663 direct = connectionTypes.Output( 

664 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

665 "calexps onto the skyMap patch geometry."), 

666 name="{coaddName}Coadd_directWarp", 

667 storageClass="ExposureF", 

668 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

669 ) 

670 psfMatched = connectionTypes.Output( 

671 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

672 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

673 name="{coaddName}Coadd_psfMatchedWarp", 

674 storageClass="ExposureF", 

675 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

676 ) 

677 # TODO DM-28769, have selectImages subtask indicate which connections they need: 

678 wcsList = connectionTypes.Input( 

679 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

680 name="{calexpType}calexp.wcs", 

681 storageClass="Wcs", 

682 dimensions=("instrument", "visit", "detector"), 

683 multiple=True, 

684 ) 

685 bboxList = connectionTypes.Input( 

686 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

687 name="{calexpType}calexp.bbox", 

688 storageClass="Box2I", 

689 dimensions=("instrument", "visit", "detector"), 

690 multiple=True, 

691 ) 

692 visitSummary = connectionTypes.Input( 

693 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask", 

694 name="{calexpType}visitSummary", 

695 storageClass="ExposureCatalog", 

696 dimensions=("instrument", "visit",), 

697 ) 

698 srcList = connectionTypes.Input( 

699 doc="Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability", 

700 name="src", 

701 storageClass="SourceCatalog", 

702 dimensions=("instrument", "visit", "detector"), 

703 multiple=True, 

704 ) 

705 

706 def __init__(self, *, config=None): 

707 super().__init__(config=config) 

708 if config.bgSubtracted: 

709 self.inputs.remove("backgroundList") 

710 if not config.doApplySkyCorr: 

711 self.inputs.remove("skyCorrList") 

712 if config.doApplyExternalSkyWcs: 

713 if config.useGlobalExternalSkyWcs: 

714 self.inputs.remove("externalSkyWcsTractCatalog") 

715 else: 

716 self.inputs.remove("externalSkyWcsGlobalCatalog") 

717 else: 

718 self.inputs.remove("externalSkyWcsTractCatalog") 

719 self.inputs.remove("externalSkyWcsGlobalCatalog") 

720 if config.doApplyExternalPhotoCalib: 

721 if config.useGlobalExternalPhotoCalib: 

722 self.inputs.remove("externalPhotoCalibTractCatalog") 

723 else: 

724 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

725 else: 

726 self.inputs.remove("externalPhotoCalibTractCatalog") 

727 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

728 if not config.doApplyFinalizedPsf: 

729 self.inputs.remove("finalizedPsfApCorrCatalog") 

730 if not config.makeDirect: 

731 self.outputs.remove("direct") 

732 if not config.makePsfMatched: 

733 self.outputs.remove("psfMatched") 

734 # TODO DM-28769: add connection per selectImages connections 

735 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

736 self.inputs.remove("visitSummary") 

737 self.inputs.remove("srcList") 

738 elif not config.select.doLegacyStarSelectionComputation: 

739 # Remove backwards-compatibility connections. 

740 self.inputs.remove("srcList") 

741 

742 

743class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

744 pipelineConnections=MakeWarpConnections): 

745 

746 def validate(self): 

747 super().validate() 

748 

749 

750class MakeWarpTask(MakeCoaddTempExpTask): 

751 """Warp and optionally PSF-Match calexps onto an a common projection 

752 """ 

753 ConfigClass = MakeWarpConfig 

754 _DefaultName = "makeWarp" 

755 

756 @utils.inheritDoc(pipeBase.PipelineTask) 

757 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

758 """ 

759 Notes 

760 ---- 

761 Construct warps for requested warp type for single epoch 

762 

763 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

764 calexps. This method is analogous to `runDataRef`. 

765 """ 

766 # Obtain the list of input detectors from calExpList. Sort them by 

767 # detector order (to ensure reproducibility). Then ensure all input 

768 # lists are in the same sorted detector order. 

769 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

770 detectorOrder.sort() 

771 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

772 

773 # Read in all inputs. 

774 inputs = butlerQC.get(inputRefs) 

775 

776 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

777 # from the dictionary so we can pass it as kwargs later. 

778 skyMap = inputs.pop("skyMap") 

779 quantumDataId = butlerQC.quantum.dataId 

780 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

781 

782 # Construct list of input DataIds expected by `run` 

783 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

784 # Construct list of packed integer IDs expected by `run` 

785 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

786 

787 # Run the selector and filter out calexps that were not selected 

788 # primarily because they do not overlap the patch 

789 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

790 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

791 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

792 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

793 

794 # Read from disk only the selected calexps 

795 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']] 

796 

797 # Extract integer visitId requested by `run` 

798 visits = [dataId['visit'] for dataId in dataIdList] 

799 visitId = visits[0] 

800 

801 if self.config.doApplyExternalSkyWcs: 

802 if self.config.useGlobalExternalSkyWcs: 

803 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

804 else: 

805 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

806 else: 

807 externalSkyWcsCatalog = None 

808 

809 if self.config.doApplyExternalPhotoCalib: 

810 if self.config.useGlobalExternalPhotoCalib: 

811 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

812 else: 

813 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

814 else: 

815 externalPhotoCalibCatalog = None 

816 

817 if self.config.doApplyFinalizedPsf: 

818 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog") 

819 else: 

820 finalizedPsfApCorrCatalog = None 

821 

822 completeIndices = self.prepareCalibratedExposures(**inputs, 

823 externalSkyWcsCatalog=externalSkyWcsCatalog, 

824 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

825 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog) 

826 # Redo the input selection with inputs with complete wcs/photocalib info. 

827 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

828 

829 results = self.run(**inputs, visitId=visitId, 

830 ccdIdList=[ccdIdList[i] for i in goodIndices], 

831 dataIdList=[dataIdList[i] for i in goodIndices], 

832 skyInfo=skyInfo) 

833 if self.config.makeDirect and results.exposures["direct"] is not None: 

834 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

835 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

836 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

837 

838 def filterInputs(self, indices, inputs): 

839 """Return task inputs with their lists filtered by indices 

840 

841 Parameters 

842 ---------- 

843 indices : `list` of integers 

844 inputs : `dict` of `list` of input connections to be passed to run 

845 """ 

846 for key in inputs.keys(): 

847 # Only down-select on list inputs 

848 if isinstance(inputs[key], list): 

849 inputs[key] = [inputs[key][ind] for ind in indices] 

850 return inputs 

851 

852 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

853 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

854 finalizedPsfApCorrCatalog=None, 

855 **kwargs): 

856 """Calibrate and add backgrounds to input calExpList in place 

857 

858 Parameters 

859 ---------- 

860 calExpList : `list` of `lsst.afw.image.Exposure` 

861 Sequence of calexps to be modified in place 

862 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional 

863 Sequence of backgrounds to be added back in if bgSubtracted=False 

864 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional 

865 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

866 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

867 Exposure catalog with external skyWcs to be applied 

868 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

869 for the catalog id, sorted on id for fast lookup. 

870 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

871 Exposure catalog with external photoCalib to be applied 

872 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

873 id for the catalog id, sorted on id for fast lookup. 

874 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

875 Exposure catalog with finalized psf models and aperture correction 

876 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses 

877 the detector id for the catalog id, sorted on id for fast lookup. 

878 

879 Returns 

880 ------- 

881 indices : `list` [`int`] 

882 Indices of calExpList and friends that have valid photoCalib/skyWcs 

883 """ 

884 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

885 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

886 

887 includeCalibVar = self.config.includeCalibVar 

888 

889 indices = [] 

890 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList, 

891 backgroundList, 

892 skyCorrList)): 

893 if not self.config.bgSubtracted: 

894 calexp.maskedImage += background.getImage() 

895 

896 detectorId = calexp.getInfo().getDetector().getId() 

897 

898 # Find the external photoCalib 

899 if externalPhotoCalibCatalog is not None: 

900 row = externalPhotoCalibCatalog.find(detectorId) 

901 if row is None: 

902 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

903 "and will not be used in the warp.", detectorId) 

904 continue 

905 photoCalib = row.getPhotoCalib() 

906 if photoCalib is None: 

907 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

908 "and will not be used in the warp.", detectorId) 

909 continue 

910 calexp.setPhotoCalib(photoCalib) 

911 else: 

912 photoCalib = calexp.getPhotoCalib() 

913 if photoCalib is None: 

914 self.log.warning("Detector id %s has None for photoCalib in the calexp " 

915 "and will not be used in the warp.", detectorId) 

916 continue 

917 

918 # Find and apply external skyWcs 

919 if externalSkyWcsCatalog is not None: 

920 row = externalSkyWcsCatalog.find(detectorId) 

921 if row is None: 

922 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

923 "and will not be used in the warp.", detectorId) 

924 continue 

925 skyWcs = row.getWcs() 

926 if skyWcs is None: 

927 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

928 "and will not be used in the warp.", detectorId) 

929 continue 

930 calexp.setWcs(skyWcs) 

931 else: 

932 skyWcs = calexp.getWcs() 

933 if skyWcs is None: 

934 self.log.warning("Detector id %s has None for skyWcs in the calexp " 

935 "and will not be used in the warp.", detectorId) 

936 continue 

937 

938 # Find and apply finalized psf and aperture correction 

939 if finalizedPsfApCorrCatalog is not None: 

940 row = finalizedPsfApCorrCatalog.find(detectorId) 

941 if row is None: 

942 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog " 

943 "and will not be used in the warp.", detectorId) 

944 continue 

945 psf = row.getPsf() 

946 if psf is None: 

947 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog " 

948 "and will not be used in the warp.", detectorId) 

949 continue 

950 calexp.setPsf(psf) 

951 apCorrMap = row.getApCorrMap() 

952 if apCorrMap is None: 

953 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog " 

954 "and will not be used in the warp.", detectorId) 

955 continue 

956 calexp.setApCorrMap(apCorrMap) 

957 

958 # Calibrate the image 

959 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

960 includeScaleUncertainty=includeCalibVar) 

961 calexp.maskedImage /= photoCalib.getCalibrationMean() 

962 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

963 # exposure.setCalib(afwImage.Calib(1.0)) 

964 

965 # Apply skycorr 

966 if self.config.doApplySkyCorr: 

967 calexp.maskedImage -= skyCorr.getImage() 

968 

969 indices.append(index) 

970 

971 return indices 

972 

973 

974def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

975 """Reorder inputRefs per outputSortKeyOrder 

976 

977 Any inputRefs which are lists will be resorted per specified key e.g., 

978 'detector.' Only iterables will be reordered, and values can be of type 

979 `lsst.pipe.base.connections.DeferredDatasetRef` or 

980 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

981 Returned lists of refs have the same length as the outputSortKeyOrder. 

982 If an outputSortKey not in the inputRef, then it will be padded with None. 

983 If an inputRef contains an inputSortKey that is not in the 

984 outputSortKeyOrder it will be removed. 

985 

986 Parameters 

987 ---------- 

988 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

989 Input references to be reordered and padded. 

990 outputSortKeyOrder : iterable 

991 Iterable of values to be compared with inputRef's dataId[dataIdKey] 

992 dataIdKey : `str` 

993 dataIdKey in the dataRefs to compare with the outputSortKeyOrder. 

994 

995 Returns: 

996 -------- 

997 inputRefs: `lsst.pipe.base.connections.QuantizedConnection` 

998 Quantized Connection with sorted DatasetRef values sorted if iterable. 

999 """ 

1000 for connectionName, refs in inputRefs: 

1001 if isinstance(refs, Iterable): 

1002 if hasattr(refs[0], "dataId"): 

1003 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

1004 else: 

1005 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

1006 if inputSortKeyOrder != outputSortKeyOrder: 

1007 setattr(inputRefs, connectionName, 

1008 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

1009 return inputRefs