Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23 

24import lsst.pex.config as pexConfig 

25import lsst.daf.persistence as dafPersist 

26import lsst.afw.image as afwImage 

27import lsst.coadd.utils as coaddUtils 

28import lsst.pipe.base as pipeBase 

29import lsst.pipe.base.connectionTypes as connectionTypes 

30import lsst.log as log 

31import lsst.utils as utils 

32import lsst.geom 

33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

34from lsst.skymap import BaseSkyMap 

35from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

36from .warpAndPsfMatch import WarpAndPsfMatchTask 

37from .coaddHelpers import groupPatchExposures, getGroupDataRef 

38from collections.abc import Iterable 

39 

40__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

41 

42 

43class MissingExposureError(Exception): 

44 """Raised when data cannot be retrieved for an exposure. 

45 When processing patches, sometimes one exposure is missing; this lets us 

46 distinguish bewteen that case, and other errors. 

47 """ 

48 pass 

49 

50 

51class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

52 """Config for MakeCoaddTempExpTask 

53 """ 

54 warpAndPsfMatch = pexConfig.ConfigurableField( 

55 target=WarpAndPsfMatchTask, 

56 doc="Task to warp and PSF-match calexp", 

57 ) 

58 doWrite = pexConfig.Field( 

59 doc="persist <coaddName>Coadd_<warpType>Warp", 

60 dtype=bool, 

61 default=True, 

62 ) 

63 bgSubtracted = pexConfig.Field( 

64 doc="Work with a background subtracted calexp?", 

65 dtype=bool, 

66 default=True, 

67 ) 

68 coaddPsf = pexConfig.ConfigField( 

69 doc="Configuration for CoaddPsf", 

70 dtype=CoaddPsfConfig, 

71 ) 

72 makeDirect = pexConfig.Field( 

73 doc="Make direct Warp/Coadds", 

74 dtype=bool, 

75 default=True, 

76 ) 

77 makePsfMatched = pexConfig.Field( 

78 doc="Make Psf-Matched Warp/Coadd?", 

79 dtype=bool, 

80 default=False, 

81 ) 

82 

83 doWriteEmptyWarps = pexConfig.Field( 

84 dtype=bool, 

85 default=False, 

86 doc="Write out warps even if they are empty" 

87 ) 

88 

89 hasFakes = pexConfig.Field( 

90 doc="Should be set to True if fake sources have been inserted into the input data.", 

91 dtype=bool, 

92 default=False, 

93 ) 

94 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

95 

96 def validate(self): 

97 CoaddBaseTask.ConfigClass.validate(self) 

98 if not self.makePsfMatched and not self.makeDirect: 

99 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

100 if self.doPsfMatch: 

101 # Backwards compatibility. 

102 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

103 self.makePsfMatched = True 

104 self.makeDirect = False 

105 

106 def setDefaults(self): 

107 CoaddBaseTask.ConfigClass.setDefaults(self) 

108 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

109 

110## \addtogroup LSST_task_documentation 

111## \{ 

112## \page MakeCoaddTempExpTask 

113## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

114## \copybrief MakeCoaddTempExpTask 

115## \} 

116 

117 

118class MakeCoaddTempExpTask(CoaddBaseTask): 

119 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 

120 

121 @anchor MakeCoaddTempExpTask_ 

122 

123 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

124 

125 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

126 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

127 - @ref pipe_tasks_makeCoaddTempExp_IO 

128 - @ref pipe_tasks_makeCoaddTempExp_Config 

129 - @ref pipe_tasks_makeCoaddTempExp_Debug 

130 - @ref pipe_tasks_makeCoaddTempExp_Example 

131 

132 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

133 

134 Warp and optionally PSF-Match calexps onto a common projection, by 

135 performing the following operations: 

136 - Group calexps by visit/run 

137 - For each visit, generate a Warp by calling method @ref makeTempExp. 

138 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 

139 on each visit 

140 

141 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

142 

143 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

144 

145 @copydoc \_\_init\_\_ 

146 

147 This task has one special keyword argument: passing reuse=True will cause 

148 the task to skip the creation of warps that are already present in the 

149 output repositories. 

150 

151 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

152 

153 This task is primarily designed to be run from the command line. 

154 

155 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

156 to process. 

157 

158 @copydoc run 

159 

160 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

161 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

162 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

163 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

164 

165 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

166 

167 See @ref MakeCoaddTempExpConfig and parameters inherited from 

168 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 

169 

170 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

171 

172 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

173 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

174 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

175 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

176 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

177 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

178 

179 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

180 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

181 In general, for a set of input images, this config should equal the FWHM of the visit 

182 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

183 of the other config options offer a reasonable starting point. 

184 The following list presents the most common problems that arise from a misconfigured 

185 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

186 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

187 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

188 Problem: Explanation. *Solution* 

189 

190 *Troublshooting PSF-Matching Configuration:* 

191 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

192 For example:_ 

193 

194 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

195 

196 Note that increasing the kernel size also increases runtime. 

197 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

198 for matching kernel. _Provide the matcher with more data by either increasing 

199 the spatial sampling by decreasing the spatial cell size,_ 

200 

201 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

202 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

203 

204 _or increasing the padding around the Science PSF, for example:_ 

205 

206 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

207 

208 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

209 matching kernel dimensions, thus increasing the number of pixels available to fit 

210 after convolving the PSF with the matching kernel. 

211 Optionally, for debugging the effects of padding, the level of padding may be manually 

212 controlled by setting turning off the automatic padding and setting the number 

213 of pixels by which to pad the PSF: 

214 

215 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

216 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

217 

218 - Deconvolution: Matching a large PSF to a smaller PSF produces 

219 a telltale noise pattern which looks like ripples or a brain. 

220 _Increase the size of the requested model PSF. For example:_ 

221 

222 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

223 

224 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

225 _Increase the width of the Gaussian basis functions. For example:_ 

226 

227 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

228 # from default [0.7, 1.5, 3.0] 

229 

230 

231 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

232 

233 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

234 

235 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

236 

237 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

238 into the larger Data Release Processing. 

239 Set up by running: 

240 

241 setup ci_hsc 

242 cd $CI_HSC_DIR 

243 # if not built already: 

244 python $(which scons) # this will take a while 

245 

246 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

247 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

248 output respository with the desired SkyMap. The command, 

249 

250 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

251 --id patch=5,4 tract=0 filter=HSC-I \ 

252 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

253 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

254 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

255 makePsfMatched=True modelPsf.defaultFwhm=11 

256 

257 writes a direct and PSF-Matched Warp to 

258 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

259 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

260 respectively. 

261 

262 @note PSF-Matching in this particular dataset would benefit from adding 

263 `--configfile ./matchingConfig.py` to 

264 the command line arguments where `matchingConfig.py` is defined by: 

265 

266 echo " 

267 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

268 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

269 

270 

271 Add the option `--help` to see more options. 

272 """ 

273 ConfigClass = MakeCoaddTempExpConfig 

274 _DefaultName = "makeCoaddTempExp" 

275 

276 def __init__(self, reuse=False, **kwargs): 

277 CoaddBaseTask.__init__(self, **kwargs) 

278 self.reuse = reuse 

279 self.makeSubtask("warpAndPsfMatch") 

280 if self.config.hasFakes: 

281 self.calexpType = "fakes_calexp" 

282 else: 

283 self.calexpType = "calexp" 

284 

285 @pipeBase.timeMethod 

286 def runDataRef(self, patchRef, selectDataList=[]): 

287 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 

288 

289 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

290 plus the camera-specific filter key (e.g. "filter" or "band") 

291 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 

292 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 

293 warps are requested. 

294 

295 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

296 

297 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

298 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

299 (assembleCoadd should determine zeropoint scaling without referring to it). 

300 """ 

301 skyInfo = self.getSkyInfo(patchRef) 

302 

303 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

304 if self.config.makePsfMatched and not self.config.makeDirect: 

305 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

306 else: 

307 primaryWarpDataset = self.getTempExpDatasetName("direct") 

308 

309 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

310 

311 if len(calExpRefList) == 0: 

312 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId) 

313 return None 

314 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

315 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

316 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

317 

318 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

319 primaryWarpDataset) 

320 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

321 

322 dataRefList = [] 

323 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

324 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

325 tempExpTuple, groupData.keys) 

326 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 

327 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

328 dataRefList.append(tempExpRef) 

329 continue 

330 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

331 

332 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

333 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

334 # of the visit in the list. 

335 try: 

336 visitId = int(tempExpRef.dataId["visit"]) 

337 except (KeyError, ValueError): 

338 visitId = i 

339 

340 calExpList = [] 

341 ccdIdList = [] 

342 dataIdList = [] 

343 

344 for calExpInd, calExpRef in enumerate(calexpRefList): 

345 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

346 calExpRef.dataId) 

347 try: 

348 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

349 except Exception: 

350 ccdId = calExpInd 

351 try: 

352 # We augment the dataRef here with the tract, which is harmless for loading things 

353 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

354 # which do. 

355 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

356 dataId=calExpRef.dataId, 

357 tract=skyInfo.tractInfo.getId()) 

358 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

359 except Exception as e: 

360 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

361 continue 

362 

363 if self.config.doApplySkyCorr: 

364 self.applySkyCorr(calExpRef, calExp) 

365 

366 calExpList.append(calExp) 

367 ccdIdList.append(ccdId) 

368 dataIdList.append(calExpRef.dataId) 

369 

370 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

371 

372 if any(exps.values()): 

373 dataRefList.append(tempExpRef) 

374 else: 

375 self.log.warning("Warp %s could not be created", tempExpRef.dataId) 

376 

377 if self.config.doWrite: 

378 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

379 if exposure is not None: 

380 self.log.info("Persisting %s", self.getTempExpDatasetName(warpType)) 

381 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

382 

383 return dataRefList 

384 

385 @pipeBase.timeMethod 

386 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

387 """Create a Warp from inputs 

388 

389 We iterate over the multiple calexps in a single exposure to construct 

390 the warp (previously called a coaddTempExp) of that exposure to the 

391 supplied tract/patch. 

392 

393 Pixels that receive no pixels are set to NAN; this is not correct 

394 (violates LSST algorithms group policy), but will be fixed up by 

395 interpolating after the coaddition. 

396 

397 @param calexpRefList: List of data references for calexps that (may) 

398 overlap the patch of interest 

399 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

400 information about the patch 

401 @param visitId: integer identifier for visit, for the table that will 

402 produce the CoaddPsf 

403 @return a pipeBase Struct containing: 

404 - exposures: a dictionary containing the warps requested: 

405 "direct": direct warp if config.makeDirect 

406 "psfMatched": PSF-matched warp if config.makePsfMatched 

407 """ 

408 warpTypeList = self.getWarpTypeList() 

409 

410 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

411 didSetMetadata = {warpType: False for warpType in warpTypeList} 

412 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

413 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

414 for warpType in warpTypeList} 

415 

416 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

417 if dataIdList is None: 

418 dataIdList = ccdIdList 

419 

420 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

421 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

422 calExpInd+1, len(calExpList), dataId) 

423 

424 try: 

425 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

426 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

427 makeDirect=self.config.makeDirect, 

428 makePsfMatched=self.config.makePsfMatched) 

429 except Exception as e: 

430 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

431 continue 

432 try: 

433 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

434 for warpType in warpTypeList: 

435 exposure = warpedAndMatched.getDict()[warpType] 

436 if exposure is None: 

437 continue 

438 coaddTempExp = coaddTempExps[warpType] 

439 if didSetMetadata[warpType]: 

440 mimg = exposure.getMaskedImage() 

441 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

442 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

443 del mimg 

444 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

445 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

446 totGoodPix[warpType] += numGoodPix[warpType] 

447 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

448 dataId, numGoodPix[warpType], 

449 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

450 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

451 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

452 coaddTempExp.setFilterLabel(exposure.getFilterLabel()) 

453 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

454 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

455 coaddTempExp.setPsf(exposure.getPsf()) 

456 didSetMetadata[warpType] = True 

457 

458 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

459 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

460 

461 except Exception as e: 

462 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e) 

463 continue 

464 

465 for warpType in warpTypeList: 

466 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

467 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

468 

469 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

470 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

471 if warpType == "direct": 

472 coaddTempExps[warpType].setPsf( 

473 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

474 self.config.coaddPsf.makeControl())) 

475 else: 

476 if not self.config.doWriteEmptyWarps: 

477 # No good pixels. Exposure still empty 

478 coaddTempExps[warpType] = None 

479 # NoWorkFound is unnecessary as the downstream tasks will 

480 # adjust the quantum accordingly, and it prevents gen2 

481 # MakeCoaddTempExp from continuing to loop over visits. 

482 

483 result = pipeBase.Struct(exposures=coaddTempExps) 

484 return result 

485 

486 def getCalibratedExposure(self, dataRef, bgSubtracted): 

487 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

488 

489 @param[in] dataRef a sensor-level data reference 

490 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

491 calexp's background background model and add it to the calexp. 

492 @return calibrated exposure 

493 

494 @raises MissingExposureError If data for the exposure is not available. 

495 

496 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

497 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

498 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

499 retrieved from the processed exposure. When 

500 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

501 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

502 Otherwise, the astrometric calibration is taken from the processed 

503 exposure. 

504 """ 

505 try: 

506 exposure = dataRef.get(self.calexpType, immediate=True) 

507 except dafPersist.NoResults as e: 

508 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

509 

510 if not bgSubtracted: 

511 background = dataRef.get("calexpBackground", immediate=True) 

512 mi = exposure.getMaskedImage() 

513 mi += background.getImage() 

514 del mi 

515 

516 if self.config.doApplyExternalPhotoCalib: 

517 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

518 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

519 photoCalib = dataRef.get(source) 

520 exposure.setPhotoCalib(photoCalib) 

521 else: 

522 photoCalib = exposure.getPhotoCalib() 

523 

524 if self.config.doApplyExternalSkyWcs: 

525 source = f"{self.config.externalSkyWcsName}_wcs" 

526 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

527 skyWcs = dataRef.get(source) 

528 exposure.setWcs(skyWcs) 

529 

530 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

531 includeScaleUncertainty=self.config.includeCalibVar) 

532 exposure.maskedImage /= photoCalib.getCalibrationMean() 

533 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

534 # exposure.setCalib(afwImage.Calib(1.0)) 

535 return exposure 

536 

537 @staticmethod 

538 def _prepareEmptyExposure(skyInfo): 

539 """Produce an empty exposure for a given patch""" 

540 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

541 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

542 .getPlaneBitMask("NO_DATA"), numpy.inf) 

543 return exp 

544 

545 def getWarpTypeList(self): 

546 """Return list of requested warp types per the config. 

547 """ 

548 warpTypeList = [] 

549 if self.config.makeDirect: 

550 warpTypeList.append("direct") 

551 if self.config.makePsfMatched: 

552 warpTypeList.append("psfMatched") 

553 return warpTypeList 

554 

555 def applySkyCorr(self, dataRef, calexp): 

556 """Apply correction to the sky background level 

557 

558 Sky corrections can be generated with the 'skyCorrection.py' 

559 executable in pipe_drivers. Because the sky model used by that 

560 code extends over the entire focal plane, this can produce 

561 better sky subtraction. 

562 

563 The calexp is updated in-place. 

564 

565 Parameters 

566 ---------- 

567 dataRef : `lsst.daf.persistence.ButlerDataRef` 

568 Data reference for calexp. 

569 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

570 Calibrated exposure. 

571 """ 

572 bg = dataRef.get("skyCorr") 

573 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

574 if isinstance(calexp, afwImage.Exposure): 

575 calexp = calexp.getMaskedImage() 

576 calexp -= bg.getImage() 

577 

578 

579class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

580 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

581 defaultTemplates={"coaddName": "deep", 

582 "skyWcsName": "jointcal", 

583 "photoCalibName": "fgcm", 

584 "calexpType": ""}): 

585 calExpList = connectionTypes.Input( 

586 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

587 name="{calexpType}calexp", 

588 storageClass="ExposureF", 

589 dimensions=("instrument", "visit", "detector"), 

590 multiple=True, 

591 deferLoad=True, 

592 ) 

593 backgroundList = connectionTypes.Input( 

594 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

595 name="calexpBackground", 

596 storageClass="Background", 

597 dimensions=("instrument", "visit", "detector"), 

598 multiple=True, 

599 ) 

600 skyCorrList = connectionTypes.Input( 

601 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

602 name="skyCorr", 

603 storageClass="Background", 

604 dimensions=("instrument", "visit", "detector"), 

605 multiple=True, 

606 ) 

607 skyMap = connectionTypes.Input( 

608 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

609 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

610 storageClass="SkyMap", 

611 dimensions=("skymap",), 

612 ) 

613 externalSkyWcsTractCatalog = connectionTypes.Input( 

614 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

615 "id for the catalog id, sorted on id for fast lookup."), 

616 name="{skyWcsName}SkyWcsCatalog", 

617 storageClass="ExposureCatalog", 

618 dimensions=("instrument", "visit", "tract"), 

619 ) 

620 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

621 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

622 "These catalogs use the detector id for the catalog id, sorted on id for " 

623 "fast lookup."), 

624 name="{skyWcsName}SkyWcsCatalog", 

625 storageClass="ExposureCatalog", 

626 dimensions=("instrument", "visit"), 

627 ) 

628 externalPhotoCalibTractCatalog = connectionTypes.Input( 

629 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

630 "detector id for the catalog id, sorted on id for fast lookup."), 

631 name="{photoCalibName}PhotoCalibCatalog", 

632 storageClass="ExposureCatalog", 

633 dimensions=("instrument", "visit", "tract"), 

634 ) 

635 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

636 doc=("Per-visit photometric calibrations computed globally (with no tract " 

637 "information). These catalogs use the detector id for the catalog id, " 

638 "sorted on id for fast lookup."), 

639 name="{photoCalibName}PhotoCalibCatalog", 

640 storageClass="ExposureCatalog", 

641 dimensions=("instrument", "visit"), 

642 ) 

643 direct = connectionTypes.Output( 

644 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

645 "calexps onto the skyMap patch geometry."), 

646 name="{coaddName}Coadd_directWarp", 

647 storageClass="ExposureF", 

648 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

649 ) 

650 psfMatched = connectionTypes.Output( 

651 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

652 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

653 name="{coaddName}Coadd_psfMatchedWarp", 

654 storageClass="ExposureF", 

655 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

656 ) 

657 # TODO DM-28769, have selectImages subtask indicate which connections they need: 

658 wcsList = connectionTypes.Input( 

659 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

660 name="{calexpType}calexp.wcs", 

661 storageClass="Wcs", 

662 dimensions=("instrument", "visit", "detector"), 

663 multiple=True, 

664 ) 

665 bboxList = connectionTypes.Input( 

666 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

667 name="{calexpType}calexp.bbox", 

668 storageClass="Box2I", 

669 dimensions=("instrument", "visit", "detector"), 

670 multiple=True, 

671 ) 

672 srcList = connectionTypes.Input( 

673 doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability", 

674 name="src", 

675 storageClass="SourceCatalog", 

676 dimensions=("instrument", "visit", "detector"), 

677 multiple=True, 

678 ) 

679 psfList = connectionTypes.Input( 

680 doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing", 

681 name="{calexpType}calexp.psf", 

682 storageClass="Psf", 

683 dimensions=("instrument", "visit", "detector"), 

684 multiple=True, 

685 ) 

686 

687 def __init__(self, *, config=None): 

688 super().__init__(config=config) 

689 if config.bgSubtracted: 

690 self.inputs.remove("backgroundList") 

691 if not config.doApplySkyCorr: 

692 self.inputs.remove("skyCorrList") 

693 if config.doApplyExternalSkyWcs: 

694 if config.useGlobalExternalSkyWcs: 

695 self.inputs.remove("externalSkyWcsTractCatalog") 

696 else: 

697 self.inputs.remove("externalSkyWcsGlobalCatalog") 

698 else: 

699 self.inputs.remove("externalSkyWcsTractCatalog") 

700 self.inputs.remove("externalSkyWcsGlobalCatalog") 

701 if config.doApplyExternalPhotoCalib: 

702 if config.useGlobalExternalPhotoCalib: 

703 self.inputs.remove("externalPhotoCalibTractCatalog") 

704 else: 

705 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

706 else: 

707 self.inputs.remove("externalPhotoCalibTractCatalog") 

708 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

709 if not config.makeDirect: 

710 self.outputs.remove("direct") 

711 if not config.makePsfMatched: 

712 self.outputs.remove("psfMatched") 

713 # TODO DM-28769: add connection per selectImages connections 

714 # instead of removing if not PsfWcsSelectImagesTask here: 

715 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

716 self.inputs.remove("srcList") 

717 if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask: 

718 self.inputs.remove("psfList") 

719 

720 

721class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

722 pipelineConnections=MakeWarpConnections): 

723 

724 def validate(self): 

725 super().validate() 

726 

727 

728class MakeWarpTask(MakeCoaddTempExpTask): 

729 """Warp and optionally PSF-Match calexps onto an a common projection 

730 """ 

731 ConfigClass = MakeWarpConfig 

732 _DefaultName = "makeWarp" 

733 

734 @utils.inheritDoc(pipeBase.PipelineTask) 

735 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

736 """ 

737 Notes 

738 ---- 

739 Construct warps for requested warp type for single epoch 

740 

741 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

742 calexps. This method is analogous to `runDataRef`. 

743 """ 

744 

745 # Ensure all input lists are in same detector order as the calExpList 

746 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] 

747 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

748 

749 # Read in all inputs. 

750 inputs = butlerQC.get(inputRefs) 

751 

752 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

753 # from the dictionary so we can pass it as kwargs later. 

754 skyMap = inputs.pop("skyMap") 

755 quantumDataId = butlerQC.quantum.dataId 

756 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

757 

758 # Construct list of input DataIds expected by `run` 

759 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

760 # Construct list of packed integer IDs expected by `run` 

761 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

762 

763 # Run the selector and filter out calexps that were not selected 

764 # primarily because they do not overlap the patch 

765 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

766 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

767 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

768 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

769 

770 # Read from disk only the selected calexps 

771 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']] 

772 

773 # Extract integer visitId requested by `run` 

774 visits = [dataId['visit'] for dataId in dataIdList] 

775 visitId = visits[0] 

776 

777 if self.config.doApplyExternalSkyWcs: 

778 if self.config.useGlobalExternalSkyWcs: 

779 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

780 else: 

781 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

782 else: 

783 externalSkyWcsCatalog = None 

784 

785 if self.config.doApplyExternalPhotoCalib: 

786 if self.config.useGlobalExternalPhotoCalib: 

787 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog") 

788 else: 

789 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

790 else: 

791 externalPhotoCalibCatalog = None 

792 

793 completeIndices = self.prepareCalibratedExposures(**inputs, 

794 externalSkyWcsCatalog=externalSkyWcsCatalog, 

795 externalPhotoCalibCatalog=externalPhotoCalibCatalog) 

796 # Redo the input selection with inputs with complete wcs/photocalib info. 

797 inputs = self.filterInputs(indices=completeIndices, inputs=inputs) 

798 

799 results = self.run(**inputs, visitId=visitId, 

800 ccdIdList=[ccdIdList[i] for i in goodIndices], 

801 dataIdList=[dataIdList[i] for i in goodIndices], 

802 skyInfo=skyInfo) 

803 if self.config.makeDirect and results.exposures["direct"] is not None: 

804 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

805 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None: 

806 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

807 

808 def filterInputs(self, indices, inputs): 

809 """Return task inputs with their lists filtered by indices 

810 

811 Parameters 

812 ---------- 

813 indices : `list` of integers 

814 inputs : `dict` of `list` of input connections to be passed to run 

815 """ 

816 for key in inputs.keys(): 

817 # Only down-select on list inputs 

818 if isinstance(inputs[key], list): 

819 inputs[key] = [inputs[key][ind] for ind in indices] 

820 return inputs 

821 

822 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, 

823 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, 

824 **kwargs): 

825 """Calibrate and add backgrounds to input calExpList in place 

826 

827 Parameters 

828 ---------- 

829 calExpList : `list` of `lsst.afw.image.Exposure` 

830 Sequence of calexps to be modified in place 

831 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional 

832 Sequence of backgrounds to be added back in if bgSubtracted=False 

833 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional 

834 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

835 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

836 Exposure catalog with external skyWcs to be applied 

837 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

838 for the catalog id, sorted on id for fast lookup. 

839 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

840 Exposure catalog with external photoCalib to be applied 

841 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

842 id for the catalog id, sorted on id for fast lookup. 

843 

844 Returns 

845 ------- 

846 indices : `list` [`int`] 

847 Indices of calExpList and friends that have valid photoCalib/skyWcs 

848 """ 

849 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

850 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

851 

852 includeCalibVar = self.config.includeCalibVar 

853 

854 indices = [] 

855 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList, 

856 backgroundList, 

857 skyCorrList)): 

858 mi = calexp.maskedImage 

859 if not self.config.bgSubtracted: 

860 mi += background.getImage() 

861 

862 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None: 

863 detectorId = calexp.getInfo().getDetector().getId() 

864 

865 # Find the external photoCalib 

866 if externalPhotoCalibCatalog is not None: 

867 row = externalPhotoCalibCatalog.find(detectorId) 

868 if row is None: 

869 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

870 "and will not be used in the warp.", detectorId) 

871 continue 

872 photoCalib = row.getPhotoCalib() 

873 if photoCalib is None: 

874 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog " 

875 "and will not be used in the warp.", detectorId) 

876 continue 

877 calexp.setPhotoCalib(photoCalib) 

878 else: 

879 photoCalib = calexp.getPhotoCalib() 

880 if photoCalib is None: 

881 self.log.warning("Detector id %s has None for photoCalib in the calexp " 

882 "and will not be used in the warp.", detectorId) 

883 continue 

884 

885 # Find and apply external skyWcs 

886 if externalSkyWcsCatalog is not None: 

887 row = externalSkyWcsCatalog.find(detectorId) 

888 if row is None: 

889 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

890 "and will not be used in the warp.", detectorId) 

891 continue 

892 skyWcs = row.getWcs() 

893 if skyWcs is None: 

894 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog " 

895 "and will not be used in the warp.", detectorId) 

896 continue 

897 calexp.setWcs(skyWcs) 

898 else: 

899 skyWcs = calexp.getWcs() 

900 if skyWcs is None: 

901 self.log.warning("Detector id %s has None for skyWcs in the calexp " 

902 "and will not be used in the warp.", detectorId) 

903 continue 

904 

905 # Calibrate the image 

906 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, 

907 includeScaleUncertainty=includeCalibVar) 

908 calexp.maskedImage /= photoCalib.getCalibrationMean() 

909 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

910 # exposure.setCalib(afwImage.Calib(1.0)) 

911 

912 # Apply skycorr 

913 if self.config.doApplySkyCorr: 

914 mi -= skyCorr.getImage() 

915 

916 indices.append(index) 

917 

918 return indices 

919 

920 

921def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): 

922 """Reorder inputRefs per outputSortKeyOrder 

923 

924 Any inputRefs which are lists will be resorted per specified key e.g., 

925 'detector.' Only iterables will be reordered, and values can be of type 

926 `lsst.pipe.base.connections.DeferredDatasetRef` or 

927 `lsst.daf.butler.core.datasets.ref.DatasetRef`. 

928 Returned lists of refs have the same length as the outputSortKeyOrder. 

929 If an outputSortKey not in the inputRef, then it will be padded with None. 

930 If an inputRef contains an inputSortKey that is not in the 

931 outputSortKeyOrder it will be removed. 

932 

933 Parameters 

934 ---------- 

935 inputRefs : `lsst.pipe.base.connections.QuantizedConnection` 

936 Input references to be reordered and padded. 

937 outputSortKeyOrder : iterable 

938 Iterable of values to be compared with inputRef's dataId[dataIdKey] 

939 dataIdKey : `str` 

940 dataIdKey in the dataRefs to compare with the outputSortKeyOrder. 

941 

942 Returns: 

943 -------- 

944 inputRefs: `lsst.pipe.base.connections.QuantizedConnection` 

945 Quantized Connection with sorted DatasetRef values sorted if iterable. 

946 """ 

947 for connectionName, refs in inputRefs: 

948 if isinstance(refs, Iterable): 

949 if hasattr(refs[0], "dataId"): 

950 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] 

951 else: 

952 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] 

953 if inputSortKeyOrder != outputSortKeyOrder: 

954 setattr(inputRefs, connectionName, 

955 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) 

956 return inputRefs