Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy 

23 

24import lsst.pex.config as pexConfig 

25import lsst.daf.persistence as dafPersist 

26import lsst.afw.image as afwImage 

27import lsst.coadd.utils as coaddUtils 

28import lsst.pipe.base as pipeBase 

29import lsst.pipe.base.connectionTypes as cT 

30import lsst.log as log 

31import lsst.utils as utils 

32import lsst.geom 

33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig 

34from lsst.skymap import BaseSkyMap 

35from .coaddBase import CoaddBaseTask, makeSkyInfo 

36from .warpAndPsfMatch import WarpAndPsfMatchTask 

37from .coaddHelpers import groupPatchExposures, getGroupDataRef 

38 

39__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"] 

40 

41 

42class MissingExposureError(Exception): 

43 """Raised when data cannot be retrieved for an exposure. 

44 When processing patches, sometimes one exposure is missing; this lets us 

45 distinguish bewteen that case, and other errors. 

46 """ 

47 pass 

48 

49 

50class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass): 

51 """Config for MakeCoaddTempExpTask 

52 """ 

53 warpAndPsfMatch = pexConfig.ConfigurableField( 

54 target=WarpAndPsfMatchTask, 

55 doc="Task to warp and PSF-match calexp", 

56 ) 

57 doWrite = pexConfig.Field( 

58 doc="persist <coaddName>Coadd_<warpType>Warp", 

59 dtype=bool, 

60 default=True, 

61 ) 

62 bgSubtracted = pexConfig.Field( 

63 doc="Work with a background subtracted calexp?", 

64 dtype=bool, 

65 default=True, 

66 ) 

67 coaddPsf = pexConfig.ConfigField( 

68 doc="Configuration for CoaddPsf", 

69 dtype=CoaddPsfConfig, 

70 ) 

71 makeDirect = pexConfig.Field( 

72 doc="Make direct Warp/Coadds", 

73 dtype=bool, 

74 default=True, 

75 ) 

76 makePsfMatched = pexConfig.Field( 

77 doc="Make Psf-Matched Warp/Coadd?", 

78 dtype=bool, 

79 default=False, 

80 ) 

81 

82 doWriteEmptyWarps = pexConfig.Field( 

83 dtype=bool, 

84 default=False, 

85 doc="Write out warps even if they are empty" 

86 ) 

87 

88 hasFakes = pexConfig.Field( 

89 doc="Should be set to True if fake sources have been inserted into the input data.", 

90 dtype=bool, 

91 default=False, 

92 ) 

93 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?") 

94 

95 def validate(self): 

96 CoaddBaseTask.ConfigClass.validate(self) 

97 if not self.makePsfMatched and not self.makeDirect: 

98 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True") 

99 if self.doPsfMatch: 

100 # Backwards compatibility. 

101 log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False") 

102 self.makePsfMatched = True 

103 self.makeDirect = False 

104 

105 def setDefaults(self): 

106 CoaddBaseTask.ConfigClass.setDefaults(self) 

107 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize 

108 

109## \addtogroup LSST_task_documentation 

110## \{ 

111## \page MakeCoaddTempExpTask 

112## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask" 

113## \copybrief MakeCoaddTempExpTask 

114## \} 

115 

116 

117class MakeCoaddTempExpTask(CoaddBaseTask): 

118 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 

119 

120 @anchor MakeCoaddTempExpTask_ 

121 

122 @section pipe_tasks_makeCoaddTempExp_Contents Contents 

123 

124 - @ref pipe_tasks_makeCoaddTempExp_Purpose 

125 - @ref pipe_tasks_makeCoaddTempExp_Initialize 

126 - @ref pipe_tasks_makeCoaddTempExp_IO 

127 - @ref pipe_tasks_makeCoaddTempExp_Config 

128 - @ref pipe_tasks_makeCoaddTempExp_Debug 

129 - @ref pipe_tasks_makeCoaddTempExp_Example 

130 

131 @section pipe_tasks_makeCoaddTempExp_Purpose Description 

132 

133 Warp and optionally PSF-Match calexps onto a common projection, by 

134 performing the following operations: 

135 - Group calexps by visit/run 

136 - For each visit, generate a Warp by calling method @ref makeTempExp. 

137 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 

138 on each visit 

139 

140 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 

141 

142 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 

143 

144 @copydoc \_\_init\_\_ 

145 

146 This task has one special keyword argument: passing reuse=True will cause 

147 the task to skip the creation of warps that are already present in the 

148 output repositories. 

149 

150 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 

151 

152 This task is primarily designed to be run from the command line. 

153 

154 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 

155 to process. 

156 

157 @copydoc run 

158 

159 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 

160 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 

161 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 

162 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 

163 

164 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 

165 

166 See @ref MakeCoaddTempExpConfig and parameters inherited from 

167 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 

168 

169 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 

170 

171 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 

172 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

173 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 

174 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 

175 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 

176 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 

177 

178 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 

179 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 

180 In general, for a set of input images, this config should equal the FWHM of the visit 

181 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 

182 of the other config options offer a reasonable starting point. 

183 The following list presents the most common problems that arise from a misconfigured 

184 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 

185 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 

186 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 

187 Problem: Explanation. *Solution* 

188 

189 *Troublshooting PSF-Matching Configuration:* 

190 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 

191 For example:_ 

192 

193 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 

194 

195 Note that increasing the kernel size also increases runtime. 

196 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 

197 for matching kernel. _Provide the matcher with more data by either increasing 

198 the spatial sampling by decreasing the spatial cell size,_ 

199 

200 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 

201 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 

202 

203 _or increasing the padding around the Science PSF, for example:_ 

204 

205 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 

206 

207 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 

208 matching kernel dimensions, thus increasing the number of pixels available to fit 

209 after convolving the PSF with the matching kernel. 

210 Optionally, for debugging the effects of padding, the level of padding may be manually 

211 controlled by setting turning off the automatic padding and setting the number 

212 of pixels by which to pad the PSF: 

213 

214 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 

215 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 

216 

217 - Deconvolution: Matching a large PSF to a smaller PSF produces 

218 a telltale noise pattern which looks like ripples or a brain. 

219 _Increase the size of the requested model PSF. For example:_ 

220 

221 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 

222 

223 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 

224 _Increase the width of the Gaussian basis functions. For example:_ 

225 

226 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 

227 # from default [0.7, 1.5, 3.0] 

228 

229 

230 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 

231 

232 MakeCoaddTempExpTask has no debug output, but its subtasks do. 

233 

234 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 

235 

236 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 

237 into the larger Data Release Processing. 

238 Set up by running: 

239 

240 setup ci_hsc 

241 cd $CI_HSC_DIR 

242 # if not built already: 

243 python $(which scons) # this will take a while 

244 

245 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 

246 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 

247 output respository with the desired SkyMap. The command, 

248 

249 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 

250 --id patch=5,4 tract=0 filter=HSC-I \ 

251 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 

252 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 

253 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \ 

254 makePsfMatched=True modelPsf.defaultFwhm=11 

255 

256 writes a direct and PSF-Matched Warp to 

257 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 

258 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 

259 respectively. 

260 

261 @note PSF-Matching in this particular dataset would benefit from adding 

262 `--configfile ./matchingConfig.py` to 

263 the command line arguments where `matchingConfig.py` is defined by: 

264 

265 echo " 

266 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 

267 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 

268 

269 

270 Add the option `--help` to see more options. 

271 """ 

272 ConfigClass = MakeCoaddTempExpConfig 

273 _DefaultName = "makeCoaddTempExp" 

274 

275 def __init__(self, reuse=False, **kwargs): 

276 CoaddBaseTask.__init__(self, **kwargs) 

277 self.reuse = reuse 

278 self.makeSubtask("warpAndPsfMatch") 

279 if self.config.hasFakes: 

280 self.calexpType = "fakes_calexp" 

281 else: 

282 self.calexpType = "calexp" 

283 

284 @pipeBase.timeMethod 

285 def runDataRef(self, patchRef, selectDataList=[]): 

286 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 

287 

288 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 

289 plus the camera-specific filter key (e.g. "filter" or "band") 

290 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 

291 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 

292 warps are requested. 

293 

294 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 

295 

296 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp 

297 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored 

298 (assembleCoadd should determine zeropoint scaling without referring to it). 

299 """ 

300 skyInfo = self.getSkyInfo(patchRef) 

301 

302 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested 

303 if self.config.makePsfMatched and not self.config.makeDirect: 

304 primaryWarpDataset = self.getTempExpDatasetName("psfMatched") 

305 else: 

306 primaryWarpDataset = self.getTempExpDatasetName("direct") 

307 

308 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) 

309 

310 if len(calExpRefList) == 0: 

311 self.log.warn("No exposures to coadd for patch %s", patchRef.dataId) 

312 return None 

313 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId) 

314 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)] 

315 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId) 

316 

317 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), 

318 primaryWarpDataset) 

319 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId) 

320 

321 dataRefList = [] 

322 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()): 

323 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset, 

324 tempExpTuple, groupData.keys) 

325 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 

326 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId) 

327 dataRefList.append(tempExpRef) 

328 continue 

329 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId) 

330 

331 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). 

332 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index 

333 # of the visit in the list. 

334 try: 

335 visitId = int(tempExpRef.dataId["visit"]) 

336 except (KeyError, ValueError): 

337 visitId = i 

338 

339 calExpList = [] 

340 ccdIdList = [] 

341 dataIdList = [] 

342 

343 for calExpInd, calExpRef in enumerate(calexpRefList): 

344 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList), 

345 calExpRef.dataId) 

346 try: 

347 ccdId = calExpRef.get("ccdExposureId", immediate=True) 

348 except Exception: 

349 ccdId = calExpInd 

350 try: 

351 # We augment the dataRef here with the tract, which is harmless for loading things 

352 # like calexps that don't need the tract, and necessary for meas_mosaic outputs, 

353 # which do. 

354 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType, 

355 dataId=calExpRef.dataId, 

356 tract=skyInfo.tractInfo.getId()) 

357 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted) 

358 except Exception as e: 

359 self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e) 

360 continue 

361 

362 if self.config.doApplySkyCorr: 

363 self.applySkyCorr(calExpRef, calExp) 

364 

365 calExpList.append(calExp) 

366 ccdIdList.append(ccdId) 

367 dataIdList.append(calExpRef.dataId) 

368 

369 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures 

370 

371 if any(exps.values()): 

372 dataRefList.append(tempExpRef) 

373 else: 

374 self.log.warn("Warp %s could not be created", tempExpRef.dataId) 

375 

376 if self.config.doWrite: 

377 for (warpType, exposure) in exps.items(): # compatible w/ Py3 

378 if exposure is not None: 

379 self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType)) 

380 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType)) 

381 

382 return dataRefList 

383 

384 @pipeBase.timeMethod 

385 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): 

386 """Create a Warp from inputs 

387 

388 We iterate over the multiple calexps in a single exposure to construct 

389 the warp (previously called a coaddTempExp) of that exposure to the 

390 supplied tract/patch. 

391 

392 Pixels that receive no pixels are set to NAN; this is not correct 

393 (violates LSST algorithms group policy), but will be fixed up by 

394 interpolating after the coaddition. 

395 

396 @param calexpRefList: List of data references for calexps that (may) 

397 overlap the patch of interest 

398 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 

399 information about the patch 

400 @param visitId: integer identifier for visit, for the table that will 

401 produce the CoaddPsf 

402 @return a pipeBase Struct containing: 

403 - exposures: a dictionary containing the warps requested: 

404 "direct": direct warp if config.makeDirect 

405 "psfMatched": PSF-matched warp if config.makePsfMatched 

406 """ 

407 warpTypeList = self.getWarpTypeList() 

408 

409 totGoodPix = {warpType: 0 for warpType in warpTypeList} 

410 didSetMetadata = {warpType: False for warpType in warpTypeList} 

411 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} 

412 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) 

413 for warpType in warpTypeList} 

414 

415 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None 

416 if dataIdList is None: 

417 dataIdList = ccdIdList 

418 

419 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): 

420 self.log.info("Processing calexp %d of %d for this Warp: id=%s", 

421 calExpInd+1, len(calExpList), dataId) 

422 

423 try: 

424 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, 

425 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, 

426 makeDirect=self.config.makeDirect, 

427 makePsfMatched=self.config.makePsfMatched) 

428 except Exception as e: 

429 self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) 

430 continue 

431 try: 

432 numGoodPix = {warpType: 0 for warpType in warpTypeList} 

433 for warpType in warpTypeList: 

434 exposure = warpedAndMatched.getDict()[warpType] 

435 if exposure is None: 

436 continue 

437 coaddTempExp = coaddTempExps[warpType] 

438 if didSetMetadata[warpType]: 

439 mimg = exposure.getMaskedImage() 

440 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() 

441 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) 

442 del mimg 

443 numGoodPix[warpType] = coaddUtils.copyGoodPixels( 

444 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) 

445 totGoodPix[warpType] += numGoodPix[warpType] 

446 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", 

447 dataId, numGoodPix[warpType], 

448 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) 

449 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: 

450 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) 

451 coaddTempExp.setFilterLabel(exposure.getFilterLabel()) 

452 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) 

453 # PSF replaced with CoaddPsf after loop if and only if creating direct warp 

454 coaddTempExp.setPsf(exposure.getPsf()) 

455 didSetMetadata[warpType] = True 

456 

457 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched 

458 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) 

459 

460 except Exception as e: 

461 self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e) 

462 continue 

463 

464 for warpType in warpTypeList: 

465 self.log.info("%sWarp has %d good pixels (%.1f%%)", 

466 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) 

467 

468 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: 

469 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) 

470 if warpType == "direct": 

471 coaddTempExps[warpType].setPsf( 

472 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, 

473 self.config.coaddPsf.makeControl())) 

474 else: 

475 if not self.config.doWriteEmptyWarps: 

476 # No good pixels. Exposure still empty 

477 coaddTempExps[warpType] = None 

478 

479 result = pipeBase.Struct(exposures=coaddTempExps) 

480 return result 

481 

482 def getCalibratedExposure(self, dataRef, bgSubtracted): 

483 """Return one calibrated Exposure, possibly with an updated SkyWcs. 

484 

485 @param[in] dataRef a sensor-level data reference 

486 @param[in] bgSubtracted return calexp with background subtracted? If False get the 

487 calexp's background background model and add it to the calexp. 

488 @return calibrated exposure 

489 

490 @raises MissingExposureError If data for the exposure is not available. 

491 

492 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration 

493 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the 

494 `name_photoCalib` dataset. Otherwise, the photometric calibration is 

495 retrieved from the processed exposure. When 

496 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration 

497 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset. 

498 Otherwise, the astrometric calibration is taken from the processed 

499 exposure. 

500 """ 

501 try: 

502 exposure = dataRef.get(self.calexpType, immediate=True) 

503 except dafPersist.NoResults as e: 

504 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e 

505 

506 if not bgSubtracted: 

507 background = dataRef.get("calexpBackground", immediate=True) 

508 mi = exposure.getMaskedImage() 

509 mi += background.getImage() 

510 del mi 

511 

512 if self.config.doApplyExternalPhotoCalib: 

513 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

514 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source) 

515 photoCalib = dataRef.get(source) 

516 exposure.setPhotoCalib(photoCalib) 

517 else: 

518 photoCalib = exposure.getPhotoCalib() 

519 

520 if self.config.doApplyExternalSkyWcs: 

521 source = f"{self.config.externalSkyWcsName}_wcs" 

522 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source) 

523 skyWcs = dataRef.get(source) 

524 exposure.setWcs(skyWcs) 

525 

526 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage, 

527 includeScaleUncertainty=self.config.includeCalibVar) 

528 exposure.maskedImage /= photoCalib.getCalibrationMean() 

529 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented. 

530 # exposure.setCalib(afwImage.Calib(1.0)) 

531 return exposure 

532 

533 @staticmethod 

534 def _prepareEmptyExposure(skyInfo): 

535 """Produce an empty exposure for a given patch""" 

536 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

537 exp.getMaskedImage().set(numpy.nan, afwImage.Mask 

538 .getPlaneBitMask("NO_DATA"), numpy.inf) 

539 return exp 

540 

541 def getWarpTypeList(self): 

542 """Return list of requested warp types per the config. 

543 """ 

544 warpTypeList = [] 

545 if self.config.makeDirect: 

546 warpTypeList.append("direct") 

547 if self.config.makePsfMatched: 

548 warpTypeList.append("psfMatched") 

549 return warpTypeList 

550 

551 def applySkyCorr(self, dataRef, calexp): 

552 """Apply correction to the sky background level 

553 

554 Sky corrections can be generated with the 'skyCorrection.py' 

555 executable in pipe_drivers. Because the sky model used by that 

556 code extends over the entire focal plane, this can produce 

557 better sky subtraction. 

558 

559 The calexp is updated in-place. 

560 

561 Parameters 

562 ---------- 

563 dataRef : `lsst.daf.persistence.ButlerDataRef` 

564 Data reference for calexp. 

565 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

566 Calibrated exposure. 

567 """ 

568 bg = dataRef.get("skyCorr") 

569 self.log.debug("Applying sky correction to %s", dataRef.dataId) 

570 if isinstance(calexp, afwImage.Exposure): 

571 calexp = calexp.getMaskedImage() 

572 calexp -= bg.getImage() 

573 

574 

575class MakeWarpConnections(pipeBase.PipelineTaskConnections, 

576 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

577 defaultTemplates={"coaddName": "deep"}): 

578 calExpList = cT.Input( 

579 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", 

580 name="calexp", 

581 storageClass="ExposureF", 

582 dimensions=("instrument", "visit", "detector"), 

583 multiple=True, 

584 deferLoad=True, 

585 ) 

586 backgroundList = cT.Input( 

587 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", 

588 name="calexpBackground", 

589 storageClass="Background", 

590 dimensions=("instrument", "visit", "detector"), 

591 multiple=True, 

592 ) 

593 skyCorrList = cT.Input( 

594 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

595 name="skyCorr", 

596 storageClass="Background", 

597 dimensions=("instrument", "visit", "detector"), 

598 multiple=True, 

599 ) 

600 skyMap = cT.Input( 

601 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

602 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

603 storageClass="SkyMap", 

604 dimensions=("skymap",), 

605 ) 

606 direct = cT.Output( 

607 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", 

608 "calexps onto the skyMap patch geometry."), 

609 name="{coaddName}Coadd_directWarp", 

610 storageClass="ExposureF", 

611 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

612 ) 

613 psfMatched = cT.Output( 

614 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", 

615 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), 

616 name="{coaddName}Coadd_psfMatchedWarp", 

617 storageClass="ExposureF", 

618 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

619 ) 

620 # TODO DM-28769, have selectImages subtask indicate which connections they need: 

621 wcsList = cT.Input( 

622 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

623 name="calexp.wcs", 

624 storageClass="Wcs", 

625 dimensions=("instrument", "visit", "detector"), 

626 multiple=True, 

627 ) 

628 bboxList = cT.Input( 

629 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch", 

630 name="calexp.bbox", 

631 storageClass="Box2I", 

632 dimensions=("instrument", "visit", "detector"), 

633 multiple=True, 

634 ) 

635 srcList = cT.Input( 

636 doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability", 

637 name="src", 

638 storageClass="SourceCatalog", 

639 dimensions=("instrument", "visit", "detector"), 

640 multiple=True, 

641 ) 

642 psfList = cT.Input( 

643 doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing", 

644 name="calexp.psf", 

645 storageClass="Psf", 

646 dimensions=("instrument", "visit", "detector"), 

647 multiple=True, 

648 ) 

649 

650 def __init__(self, *, config=None): 

651 super().__init__(config=config) 

652 if config.bgSubtracted: 

653 self.inputs.remove("backgroundList") 

654 if not config.doApplySkyCorr: 

655 self.inputs.remove("skyCorrList") 

656 if not config.makeDirect: 

657 self.outputs.remove("direct") 

658 if not config.makePsfMatched: 

659 self.outputs.remove("psfMatched") 

660 # TODO DM-28769: add connection per selectImages connections 

661 # instead of removing if not PsfWcsSelectImagesTask here: 

662 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask: 

663 self.inputs.remove("srcList") 

664 if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask: 

665 self.inputs.remove("psfList") 

666 

667 

668class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig, 

669 pipelineConnections=MakeWarpConnections): 

670 

671 def validate(self): 

672 super().validate() 

673 # TODO: Remove this constraint after DM-17062 

674 if self.doApplyExternalPhotoCalib: 

675 raise RuntimeError("Gen3 MakeWarpTask cannot apply external PhotoCalib results. " 

676 "Please set doApplyExternalPhotoCalib=False.") 

677 if self.doApplyExternalSkyWcs: 

678 raise RuntimeError("Gen3 MakeWarpTask cannot apply external SkyWcs results. " 

679 "Please set doApplyExternalSkyWcs=False.") 

680 

681 

682class MakeWarpTask(MakeCoaddTempExpTask): 

683 """Warp and optionally PSF-Match calexps onto an a common projection 

684 

685 First Draft of a Gen3 compatible MakeWarpTask which 

686 currently does not handle doApplyExternalPhotoCalib=True or 

687 doApplyExternalSkyWcs=True. 

688 """ 

689 ConfigClass = MakeWarpConfig 

690 _DefaultName = "makeWarp" 

691 

692 @utils.inheritDoc(pipeBase.PipelineTask) 

693 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

694 """ 

695 Notes 

696 ---- 

697 Construct warps for requested warp type for single epoch 

698 

699 PipelineTask (Gen3) entry point to warp and optionally PSF-match 

700 calexps. This method is analogous to `runDataRef`. 

701 """ 

702 # Read in all inputs. 

703 inputs = butlerQC.get(inputRefs) 

704 

705 # Construct skyInfo expected by `run`. We remove the SkyMap itself 

706 # from the dictionary so we can pass it as kwargs later. 

707 skyMap = inputs.pop("skyMap") 

708 quantumDataId = butlerQC.quantum.dataId 

709 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch']) 

710 

711 # Construct list of input DataIds expected by `run` 

712 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList] 

713 # Construct list of packed integer IDs expected by `run` 

714 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList] 

715 

716 # Run the selector and filter out calexps that were not selected 

717 # primarily because they do not overlap the patch 

718 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners() 

719 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

720 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList) 

721 inputs = self.filterInputs(indices=goodIndices, inputs=inputs) 

722 

723 # Read from disk only the selected calexps 

724 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']] 

725 

726 # Extract integer visitId requested by `run` 

727 visits = [dataId['visit'] for dataId in dataIdList] 

728 assert(all(visits[0] == visit for visit in visits)) 

729 visitId = visits[0] 

730 

731 self.prepareCalibratedExposures(**inputs) 

732 results = self.run(**inputs, visitId=visitId, 

733 ccdIdList=[ccdIdList[i] for i in goodIndices], 

734 dataIdList=[dataIdList[i] for i in goodIndices], 

735 skyInfo=skyInfo) 

736 if self.config.makeDirect: 

737 butlerQC.put(results.exposures["direct"], outputRefs.direct) 

738 if self.config.makePsfMatched: 

739 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched) 

740 

741 def filterInputs(self, indices, inputs): 

742 """Return task inputs with their lists filtered by indices 

743 

744 Parameters 

745 ---------- 

746 indices : `list` of integers 

747 inputs : `dict` of `list` of input connections to be passed to run 

748 """ 

749 for key in inputs.keys(): 

750 inputs[key] = [inputs[key][ind] for ind in indices] 

751 return inputs 

752 

753 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, **kwargs): 

754 """Calibrate and add backgrounds to input calExpList in place 

755 

756 TODO DM-17062: apply jointcal/meas_mosaic here 

757 

758 Parameters 

759 ---------- 

760 calExpList : `list` of `lsst.afw.image.Exposure` 

761 Sequence of calexps to be modified in place 

762 backgroundList : `list` of `lsst.afw.math.backgroundList` 

763 Sequence of backgrounds to be added back in if bgSubtracted=False 

764 skyCorrList : `list` of `lsst.afw.math.backgroundList` 

765 Sequence of background corrections to be subtracted if doApplySkyCorr=True 

766 """ 

767 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList 

768 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList 

769 for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList): 

770 mi = calexp.maskedImage 

771 if not self.config.bgSubtracted: 

772 mi += background.getImage() 

773 if self.config.doApplySkyCorr: 

774 mi -= skyCorr.getImage()