Coverage for python/lsst/pipe/tasks/coaddBase.py: 48%

100 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-30 11:31 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2015 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import lsst.pex.config as pexConfig 

23import lsst.geom as geom 

24import lsst.afw.geom as afwGeom 

25import lsst.afw.image as afwImage 

26import lsst.pipe.base as pipeBase 

27import lsst.meas.algorithms as measAlg 

28 

29from lsst.afw.fits import FitsError 

30from lsst.coadd.utils import CoaddDataIdContainer 

31from lsst.meas.algorithms import ScaleVarianceTask 

32from .selectImages import WcsSelectImagesTask, SelectStruct 

33from .coaddInputRecorder import CoaddInputRecorderTask 

34 

35__all__ = ["CoaddBaseTask", "getSkyInfo", "makeSkyInfo", "makeCoaddSuffix"] 

36 

37 

38class CoaddBaseConfig(pexConfig.Config): 

39 """!Configuration parameters for CoaddBaseTask 

40 

41 @anchor CoaddBaseConfig_ 

42 

43 @brief Configuration parameters shared between MakeCoaddTempExp and AssembleCoadd 

44 """ 

45 coaddName = pexConfig.Field( 

46 doc="Coadd name: typically one of deep or goodSeeing.", 

47 dtype=str, 

48 default="deep", 

49 ) 

50 select = pexConfig.ConfigurableField( 

51 doc="Image selection subtask.", 

52 target=WcsSelectImagesTask, 

53 ) 

54 badMaskPlanes = pexConfig.ListField( 

55 dtype=str, 

56 doc="Mask planes that, if set, the associated pixel should not be included in the coaddTempExp.", 

57 default=("NO_DATA",), 

58 ) 

59 inputRecorder = pexConfig.ConfigurableField( 

60 doc="Subtask that helps fill CoaddInputs catalogs added to the final Exposure", 

61 target=CoaddInputRecorderTask 

62 ) 

63 doPsfMatch = pexConfig.Field( 

64 dtype=bool, 

65 doc="Match to modelPsf? Deprecated. Sets makePsfMatched=True, makeDirect=False", 

66 default=False 

67 ) 

68 modelPsf = measAlg.GaussianPsfFactory.makeField(doc="Model Psf factory") 

69 doApplyExternalPhotoCalib = pexConfig.Field( 

70 dtype=bool, 

71 default=False, 

72 doc=("Whether to apply external photometric calibration via an " 

73 "`lsst.afw.image.PhotoCalib` object. Uses the " 

74 "`externalPhotoCalibName` field to determine which calibration " 

75 "to load.") 

76 ) 

77 useGlobalExternalPhotoCalib = pexConfig.Field( 

78 dtype=bool, 

79 default=True, 

80 doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations " 

81 "that are not run per-tract. When False, use per-tract photometric " 

82 "calibration files.") 

83 ) 

84 externalPhotoCalibName = pexConfig.ChoiceField( 

85 # TODO: Remove this config with the removal of Gen2 in DM-20572. 

86 dtype=str, 

87 doc=("Type of external PhotoCalib if `doApplyExternalPhotoCalib` is True. " 

88 "This field is only used for Gen2 middleware."), 

89 default="jointcal", 

90 allowed={ 

91 "jointcal": "Use jointcal_photoCalib", 

92 "fgcm": "Use fgcm_photoCalib", 

93 "fgcm_tract": "Use fgcm_tract_photoCalib" 

94 } 

95 ) 

96 doApplyExternalSkyWcs = pexConfig.Field( 

97 dtype=bool, 

98 default=False, 

99 doc=("Whether to apply external astrometric calibration via an " 

100 "`lsst.afw.geom.SkyWcs` object. Uses `externalSkyWcsName` " 

101 "field to determine which calibration to load.") 

102 ) 

103 useGlobalExternalSkyWcs = pexConfig.Field( 

104 dtype=bool, 

105 default=False, 

106 doc=("When using doApplyExternalSkyWcs, use 'global' calibrations " 

107 "that are not run per-tract. When False, use per-tract wcs " 

108 "files.") 

109 ) 

110 externalSkyWcsName = pexConfig.ChoiceField( 

111 # TODO: Remove this config with the removal of Gen2 in DM-20572. 

112 dtype=str, 

113 doc=("Type of external SkyWcs if `doApplyExternalSkyWcs` is True. " 

114 "This field is only used for Gen2 middleware."), 

115 default="jointcal", 

116 allowed={ 

117 "jointcal": "Use jointcal_wcs" 

118 } 

119 ) 

120 includeCalibVar = pexConfig.Field( 

121 dtype=bool, 

122 doc="Add photometric calibration variance to warp variance plane.", 

123 default=False 

124 ) 

125 matchingKernelSize = pexConfig.Field( 125 ↛ exitline 125 didn't jump to the function exit

126 dtype=int, 

127 doc="Size in pixels of matching kernel. Must be odd.", 

128 default=21, 

129 check=lambda x: x % 2 == 1 

130 ) 

131 

132 

133class CoaddTaskRunner(pipeBase.TaskRunner): 

134 

135 @staticmethod 

136 def getTargetList(parsedCmd, **kwargs): 

137 return pipeBase.TaskRunner.getTargetList(parsedCmd, selectDataList=parsedCmd.selectId.dataList, 

138 **kwargs) 

139 

140 

141class CoaddBaseTask(pipeBase.CmdLineTask, pipeBase.PipelineTask): 

142 """!Base class for coaddition. 

143 

144 Subclasses must specify _DefaultName 

145 """ 

146 ConfigClass = CoaddBaseConfig 

147 RunnerClass = CoaddTaskRunner 

148 

149 def __init__(self, **kwargs): 

150 super().__init__(**kwargs) 

151 self.makeSubtask("select") 

152 self.makeSubtask("inputRecorder") 

153 

154 def selectExposures(self, patchRef, skyInfo=None, selectDataList=[]): 

155 """! 

156 @brief Select exposures to coadd 

157 

158 Get the corners of the bbox supplied in skyInfo using @ref geom.Box2D and convert the pixel 

159 positions of the bbox corners to sky coordinates using 

160 @ref afw::geom::SkyWcs::pixelToSky "skyInfo.wcs.pixelToSky". Use the 

161 @ref selectImages::WcsSelectImagesTask "WcsSelectImagesTask" to select exposures that lie 

162 inside the patch indicated by the dataRef. 

163 

164 @param[in] patchRef data reference for sky map patch. Must include keys "tract", "patch", 

165 plus the camera-specific filter key (e.g. "filter" or "band") 

166 @param[in] skyInfo geometry for the patch; output from getSkyInfo 

167 @param[in] selectDataList list of @ref selectImages::SelectStruct "SelectStruct" 

168 to consider for selection 

169 @return a list of science exposures to coadd, as butler data references 

170 """ 

171 if skyInfo is None: 

172 skyInfo = self.getSkyInfo(patchRef) 

173 cornerPosList = geom.Box2D(skyInfo.bbox).getCorners() 

174 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList] 

175 return self.select.runDataRef(patchRef, coordList, selectDataList=selectDataList).dataRefList 

176 

177 def getSkyInfo(self, patchRef): 

178 """! 

179 @brief Use @ref coaddBase::getSkyInfo "getSkyInfo" to return the skyMap, 

180 tract and patch information, wcs and the outer bbox 

181 of the patch. 

182 

183 @param[in] patchRef data reference for sky map. Must include keys "tract" and "patch" 

184 

185 @return pipe_base Struct containing: 

186 - skyMap: sky map 

187 - tractInfo: information for chosen tract of sky map 

188 - patchInfo: information about chosen patch of tract 

189 - wcs: WCS of tract 

190 - bbox: outer bbox of patch, as an geom Box2I 

191 """ 

192 return getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef) 

193 

194 def getCoaddDatasetName(self, warpType="direct"): 

195 """Return coadd name for given warpType and task config 

196 

197 Parameters 

198 ---------- 

199 warpType : string 

200 Either 'direct' or 'psfMatched' 

201 

202 Returns 

203 ------- 

204 CoaddDatasetName : `string` 

205 """ 

206 return self.config.coaddName + "Coadd" + makeCoaddSuffix(warpType) 

207 

208 def getTempExpDatasetName(self, warpType="direct"): 

209 """Return warp name for given warpType and task config 

210 

211 Parameters 

212 ---------- 

213 warpType : string 

214 Either 'direct' or 'psfMatched' 

215 

216 Returns 

217 ------- 

218 WarpDatasetName : `string` 

219 """ 

220 return self.config.coaddName + "Coadd_" + warpType + "Warp" 

221 

222 @classmethod 

223 def _makeArgumentParser(cls): 

224 """Create an argument parser 

225 """ 

226 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

227 parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2", 

228 ContainerClass=CoaddDataIdContainer) 

229 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

230 ContainerClass=SelectDataIdContainer) 

231 return parser 

232 

233 def _getConfigName(self): 

234 """Return the name of the config dataset 

235 """ 

236 return "%s_%s_config" % (self.config.coaddName, self._DefaultName) 

237 

238 def _getMetadataName(self): 

239 """Return the name of the metadata dataset 

240 """ 

241 return "%s_%s_metadata" % (self.config.coaddName, self._DefaultName) 

242 

243 def getBadPixelMask(self): 

244 """! 

245 @brief Convenience method to provide the bitmask from the mask plane names 

246 """ 

247 return afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes) 

248 

249 

250class SelectDataIdContainer(pipeBase.DataIdContainer): 

251 """! 

252 @brief A dataId container for inputs to be selected. 

253 

254 Read the header (including the size and Wcs) for all specified 

255 inputs and pass those along, ultimately for the SelectImagesTask. 

256 This is most useful when used with multiprocessing, as input headers are 

257 only read once. 

258 """ 

259 

260 def makeDataRefList(self, namespace): 

261 """Add a dataList containing useful information for selecting images""" 

262 super(SelectDataIdContainer, self).makeDataRefList(namespace) 

263 self.dataList = [] 

264 for ref in self.refList: 

265 try: 

266 md = ref.get("calexp_md", immediate=True) 

267 wcs = afwGeom.makeSkyWcs(md) 

268 data = SelectStruct(dataRef=ref, wcs=wcs, bbox=afwImage.bboxFromMetadata(md)) 

269 except FitsError: 

270 namespace.log.warning("Unable to construct Wcs from %s", ref.dataId) 

271 continue 

272 self.dataList.append(data) 

273 

274 

275def getSkyInfo(coaddName, patchRef): 

276 """! 

277 @brief Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded. 

278 

279 @param[in] coaddName coadd name; typically one of deep or goodSeeing 

280 @param[in] patchRef data reference for sky map. Must include keys "tract" and "patch" 

281 

282 @return pipe_base Struct containing: 

283 - skyMap: sky map 

284 - tractInfo: information for chosen tract of sky map 

285 - patchInfo: information about chosen patch of tract 

286 - wcs: WCS of tract 

287 - bbox: outer bbox of patch, as an geom Box2I 

288 """ 

289 skyMap = patchRef.get(coaddName + "Coadd_skyMap") 

290 return makeSkyInfo(skyMap, patchRef.dataId["tract"], patchRef.dataId["patch"]) 

291 

292 

293def makeSkyInfo(skyMap, tractId, patchId): 

294 """Return SkyInfo Struct 

295 

296 Constructs SkyInfo used by coaddition tasks for multiple 

297 patchId formats. 

298 

299 Parameters 

300 ---------- 

301 skyMap : `lsst.skyMap.SkyMap` 

302 tractId : int 

303 patchId : str or int or tuple of int 

304 Either Gen2-style comma delimited string (e.g. '4,5'), 

305 tuple of integers (e.g (4, 5), Gen3-style integer. 

306 """ 

307 tractInfo = skyMap[tractId] 

308 

309 if isinstance(patchId, str) and ',' in patchId: 

310 # patch format is "xIndex,yIndex" 

311 patchIndex = tuple(int(i) for i in patchId.split(",")) 

312 else: 

313 patchIndex = patchId 

314 

315 patchInfo = tractInfo.getPatchInfo(patchIndex) 

316 

317 return pipeBase.Struct( 

318 skyMap=skyMap, 

319 tractInfo=tractInfo, 

320 patchInfo=patchInfo, 

321 wcs=tractInfo.getWcs(), 

322 bbox=patchInfo.getOuterBBox(), 

323 ) 

324 

325 

326def scaleVariance(maskedImage, maskPlanes, log=None): 

327 """! 

328 @brief Scale the variance in a maskedImage 

329 

330 The variance plane in a convolved or warped image (or a coadd derived 

331 from warped images) does not accurately reflect the noise properties of 

332 the image because variance has been lost to covariance. This function 

333 attempts to correct for this by scaling the variance plane to match 

334 the observed variance in the image. This is not perfect (because we're 

335 not tracking the covariance) but it's simple and is often good enough. 

336 

337 @deprecated Use the ScaleVarianceTask instead. 

338 

339 @param maskedImage MaskedImage to operate on; variance will be scaled 

340 @param maskPlanes List of mask planes for pixels to reject 

341 @param log Log for reporting the renormalization factor; or None 

342 @return renormalisation factor 

343 """ 

344 config = ScaleVarianceTask.ConfigClass() 

345 config.maskPlanes = maskPlanes 

346 task = ScaleVarianceTask(config=config, name="scaleVariance", log=log) 

347 return task.run(maskedImage) 

348 

349 

350def makeCoaddSuffix(warpType="direct"): 

351 """Return coadd suffix for warpType 

352 

353 Parameters 

354 ---------- 

355 warpType : string 

356 Either 'direct' or 'psfMatched' 

357 

358 Returns 

359 ------- 

360 CoaddSuffix : `string` 

361 """ 

362 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:] 

363 return suffix 

364 

365 

366def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None): 

367 """Match the order of one list to another, padding if necessary 

368 

369 Parameters 

370 ---------- 

371 inputList : list 

372 List to be reordered and padded. Elements can be any type. 

373 inputKeys : iterable 

374 Iterable of values to be compared with outputKeys. 

375 Length must match `inputList` 

376 outputKeys : iterable 

377 Iterable of values to be compared with inputKeys. 

378 padWith : 

379 Any value to be inserted where inputKey not in outputKeys 

380 

381 Returns 

382 ------- 

383 list 

384 Copy of inputList reordered per outputKeys and padded with `padWith` 

385 so that the length matches length of outputKeys. 

386 """ 

387 outputList = [] 

388 for d in outputKeys: 

389 if d in inputKeys: 

390 outputList.append(inputList[inputKeys.index(d)]) 

391 else: 

392 outputList.append(padWith) 

393 return outputList