Coverage for python/lsst/summit/utils/bestEffort.py: 14%

101 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-14 11:17 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import logging 

24from lsst.utils import getPackageDir 

25from lsst.ip.isr import IsrTask 

26import lsst.daf.butler as dafButler 

27from lsst.daf.butler.registry import ConflictingDefinitionError 

28 

29from lsst.summit.utils.quickLook import QuickLookIsrTask 

30from lsst.summit.utils.butlerUtils import getLatissDefaultCollections 

31 

32# TODO: add attempt for fringe once registry & templates are fixed 

33 

34CURRENT_RUN = "LATISS/runs/quickLook/1" 

35ALLOWED_REPOS = ['/repo/main', '/repo/LATISS', '/readonly/repo/main'] 

36 

37 

38class BestEffortIsr(): 

39 """Class for getting an assembled image with the maximum amount of isr. 

40 

41 BestEffortIsr.getExposure(dataId) returns an assembled image with as much 

42 isr performed as possible, dictated by the calibration products available, 

43 and optionally interpolates over cosmic rays. If an image image already 

44 exists in the butler, it is returned (for the sake of speed), otherwise it 

45 is generated and put(). Calibration products are loaded and cached to 

46 improve performance. 

47 

48 This class uses the ``quickLookIsrTask``, see docs there for details. 

49 

50 Acceptable repodir values are currently listed in ALLOWED_REPOS. This will 

51 be updated (removed) once DM-33849 is done. 

52 

53 defaultExtraIsrOptions is a dict of options applied to all images. 

54 

55 Parameters 

56 ---------- 

57 repoDir : `str` 

58 The repo root. Will be removed after DM-33849. 

59 extraCollections : `list` of `str`, optional 

60 Extra collections to add to the butler init. Collections are prepended. 

61 defaultExtraIsrOptions : `dict`, optional 

62 A dict of extra isr config options to apply. Each key should be an 

63 attribute of an isrTaskConfigClass. 

64 doRepairCosmics : `bool`, optional 

65 Repair cosmic ray hits? 

66 doWrite : `bool`, optional 

67 Write the outputs to the quickLook rerun/collection? 

68 

69 Raises 

70 ------ 

71 FileNotFoundError: 

72 Raised when a butler cannot be automatically instantiated using 

73 the DAF_BUTLER_REPOSITORY_INDEX environment variable. 

74 """ 

75 _datasetName = 'quickLookExp' 

76 

77 def __init__(self, *, 

78 extraCollections=[], 

79 defaultExtraIsrOptions={}, 

80 doRepairCosmics=True, 

81 doWrite=True, 

82 embargo=False): 

83 self.log = logging.getLogger(__name__) 

84 

85 collections = getLatissDefaultCollections() 

86 self.collections = extraCollections + collections 

87 self.log.info(f'Instantiating butler with collections={self.collections}') 

88 try: 

89 repoString = "LATISS" if not embargo else "/repo/embargo" 

90 self.butler = dafButler.Butler(repoString, collections=self.collections, 

91 instrument='LATISS', 

92 run=CURRENT_RUN if doWrite else None) 

93 except (FileNotFoundError, RuntimeError): 

94 # Depending on the value of DAF_BUTLER_REPOSITORY_INDEX and whether 

95 # it is present and blank, or just not set, both these exception 

96 # types can be raised, see 

97 # tests/test_butlerUtils.py:ButlerInitTestCase 

98 # for details and tests which confirm these have not changed 

99 raise FileNotFoundError # unify exception type 

100 

101 quickLookIsrConfig = QuickLookIsrTask.ConfigClass() 

102 quickLookIsrConfig.doRepairCosmics = doRepairCosmics 

103 self.doWrite = doWrite # the task, as run by run() method, can't do the write, so we handle in here 

104 self.quickLookIsrTask = QuickLookIsrTask(config=quickLookIsrConfig) 

105 

106 self.defaultExtraIsrOptions = defaultExtraIsrOptions 

107 

108 self._cache = {} 

109 self._cacheIsForDetector = None 

110 

111 def _applyConfigOverrides(self, config, overrides): 

112 """Update a config class with a dict of options. 

113 

114 Parameters 

115 ---------- 

116 config : `lsst.pex.config.Config` 

117 The config class to update. 

118 overrides : `dict` 

119 The override options as a dict. 

120 

121 Raises 

122 ------ 

123 ValueError 

124 Raised if the override option isn't found in the config. 

125 """ 

126 for option, value in overrides.items(): 

127 if hasattr(config, option): 

128 setattr(config, option, value) 

129 self.log.info(f"Set isr config override {option} to {value}") 

130 else: 

131 raise ValueError(f"Override option {option} not found in isrConfig") 

132 

133 @staticmethod 

134 def updateDataId(expIdOrDataId, **kwargs): 

135 """Sanitize the expIdOrDataId to allow support both expIds and dataIds 

136 

137 Supports expId as an integer, or a complete or partial dict. The dict 

138 is updated with the supplied kwargs. 

139 

140 Parameters 

141 ---------- 

142 expIdOrDataId : `int` or `dict` or `lsst.daf.butler.DataCoordinate` or 

143 `lsst.daf.butler.DimensionRecord` 

144 The exposure id as an int, or the dataId as as dict, or an 

145 expRecord or a dataCoordinate. 

146 

147 Returns 

148 ------- 

149 dataId : `dict` 

150 The sanitized dataId. 

151 """ 

152 match expIdOrDataId: 

153 case int() as expId: 

154 dataId = {"expId": expId} 

155 dataId.update(**kwargs) 

156 return dataId 

157 case dafButler.DataCoordinate() as dataId: 

158 return dafButler.DataCoordinate.standardize(dataId, **kwargs) 

159 case dafButler.DimensionRecord() as record: 

160 return dafButler.DataCoordinate.standardize(record.dataId, **kwargs) 

161 case dict() as dataId: 

162 dataId.update(**kwargs) 

163 return dataId 

164 raise RuntimeError(f"Invalid expId or dataId type {expIdOrDataId}: {type(expIdOrDataId)}") 

165 

166 def clearCache(self): 

167 """Clear the internal cache of loaded calibration products. 

168 

169 Only necessary if you want to use an existing bestEffortIsr object 

170 after adding new calibration products to the calibration collection. 

171 """ 

172 self._cache = {} 

173 

174 def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, forceRemake=False, 

175 **kwargs): 

176 """Get the postIsr and cosmic-repaired image for this dataId. 

177 

178 Note that when using the forceRemake option the image will not be 

179 written to the repo for reuse. 

180 

181 Parameters 

182 ---------- 

183 expIdOrDataId : `dict` 

184 The dataId 

185 extraIsrOptions : `dict`, optional 

186 extraIsrOptions is a dict of extra isr options applied to this 

187 image only. 

188 skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK! 

189 Skip doing cosmic ray repair for this image? 

190 forceRemake : `bool` 

191 Remake the exposure even if there is a pre-existing one in the 

192 repo. Images that are force-remade are never written, as this is 

193 assumed to be used for testing/debug purposes, as opposed to normal 

194 operation. For updating individual images, removal from the 

195 registry can be used, and for bulk-updates the overall run number 

196 can be incremented. 

197 

198 Returns 

199 ------- 

200 exp : `lsst.afw.image.Exposure` 

201 The postIsr exposure 

202 """ 

203 dataId = self.updateDataId(expIdOrDataId, **kwargs) 

204 if 'detector' not in dataId: 

205 raise ValueError('dataId must contain a detector. Either specify a detector as a kwarg,' 

206 ' or use a fully-qualified dataId') 

207 

208 if not forceRemake: 

209 try: 

210 exp = self.butler.get(self._datasetName, dataId) 

211 self.log.info("Found a ready-made quickLookExp in the repo. Returning that.") 

212 return exp 

213 except LookupError: 

214 pass 

215 

216 try: 

217 raw = self.butler.get('raw', dataId) 

218 except LookupError: 

219 raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None 

220 

221 # default options that are probably good for most engineering time 

222 isrConfig = IsrTask.ConfigClass() 

223 packageDir = getPackageDir("summit_utils") 

224 isrConfig.load(os.path.join(packageDir, "config", "quickLookIsr.py")) 

225 

226 # apply general overrides 

227 self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions) 

228 # apply per-image overrides 

229 self._applyConfigOverrides(isrConfig, extraIsrOptions) 

230 

231 isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel', 

232 'bfGains', 'ptc'] 

233 

234 if self._cacheIsForDetector != dataId['detector']: 

235 self.clearCache() 

236 self._cacheIsForDetector = dataId['detector'] 

237 

238 isrDict = {} 

239 # we build a cache of all the isr components which will be used to save 

240 # the IO time on subsequent calls. This assumes people will not update 

241 # calibration products while this object lives, but this is a fringe 

242 # use case, and if they do, all they would need to do would be call 

243 # .clearCache() and this will rebuild with the new products. 

244 for component in isrParts: 

245 if component in self._cache and component != 'flat': 

246 self.log.info(f"Using {component} from cache...") 

247 isrDict[component] = self._cache[component] 

248 continue 

249 if self.butler.exists(component, dataId): 

250 try: 

251 # TODO: add caching for flats 

252 item = self.butler.get(component, dataId=dataId) 

253 self._cache[component] = item 

254 isrDict[component] = self._cache[component] 

255 self.log.info(f"Loaded {component} to cache") 

256 except Exception: # now that we log the exception, we can catch all errors 

257 # the product *should* exist but the get() failed, so log 

258 # a very loud warning inc. the traceback as this is a sign 

259 # of butler/database failures or something like that. 

260 self.log.critical(f'Failed to find expected data product {component}!') 

261 self.log.exception(f'Finding failure for {component}:') 

262 else: 

263 self.log.debug('No %s found for %s', component, dataId) 

264 

265 quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure 

266 

267 if self.doWrite and not forceRemake: 

268 try: 

269 self.butler.put(quickLookExp, self._datasetName, dataId) 

270 self.log.info(f'Put {self._datasetName} for {dataId}') 

271 except ConflictingDefinitionError: 

272 # TODO: DM-34302 fix this message so that it's less scary for 

273 # users. Do this by having daemons know they're daemons. 

274 self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)') 

275 pass 

276 

277 return quickLookExp