Coverage for python/lsst/summit/utils/bestEffort.py: 17%

87 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-09 13:06 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from sqlite3 import OperationalError 

23 

24import logging 

25from lsst.ip.isr import IsrTask 

26import lsst.daf.butler as dafButler 

27from lsst.daf.butler.registry import ConflictingDefinitionError 

28 

29from lsst.summit.utils.quickLook import QuickLookIsrTask 

30from lsst.summit.utils.butlerUtils import getLatissDefaultCollections 

31 

32# TODO: add attempt for fringe once registry & templates are fixed 

33 

34CURRENT_RUN = "LATISS/runs/quickLook/1" 

35ALLOWED_REPOS = ['/repo/main', '/repo/LATISS', '/readonly/repo/main'] 

36 

37 

38class BestEffortIsr(): 

39 """Class for getting an assembled image with the maximum amount of isr. 

40 

41 BestEffortIsr.getExposure(dataId) returns an assembled image with as much 

42 isr performed as possible, dictated by the calibration products available, 

43 and optionally interpolates over cosmic rays. If an image image already 

44 exists in the butler, it is returned (for the sake of speed), otherwise it 

45 is generated and put(). Calibration products are loaded and cached to 

46 improve performance. 

47 

48 This class uses the ``quickLookIsrTask``, see docs there for details. 

49 

50 Acceptable repodir values are currently listed in ALLOWED_REPOS. This will 

51 be updated (removed) once DM-33849 is done. 

52 

53 defaultExtraIsrOptions is a dict of options applied to all images. 

54 

55 Parameters 

56 ---------- 

57 repoDir : `str` 

58 The repo root. Will be removed after DM-33849. 

59 extraCollections : `list` of `str`, optional 

60 Extra collections to add to the butler init. Collections are prepended. 

61 defaultExtraIsrOptions : `dict`, optional 

62 A dict of extra isr config options to apply. Each key should be an 

63 attribute of an isrTaskConfigClass. 

64 doRepairCosmics : `bool`, optional 

65 Repair cosmic ray hits? 

66 doWrite : `bool`, optional 

67 Write the outputs to the quickLook rerun/collection? 

68 

69 Raises 

70 ------ 

71 FileNotFoundError: 

72 Raised when a butler cannot be automatically instantiated using 

73 the DAF_BUTLER_REPOSITORY_INDEX environment variable. 

74 """ 

75 _datasetName = 'quickLookExp' 

76 

77 def __init__(self, *, 

78 extraCollections=[], 

79 defaultExtraIsrOptions={}, 

80 doRepairCosmics=True, 

81 doWrite=True, 

82 embargo=False): 

83 self.log = logging.getLogger(__name__) 

84 

85 collections = getLatissDefaultCollections() 

86 self.collections = extraCollections + collections 

87 self.log.info(f'Instantiating butler with collections={self.collections}') 

88 try: 

89 repoString = "LATISS" if not embargo else "/repo/embargo" 

90 self.butler = dafButler.Butler(repoString, collections=self.collections, 

91 instrument='LATISS', 

92 run=CURRENT_RUN if doWrite else None) 

93 except(FileNotFoundError, RuntimeError): 

94 # Depending on the value of DAF_BUTLER_REPOSITORY_INDEX and whether 

95 # it is present and blank, or just not set, both these exception 

96 # types can be raised, see 

97 # tests/test_butlerUtils.py:ButlerInitTestCase 

98 # for details and tests which confirm these have not changed 

99 raise FileNotFoundError # unify exception type 

100 

101 quickLookIsrConfig = QuickLookIsrTask.ConfigClass() 

102 quickLookIsrConfig.doRepairCosmics = doRepairCosmics 

103 self.doWrite = doWrite # the task, as run by run() method, can't do the write, so we handle in here 

104 self.quickLookIsrTask = QuickLookIsrTask(config=quickLookIsrConfig) 

105 

106 self.defaultExtraIsrOptions = defaultExtraIsrOptions 

107 

108 self._cache = {} 

109 

110 def _applyConfigOverrides(self, config, overrides): 

111 """Update a config class with a dict of options. 

112 

113 Parameters 

114 ---------- 

115 config : `lsst.pex.config.Config` 

116 The config class to update. 

117 overrides : `dict` 

118 The override options as a dict. 

119 

120 Raises 

121 ------ 

122 ValueError 

123 Raised if the override option isn't found in the config. 

124 """ 

125 for option, value in overrides.items(): 

126 if hasattr(config, option): 

127 setattr(config, option, value) 

128 self.log.info(f"Set isr config override {option} to {value}") 

129 else: 

130 raise ValueError(f"Override option {option} not found in isrConfig") 

131 

132 @staticmethod 

133 def _parseExpIdOrDataId(expIdOrDataId, **kwargs): 

134 """Sanitize the expIdOrDataId to allow support both expIds and dataIds 

135 

136 Supports expId as an integer, or a complete or partial dict. The dict 

137 is updated with the supplied kwargs. 

138 

139 Parameters 

140 ---------- 

141 expIdOrDataId : `int` or `dict 

142 The exposure id as an int or the dataId as as dict. 

143 

144 Returns 

145 ------- 

146 dataId : `dict` 

147 The sanitized dataId. 

148 """ 

149 if type(expIdOrDataId) == int: 

150 _dataId = {'expId': expIdOrDataId} 

151 elif type(expIdOrDataId) == dict: 

152 _dataId = expIdOrDataId 

153 _dataId.update(kwargs) 

154 else: 

155 raise RuntimeError(f"Invalid expId or dataId type {expIdOrDataId}") 

156 return _dataId 

157 

158 def clearCache(self): 

159 """Clear the internal cache of loaded calibration products. 

160 

161 Only necessary if you want to use an existing bestEffortIsr object 

162 after adding new calibration products to the calibration collection. 

163 """ 

164 self._cache = {} 

165 

166 def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, forceRemake=False, 

167 **kwargs): 

168 """Get the postIsr and cosmic-repaired image for this dataId. 

169 

170 Note that when using the forceRemake option the image will not be 

171 written to the repo for reuse. 

172 

173 Parameters 

174 ---------- 

175 expIdOrDataId : `dict` 

176 The dataId 

177 extraIsrOptions : `dict`, optional 

178 extraIsrOptions is a dict of extra isr options applied to this 

179 image only. 

180 skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK! 

181 Skip doing cosmic ray repair for this image? 

182 forceRemake : `bool` 

183 Remake the exposure even if there is a pre-existing one in the 

184 repo. Images that are force-remade are never written, as this is 

185 assumed to be used for testing/debug purposes, as opposed to normal 

186 operation. For updating individual images, removal from the 

187 registry can be used, and for bulk-updates the overall run number 

188 can be incremented. 

189 

190 Returns 

191 ------- 

192 exp : `lsst.afw.image.Exposure` 

193 The postIsr exposure 

194 """ 

195 dataId = self._parseExpIdOrDataId(expIdOrDataId, **kwargs) 

196 

197 if not forceRemake: 

198 try: 

199 exp = self.butler.get(self._datasetName, dataId=dataId) 

200 self.log.info("Found a ready-made quickLookExp in the repo. Returning that.") 

201 return exp 

202 except LookupError: 

203 pass 

204 

205 try: 

206 raw = self.butler.get('raw', dataId=dataId) 

207 except LookupError: 

208 raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None 

209 

210 # default options that are probably good for most engineering time 

211 isrConfig = IsrTask.ConfigClass() 

212 isrConfig.doWrite = False # this task writes separately, no need for this 

213 isrConfig.doSaturation = True # saturation very important for roundness measurement in qfm 

214 isrConfig.doSaturationInterpolation = True 

215 isrConfig.overscan.leadingColumnsToSkip = 5 

216 isrConfig.overscan.fitType = 'MEDIAN_PER_ROW' 

217 

218 # apply general overrides 

219 self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions) 

220 # apply per-image overrides 

221 self._applyConfigOverrides(isrConfig, extraIsrOptions) 

222 

223 isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel', 

224 'bfGains', 'ptc'] 

225 

226 isrDict = {} 

227 # we build a cache of all the isr components which will be used to save 

228 # the IO time on subsequent calls. This assumes people will not update 

229 # calibration products while this object lives, but this is a fringe 

230 # use case, and if they do, all they would need to do would be call 

231 # .clearCache() and this will rebuild with the new products. 

232 for component in isrParts: 

233 if component in self._cache and component != 'flat': 

234 self.log.info(f"Using {component} from cache...") 

235 isrDict[component] = self._cache[component] 

236 continue 

237 try: 

238 # TODO: add caching for flats 

239 item = self.butler.get(component, dataId=dataId) 

240 self._cache[component] = item 

241 isrDict[component] = self._cache[component] 

242 except (RuntimeError, LookupError, OperationalError): 

243 pass 

244 

245 quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure 

246 

247 if self.doWrite and not forceRemake: 

248 try: 

249 self.butler.put(quickLookExp, self._datasetName, dataId) 

250 self.log.info(f'Put {self._datasetName} for {dataId}') 

251 except ConflictingDefinitionError: 

252 # TODO: DM-34302 fix this message so that it's less scary for 

253 # users. Do this by having daemons know they're daemons. 

254 self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)') 

255 pass 

256 

257 return quickLookExp