Coverage for python/lsst/summit/utils/bestEffort.py: 17%

85 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-18 02:50 -0700

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from sqlite3 import OperationalError 

23 

24import logging 

25from lsst.ip.isr import IsrTask 

26import lsst.daf.butler as dafButler 

27from lsst.daf.butler.registry import ConflictingDefinitionError 

28 

29from lsst.summit.utils.quickLook import QuickLookIsrTask 

30from lsst.summit.utils.butlerUtils import getLatissDefaultCollections 

31 

32# TODO: add attempt for fringe once registry & templates are fixed 

33 

34CURRENT_RUN = "LATISS/runs/quickLook/1" 

35ALLOWED_REPOS = ['/repo/main', '/repo/LATISS', '/readonly/repo/main'] 

36 

37 

38class BestEffortIsr(): 

39 """Class for getting an assembled image with the maximum amount of isr. 

40 

41 BestEffortIsr.getExposure(dataId) returns an assembled image with as much 

42 isr performed as possible, dictated by the calibration products available, 

43 and optionally interpolates over cosmic rays. If an image image already 

44 exists in the butler, it is returned (for the sake of speed), otherwise it 

45 is generated and put(). Calibration products are loaded and cached to 

46 improve performance. 

47 

48 This class uses the ``quickLookIsrTask``, see docs there for details. 

49 

50 Acceptable repodir values are currently listed in ALLOWED_REPOS. This will 

51 be updated (removed) once DM-33849 is done. 

52 

53 defaultExtraIsrOptions is a dict of options applied to all images. 

54 

55 Parameters 

56 ---------- 

57 repoDir : `str` 

58 The repo root. Will be removed after DM-33849. 

59 extraCollections : `list` of `str`, optional 

60 Extra collections to add to the butler init. Collections are prepended. 

61 defaultExtraIsrOptions : `dict`, optional 

62 A dict of extra isr config options to apply. Each key should be an 

63 attribute of an isrTaskConfigClass. 

64 doRepairCosmics : `bool`, optional 

65 Repair cosmic ray hits? 

66 doWrite : `bool`, optional 

67 Write the outputs to the quickLook rerun/collection? 

68 

69 Raises 

70 ------ 

71 FileNotFoundError: 

72 Raised when a butler cannot be automatically instantiated using 

73 the DAF_BUTLER_REPOSITORY_INDEX environment variable. 

74 """ 

75 _datasetName = 'quickLookExp' 

76 

77 def __init__(self, *, 

78 extraCollections=[], defaultExtraIsrOptions={}, doRepairCosmics=True, doWrite=True): 

79 self.log = logging.getLogger(__name__) 

80 

81 collections = getLatissDefaultCollections() 

82 self.collections = extraCollections + collections 

83 self.log.info(f'Instantiating butler with collections={self.collections}') 

84 try: 

85 self.butler = dafButler.Butler('LATISS', collections=self.collections, 

86 instrument='LATISS', 

87 run=CURRENT_RUN if doWrite else None) 

88 except(FileNotFoundError, RuntimeError): 

89 # Depending on the value of DAF_BUTLER_REPOSITORY_INDEX and whether 

90 # it is present and blank, or just not set, both these exception 

91 # types can be raised, see 

92 # tests/test_butlerUtils.py:ButlerInitTestCase 

93 # for details and tests which confirm these have not changed 

94 raise FileNotFoundError # unify exception type 

95 

96 quickLookIsrConfig = QuickLookIsrTask.ConfigClass() 

97 quickLookIsrConfig.doRepairCosmics = doRepairCosmics 

98 self.doWrite = doWrite # the task, as run by run() method, can't do the write, so we handle in here 

99 self.quickLookIsrTask = QuickLookIsrTask(config=quickLookIsrConfig) 

100 

101 self.defaultExtraIsrOptions = defaultExtraIsrOptions 

102 

103 self._cache = {} 

104 

105 def _applyConfigOverrides(self, config, overrides): 

106 """Update a config class with a dict of options. 

107 

108 Parameters 

109 ---------- 

110 config : `lsst.pex.config.Config` 

111 The config class to update. 

112 overrides : `dict` 

113 The override options as a dict. 

114 

115 Raises 

116 ------ 

117 ValueError 

118 Raised if the override option isn't found in the config. 

119 """ 

120 for option, value in overrides.items(): 

121 if hasattr(config, option): 

122 setattr(config, option, value) 

123 self.log.info(f"Set isr config override {option} to {value}") 

124 else: 

125 raise ValueError(f"Override option {option} not found in isrConfig") 

126 

127 @staticmethod 

128 def _parseExpIdOrDataId(expIdOrDataId, **kwargs): 

129 """Sanitize the expIdOrDataId to allow support both expIds and dataIds 

130 

131 Supports expId as an integer, or a complete or partial dict. The dict 

132 is updated with the supplied kwargs. 

133 

134 Parameters 

135 ---------- 

136 expIdOrDataId : `int` or `dict 

137 The exposure id as an int or the dataId as as dict. 

138 

139 Returns 

140 ------- 

141 dataId : `dict` 

142 The sanitized dataId. 

143 """ 

144 if type(expIdOrDataId) == int: 

145 _dataId = {'expId': expIdOrDataId} 

146 elif type(expIdOrDataId) == dict: 

147 _dataId = expIdOrDataId 

148 _dataId.update(kwargs) 

149 else: 

150 raise RuntimeError(f"Invalid expId or dataId type {expIdOrDataId}") 

151 return _dataId 

152 

153 def clearCache(self): 

154 """Clear the internal cache of loaded calibration products. 

155 

156 Only necessary if you want to use an existing bestEffortIsr object 

157 after adding new calibration products to the calibration collection. 

158 """ 

159 self._cache = {} 

160 

161 def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, **kwargs): 

162 """Get the postIsr and cosmic-repaired image for this dataId. 

163 

164 Parameters 

165 ---------- 

166 expIdOrDataId : `dict` 

167 The dataId 

168 extraIsrOptions : `dict`, optional 

169 extraIsrOptions is a dict of extra isr options applied to this 

170 image only. 

171 skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK! 

172 Skip doing cosmic ray repair for this image? 

173 

174 Returns 

175 ------- 

176 exp : `lsst.afw.image.Exposure` 

177 The postIsr exposure 

178 """ 

179 dataId = self._parseExpIdOrDataId(expIdOrDataId, **kwargs) 

180 

181 try: 

182 exp = self.butler.get(self._datasetName, dataId=dataId) 

183 self.log.info("Found a ready-made quickLookExp in the repo. Returning that.") 

184 return exp 

185 except LookupError: 

186 pass 

187 

188 try: 

189 raw = self.butler.get('raw', dataId=dataId) 

190 except LookupError: 

191 raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None 

192 

193 # default options that are probably good for most engineering time 

194 isrConfig = IsrTask.ConfigClass() 

195 isrConfig.doWrite = False # this task writes separately, no need for this 

196 isrConfig.doSaturation = True # saturation very important for roundness measurement in qfm 

197 isrConfig.doSaturationInterpolation = True 

198 isrConfig.overscan.leadingColumnsToSkip = 5 

199 isrConfig.overscan.fitType = 'MEDIAN_PER_ROW' 

200 

201 # apply general overrides 

202 self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions) 

203 # apply per-image overrides 

204 self._applyConfigOverrides(isrConfig, extraIsrOptions) 

205 

206 isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel', 

207 'bfGains', 'ptc'] 

208 

209 isrDict = {} 

210 # we build a cache of all the isr components which will be used to save 

211 # the IO time on subsequent calls. This assumes people will not update 

212 # calibration products while this object lives, but this is a fringe 

213 # use case, and if they do, all they would need to do would be call 

214 # .clearCache() and this will rebuild with the new products. 

215 for component in isrParts: 

216 if component in self._cache and component != 'flat': 

217 self.log.info(f"Using {component} from cache...") 

218 isrDict[component] = self._cache[component] 

219 continue 

220 try: 

221 # TODO: add caching for flats 

222 item = self.butler.get(component, dataId=dataId) 

223 self._cache[component] = item 

224 isrDict[component] = self._cache[component] 

225 except (RuntimeError, LookupError, OperationalError): 

226 pass 

227 

228 quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure 

229 

230 if self.doWrite: 

231 try: 

232 self.butler.put(quickLookExp, self._datasetName, dataId) 

233 self.log.info(f'Put {self._datasetName} for {dataId}') 

234 except ConflictingDefinitionError: 

235 # TODO: DM-34302 fix this message so that it's less scary for 

236 # users. Do this by having daemons know they're daemons. 

237 self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)') 

238 pass 

239 

240 return quickLookExp