Coverage for python/lsst/obs/base/butler_tests.py: 15%

104 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2024-01-25 04:31 +0000

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import abc 

23import collections 

24import inspect 

25import unittest 

26 

27from lsst.daf.butler import Butler 

28from lsst.daf.butler.registry import DataIdValueError 

29 

30__all__ = ["ButlerGetTests"] 

31 

32 

33class ButlerGetTests(metaclass=abc.ABCMeta): 

34 """Tests of obs_* Butler get() functionality. 

35 

36 In the subclasses's setUp(): 

37 * Call setUp_butler_get() to fill in required parameters. 

38 """ 

39 

40 def setUp_butler_get( 

41 self, 

42 ccdExposureId_bits=None, 

43 exposureIds=None, 

44 filters=None, 

45 exptimes=None, 

46 detectorIds=None, 

47 detector_names=None, 

48 detector_serials=None, 

49 dimensions=None, 

50 sky_origin=None, 

51 raw_subsets=None, 

52 good_detectorIds=None, 

53 bad_detectorIds=None, 

54 linearizer_type=None, 

55 raw_header_wcs=None, 

56 ): 

57 """ 

58 Set up the necessary variables for butlerGet tests. 

59 

60 All "exposure name" entries below should correspond to an entry in 

61 self.dataIds. 

62 

63 Parameters 

64 ---------- 

65 

66 ccdExposureId_bits : `int` 

67 expected value of ccdExposureId_bits 

68 exposureIds : `dict` 

69 dict of exposure name : ccdExposureId (the number as returned by 

70 the butler) 

71 filters : `dict` 

72 dict of exposure name : filter name 

73 exptimes : `dict` 

74 dict of exposure name : exposure time 

75 detector_names : `dict` 

76 dict of exposure name : detector name 

77 detectorIds : `dict` 

78 dict of exposure name : detectorId 

79 detector_serials : `dict` 

80 dict of exposure name : detector serial 

81 dimensions : `dict` 

82 dict of exposure name : dimensions (as a geom.Extent2I) 

83 sky_origin : `tuple` of `float` 

84 Longitude, Latitude of 'raw' exposure 

85 raw_subsets : `tuple` of (kwargs, `int`) 

86 keyword args and expected number of subsets for 

87 ``butler.subset('raw', **kwargs)`` 

88 good_detectorIds : `list` of `int` 

89 list of valid ccd numbers 

90 bad_detectorIds : `list` of `int` 

91 list of invalid ccd numbers 

92 linearizer_type : `dict` 

93 dict of detectorId (usually `int`): LinearizerType 

94 (e.g. lsst.ip.isr.LinearizeLookupTable.LinearityType), 

95 or unittest.SkipTest to skip all linearizer tests. 

96 raw_header_wcs : `lsst.afw.geom.SkyWcs` 

97 The SkyWcs object that should be returned by 

98 ``butler.get("raw_header_wcs", dataId=self.dataIds["raw"])`` 

99 """ 

100 

101 fields = [ 

102 "ccdExposureId_bits", 

103 "exposureIds", 

104 "filters", 

105 "exptimes", 

106 "detector_names", 

107 "detectorIds", 

108 "detector_serials", 

109 "dimensions", 

110 "sky_origin", 

111 "raw_subsets", 

112 "good_detectorIds", 

113 "bad_detectorIds", 

114 "linearizer_type", 

115 "raw_header_wcs", 

116 ] 

117 ButlerGet = collections.namedtuple("ButlerGetData", fields) 

118 

119 self.butler_get_data = ButlerGet( 

120 ccdExposureId_bits=ccdExposureId_bits, 

121 exposureIds=exposureIds, 

122 filters=filters, 

123 exptimes=exptimes, 

124 detectorIds=detectorIds, 

125 detector_names=detector_names, 

126 detector_serials=detector_serials, 

127 dimensions=dimensions, 

128 sky_origin=sky_origin, 

129 raw_subsets=raw_subsets, 

130 good_detectorIds=good_detectorIds, 

131 bad_detectorIds=bad_detectorIds, 

132 linearizer_type=linearizer_type, 

133 raw_header_wcs=raw_header_wcs, 

134 ) 

135 

136 def _require_gen2(self): 

137 if isinstance(self.butler, Butler): 

138 self.skipTest("This test requires daf_persistence Butler, not daf_butler Butler.") 

139 

140 def _is_gen3(self): 

141 if isinstance(self.butler, Butler): 

142 return True 

143 return False 

144 

145 def test_exposureId_bits(self): 

146 self._require_gen2() 

147 bits = self.butler.get("ccdExposureId_bits") 

148 self.assertEqual(bits, self.butler_get_data.ccdExposureId_bits) 

149 

150 def _test_exposure(self, name): 

151 if self.dataIds[name] is unittest.SkipTest: 

152 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

153 exp = self.butler.get(name, self.dataIds[name]) 

154 

155 md_component = ".metadata" if self._is_gen3() else "_md" 

156 exp_md = self.butler.get(name + md_component, self.dataIds[name]) 

157 self.assertEqual(type(exp_md), type(exp.getMetadata())) 

158 

159 self.assertEqual(exp.getDimensions(), self.butler_get_data.dimensions[name]) 

160 detector = exp.detector 

161 # Some calibration files are missing the detector. 

162 if detector: 

163 self.assertEqual(detector.getId(), self.butler_get_data.detectorIds[name]) 

164 self.assertEqual(detector.getName(), self.butler_get_data.detector_names[name]) 

165 self.assertEqual(detector.getSerial(), self.butler_get_data.detector_serials[name]) 

166 # obs_test does not have physical filters, so include a fallback 

167 exposureFilter = exp.getFilter() 

168 if exposureFilter: 

169 if exposureFilter.hasPhysicalLabel(): 

170 filterName = exposureFilter.physicalLabel 

171 else: 

172 filterName = exposureFilter.bandLabel 

173 else: 

174 filterName = "_unknown_" 

175 self.assertEqual(filterName, self.butler_get_data.filters[name]) 

176 if not self._is_gen3(): 

177 exposureId = self.butler.get("ccdExposureId", dataId=self.dataIds[name]) 

178 self.assertEqual(exposureId, self.butler_get_data.exposureIds[name]) 

179 self.assertEqual(exp.getInfo().getVisitInfo().getExposureTime(), self.butler_get_data.exptimes[name]) 

180 return exp 

181 

182 def test_raw(self): 

183 exp = self._test_exposure("raw") 

184 # We only test the existence of WCS in the raw files, since it's only 

185 # well-defined for raw, and other exposure types could have or not 

186 # have a WCS depending on various implementation details. 

187 # Even for raw, there are data that do not have a WCS, e.g. teststand 

188 # data 

189 if self.butler_get_data.sky_origin is not unittest.SkipTest: 

190 self.assertEqual(exp.hasWcs(), True) 

191 origin = exp.getWcs().getSkyOrigin() 

192 self.assertAlmostEqual(origin.getLongitude().asDegrees(), self.butler_get_data.sky_origin[0]) 

193 self.assertAlmostEqual(origin.getLatitude().asDegrees(), self.butler_get_data.sky_origin[1]) 

194 

195 def test_bias(self): 

196 self._test_exposure("bias") 

197 

198 def test_dark(self): 

199 self._test_exposure("dark") 

200 

201 def test_flat(self): 

202 self._test_exposure("flat") 

203 

204 def test_raw_header_wcs(self): 

205 """Test that `raw_header_wcs` returns the unmodified header of the raw 

206 image.""" 

207 if self.butler_get_data.raw_header_wcs is None: 

208 self.skipTest("Skipping raw header WCS test since no reference provided.") 

209 # Gen3 will not understand this at the moment (DM-35031). 

210 wcs = self.butler.get("raw_header_wcs", self.dataIds["raw"]) 

211 self.assertEqual(wcs, self.butler_get_data.raw_header_wcs) 

212 

213 def test_subset_raw(self): 

214 for kwargs, expect in self.butler_get_data.raw_subsets: 

215 if self._is_gen3(): 

216 # If one of the keyword args looks like a dimension record 

217 # subquery, pull it out into the WHERE clause. 

218 where = [] 

219 bind = {} 

220 for k, v in list(kwargs.items()): 

221 if "." in k: 

222 bindval = k.replace(".", "_") 

223 where.append(f"{k} = {bindval}") 

224 bind[bindval] = v 

225 del kwargs[k] 

226 try: 

227 subset = set( 

228 self.butler.registry.queryDatasets( 

229 "raw", **kwargs, bind=bind, where=" AND ".join(where) 

230 ) 

231 ) 

232 except DataIdValueError: 

233 # This means one of the dataId values does not exist. 

234 subset = {} 

235 else: 

236 subset = self.butler.subset("raw", **kwargs) 

237 

238 self.assertEqual(len(subset), expect, msg="Failed for kwargs: {}".format(kwargs)) 

239 

240 def test_get_linearizer(self): 

241 """Test that we can get a linearizer for good detectorIds.""" 

242 if self.butler_get_data.linearizer_type is unittest.SkipTest: 

243 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

244 

245 camera = self.butler.get("camera") 

246 for detectorId in self.butler_get_data.good_detectorIds: 

247 detector = camera[detectorId] 

248 if self._is_gen3(): 

249 kwargs = {"detector": detectorId} 

250 else: 

251 kwargs = {"dataId": dict(ccd=detectorId), "immediate": True} 

252 linearizer = self.butler.get("linearizer", **kwargs) 

253 self.assertEqual(linearizer.LinearityType, self.butler_get_data.linearizer_type[detectorId]) 

254 linearizer.checkDetector(detector) 

255 

256 def test_get_linearizer_bad_detectorIds(self): 

257 """Do bad detectorIds raise?""" 

258 if self.butler_get_data.linearizer_type is unittest.SkipTest: 

259 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

260 

261 for badccd in self.butler_get_data.bad_detectorIds: 

262 if self._is_gen3(): 

263 kwargs = {"detector": badccd} 

264 else: 

265 kwargs = {"dataId": dict(ccd=badccd), "immediate": True} 

266 with self.assertRaises(RuntimeError): 

267 self.butler.get("linearizer", **kwargs)