Coverage for python/lsst/obs/base/butler_tests.py: 15%

82 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-05 12:08 +0000

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import collections 

23import inspect 

24import unittest 

25 

26from lsst.daf.butler.registry import DataIdValueError 

27 

28__all__ = ["ButlerGetTests"] 

29 

30 

31class ButlerGetTests: 

32 """Tests of obs_* Butler get() functionality. 

33 

34 In the subclasses's setUp(): 

35 * Call setUp_butler_get() to fill in required parameters. 

36 """ 

37 

38 def setUp_butler_get( 

39 self, 

40 ccdExposureId_bits=None, 

41 exposureIds=None, 

42 filters=None, 

43 exptimes=None, 

44 detectorIds=None, 

45 detector_names=None, 

46 detector_serials=None, 

47 dimensions=None, 

48 sky_origin=None, 

49 raw_subsets=None, 

50 good_detectorIds=None, 

51 bad_detectorIds=None, 

52 linearizer_type=None, 

53 raw_header_wcs=None, 

54 ): 

55 """ 

56 Set up the necessary variables for butlerGet tests. 

57 

58 All "exposure name" entries below should correspond to an entry in 

59 self.dataIds. 

60 

61 Parameters 

62 ---------- 

63 ccdExposureId_bits : `int` 

64 expected value of ccdExposureId_bits 

65 exposureIds : `dict` 

66 dict of exposure name : ccdExposureId (the number as returned by 

67 the butler) 

68 filters : `dict` 

69 dict of exposure name : filter name 

70 exptimes : `dict` 

71 dict of exposure name : exposure time 

72 detector_names : `dict` 

73 dict of exposure name : detector name 

74 detectorIds : `dict` 

75 dict of exposure name : detectorId 

76 detector_serials : `dict` 

77 dict of exposure name : detector serial 

78 dimensions : `dict` 

79 dict of exposure name : dimensions (as a geom.Extent2I) 

80 sky_origin : `tuple` of `float` 

81 Longitude, Latitude of 'raw' exposure 

82 raw_subsets : `tuple` of (kwargs, `int`) 

83 keyword args and expected number of subsets for 

84 ``butler.subset('raw', **kwargs)`` 

85 good_detectorIds : `list` of `int` 

86 list of valid ccd numbers 

87 bad_detectorIds : `list` of `int` 

88 list of invalid ccd numbers 

89 linearizer_type : `dict` 

90 dict of detectorId (usually `int`): LinearizerType 

91 (e.g. lsst.ip.isr.LinearizeLookupTable.LinearityType), 

92 or unittest.SkipTest to skip all linearizer tests. 

93 raw_header_wcs : `lsst.afw.geom.SkyWcs` 

94 The SkyWcs object that should be returned by 

95 ``butler.get("raw_header_wcs", dataId=self.dataIds["raw"])`` 

96 """ 

97 fields = [ 

98 "ccdExposureId_bits", 

99 "exposureIds", 

100 "filters", 

101 "exptimes", 

102 "detector_names", 

103 "detectorIds", 

104 "detector_serials", 

105 "dimensions", 

106 "sky_origin", 

107 "raw_subsets", 

108 "good_detectorIds", 

109 "bad_detectorIds", 

110 "linearizer_type", 

111 "raw_header_wcs", 

112 ] 

113 ButlerGet = collections.namedtuple("ButlerGetData", fields) 

114 

115 self.butler_get_data = ButlerGet( 

116 ccdExposureId_bits=ccdExposureId_bits, 

117 exposureIds=exposureIds, 

118 filters=filters, 

119 exptimes=exptimes, 

120 detectorIds=detectorIds, 

121 detector_names=detector_names, 

122 detector_serials=detector_serials, 

123 dimensions=dimensions, 

124 sky_origin=sky_origin, 

125 raw_subsets=raw_subsets, 

126 good_detectorIds=good_detectorIds, 

127 bad_detectorIds=bad_detectorIds, 

128 linearizer_type=linearizer_type, 

129 raw_header_wcs=raw_header_wcs, 

130 ) 

131 

132 def _test_exposure(self, name): 

133 if self.dataIds[name] is unittest.SkipTest: 

134 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

135 exp = self.butler.get(name, self.dataIds[name]) 

136 

137 md_component = ".metadata" 

138 exp_md = self.butler.get(name + md_component, self.dataIds[name]) 

139 self.assertEqual(type(exp_md), type(exp.getMetadata())) 

140 

141 self.assertEqual(exp.getDimensions(), self.butler_get_data.dimensions[name]) 

142 detector = exp.detector 

143 # Some calibration files are missing the detector. 

144 if detector: 

145 self.assertEqual(detector.getId(), self.butler_get_data.detectorIds[name]) 

146 self.assertEqual(detector.getName(), self.butler_get_data.detector_names[name]) 

147 self.assertEqual(detector.getSerial(), self.butler_get_data.detector_serials[name]) 

148 # obs_test does not have physical filters, so include a fallback 

149 exposureFilter = exp.getFilter() 

150 if exposureFilter: 

151 if exposureFilter.hasPhysicalLabel(): 

152 filterName = exposureFilter.physicalLabel 

153 else: 

154 filterName = exposureFilter.bandLabel 

155 else: 

156 filterName = "_unknown_" 

157 self.assertEqual(filterName, self.butler_get_data.filters[name]) 

158 self.assertEqual(exp.getInfo().getVisitInfo().getExposureTime(), self.butler_get_data.exptimes[name]) 

159 return exp 

160 

161 def test_raw(self): 

162 exp = self._test_exposure("raw") 

163 # We only test the existence of WCS in the raw files, since it's only 

164 # well-defined for raw, and other exposure types could have or not 

165 # have a WCS depending on various implementation details. 

166 # Even for raw, there are data that do not have a WCS, e.g. teststand 

167 # data 

168 if self.butler_get_data.sky_origin is not unittest.SkipTest: 

169 self.assertEqual(exp.hasWcs(), True) 

170 origin = exp.getWcs().getSkyOrigin() 

171 self.assertAlmostEqual(origin.getLongitude().asDegrees(), self.butler_get_data.sky_origin[0]) 

172 self.assertAlmostEqual(origin.getLatitude().asDegrees(), self.butler_get_data.sky_origin[1]) 

173 

174 def test_bias(self): 

175 self._test_exposure("bias") 

176 

177 def test_dark(self): 

178 self._test_exposure("dark") 

179 

180 def test_flat(self): 

181 self._test_exposure("flat") 

182 

183 def test_raw_header_wcs(self): 

184 """Test that `raw_header_wcs` returns the unmodified header of the raw 

185 image. 

186 """ 

187 if self.butler_get_data.raw_header_wcs is None: 

188 self.skipTest("Skipping raw header WCS test since no reference provided.") 

189 # Gen3 will not understand this at the moment (DM-35031). 

190 wcs = self.butler.get("raw_header_wcs", self.dataIds["raw"]) 

191 self.assertEqual(wcs, self.butler_get_data.raw_header_wcs) 

192 

193 def test_subset_raw(self): 

194 for kwargs, expect in self.butler_get_data.raw_subsets: 

195 # If one of the keyword args looks like a dimension record 

196 # subquery, pull it out into the WHERE clause. 

197 where = [] 

198 bind = {} 

199 for k, v in list(kwargs.items()): 

200 if "." in k: 

201 bindval = k.replace(".", "_") 

202 where.append(f"{k} = {bindval}") 

203 bind[bindval] = v 

204 del kwargs[k] 

205 try: 

206 subset = set( 

207 self.butler.registry.queryDatasets("raw", **kwargs, bind=bind, where=" AND ".join(where)) 

208 ) 

209 except DataIdValueError: 

210 # This means one of the dataId values does not exist. 

211 subset = {} 

212 

213 self.assertEqual(len(subset), expect, msg=f"Failed for kwargs: {kwargs}") 

214 

215 def test_get_linearizer(self): 

216 """Test that we can get a linearizer for good detectorIds.""" 

217 if self.butler_get_data.linearizer_type is unittest.SkipTest: 

218 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

219 

220 camera = self.butler.get("camera") 

221 for detectorId in self.butler_get_data.good_detectorIds: 

222 detector = camera[detectorId] 

223 kwargs = {"detector": detectorId} 

224 linearizer = self.butler.get("linearizer", **kwargs) 

225 self.assertEqual(linearizer.LinearityType, self.butler_get_data.linearizer_type[detectorId]) 

226 linearizer.checkDetector(detector) 

227 

228 def test_get_linearizer_bad_detectorIds(self): 

229 """Check that bad detectorIds raise.""" 

230 if self.butler_get_data.linearizer_type is unittest.SkipTest: 

231 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

232 

233 for badccd in self.butler_get_data.bad_detectorIds: 

234 kwargs = {"detector": badccd} 

235 with self.assertRaises(RuntimeError): 

236 self.butler.get("linearizer", **kwargs)