Coverage for python/lsst/obs/base/butler_tests.py: 15%

83 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-09 09:23 +0000

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import abc 

23import collections 

24import inspect 

25import unittest 

26 

27from lsst.daf.butler.registry import DataIdValueError 

28 

29__all__ = ["ButlerGetTests"] 

30 

31 

32class ButlerGetTests(metaclass=abc.ABCMeta): 

33 """Tests of obs_* Butler get() functionality. 

34 

35 In the subclasses's setUp(): 

36 * Call setUp_butler_get() to fill in required parameters. 

37 """ 

38 

39 def setUp_butler_get( 

40 self, 

41 ccdExposureId_bits=None, 

42 exposureIds=None, 

43 filters=None, 

44 exptimes=None, 

45 detectorIds=None, 

46 detector_names=None, 

47 detector_serials=None, 

48 dimensions=None, 

49 sky_origin=None, 

50 raw_subsets=None, 

51 good_detectorIds=None, 

52 bad_detectorIds=None, 

53 linearizer_type=None, 

54 raw_header_wcs=None, 

55 ): 

56 """ 

57 Set up the necessary variables for butlerGet tests. 

58 

59 All "exposure name" entries below should correspond to an entry in 

60 self.dataIds. 

61 

62 Parameters 

63 ---------- 

64 

65 ccdExposureId_bits : `int` 

66 expected value of ccdExposureId_bits 

67 exposureIds : `dict` 

68 dict of exposure name : ccdExposureId (the number as returned by 

69 the butler) 

70 filters : `dict` 

71 dict of exposure name : filter name 

72 exptimes : `dict` 

73 dict of exposure name : exposure time 

74 detector_names : `dict` 

75 dict of exposure name : detector name 

76 detectorIds : `dict` 

77 dict of exposure name : detectorId 

78 detector_serials : `dict` 

79 dict of exposure name : detector serial 

80 dimensions : `dict` 

81 dict of exposure name : dimensions (as a geom.Extent2I) 

82 sky_origin : `tuple` of `float` 

83 Longitude, Latitude of 'raw' exposure 

84 raw_subsets : `tuple` of (kwargs, `int`) 

85 keyword args and expected number of subsets for 

86 ``butler.subset('raw', **kwargs)`` 

87 good_detectorIds : `list` of `int` 

88 list of valid ccd numbers 

89 bad_detectorIds : `list` of `int` 

90 list of invalid ccd numbers 

91 linearizer_type : `dict` 

92 dict of detectorId (usually `int`): LinearizerType 

93 (e.g. lsst.ip.isr.LinearizeLookupTable.LinearityType), 

94 or unittest.SkipTest to skip all linearizer tests. 

95 raw_header_wcs : `lsst.afw.geom.SkyWcs` 

96 The SkyWcs object that should be returned by 

97 ``butler.get("raw_header_wcs", dataId=self.dataIds["raw"])`` 

98 """ 

99 

100 fields = [ 

101 "ccdExposureId_bits", 

102 "exposureIds", 

103 "filters", 

104 "exptimes", 

105 "detector_names", 

106 "detectorIds", 

107 "detector_serials", 

108 "dimensions", 

109 "sky_origin", 

110 "raw_subsets", 

111 "good_detectorIds", 

112 "bad_detectorIds", 

113 "linearizer_type", 

114 "raw_header_wcs", 

115 ] 

116 ButlerGet = collections.namedtuple("ButlerGetData", fields) 

117 

118 self.butler_get_data = ButlerGet( 

119 ccdExposureId_bits=ccdExposureId_bits, 

120 exposureIds=exposureIds, 

121 filters=filters, 

122 exptimes=exptimes, 

123 detectorIds=detectorIds, 

124 detector_names=detector_names, 

125 detector_serials=detector_serials, 

126 dimensions=dimensions, 

127 sky_origin=sky_origin, 

128 raw_subsets=raw_subsets, 

129 good_detectorIds=good_detectorIds, 

130 bad_detectorIds=bad_detectorIds, 

131 linearizer_type=linearizer_type, 

132 raw_header_wcs=raw_header_wcs, 

133 ) 

134 

135 def _test_exposure(self, name): 

136 if self.dataIds[name] is unittest.SkipTest: 

137 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

138 exp = self.butler.get(name, self.dataIds[name]) 

139 

140 md_component = ".metadata" 

141 exp_md = self.butler.get(name + md_component, self.dataIds[name]) 

142 self.assertEqual(type(exp_md), type(exp.getMetadata())) 

143 

144 self.assertEqual(exp.getDimensions(), self.butler_get_data.dimensions[name]) 

145 detector = exp.detector 

146 # Some calibration files are missing the detector. 

147 if detector: 

148 self.assertEqual(detector.getId(), self.butler_get_data.detectorIds[name]) 

149 self.assertEqual(detector.getName(), self.butler_get_data.detector_names[name]) 

150 self.assertEqual(detector.getSerial(), self.butler_get_data.detector_serials[name]) 

151 # obs_test does not have physical filters, so include a fallback 

152 exposureFilter = exp.getFilter() 

153 if exposureFilter: 

154 if exposureFilter.hasPhysicalLabel(): 

155 filterName = exposureFilter.physicalLabel 

156 else: 

157 filterName = exposureFilter.bandLabel 

158 else: 

159 filterName = "_unknown_" 

160 self.assertEqual(filterName, self.butler_get_data.filters[name]) 

161 self.assertEqual(exp.getInfo().getVisitInfo().getExposureTime(), self.butler_get_data.exptimes[name]) 

162 return exp 

163 

164 def test_raw(self): 

165 exp = self._test_exposure("raw") 

166 # We only test the existence of WCS in the raw files, since it's only 

167 # well-defined for raw, and other exposure types could have or not 

168 # have a WCS depending on various implementation details. 

169 # Even for raw, there are data that do not have a WCS, e.g. teststand 

170 # data 

171 if self.butler_get_data.sky_origin is not unittest.SkipTest: 

172 self.assertEqual(exp.hasWcs(), True) 

173 origin = exp.getWcs().getSkyOrigin() 

174 self.assertAlmostEqual(origin.getLongitude().asDegrees(), self.butler_get_data.sky_origin[0]) 

175 self.assertAlmostEqual(origin.getLatitude().asDegrees(), self.butler_get_data.sky_origin[1]) 

176 

177 def test_bias(self): 

178 self._test_exposure("bias") 

179 

180 def test_dark(self): 

181 self._test_exposure("dark") 

182 

183 def test_flat(self): 

184 self._test_exposure("flat") 

185 

186 def test_raw_header_wcs(self): 

187 """Test that `raw_header_wcs` returns the unmodified header of the raw 

188 image.""" 

189 if self.butler_get_data.raw_header_wcs is None: 

190 self.skipTest("Skipping raw header WCS test since no reference provided.") 

191 # Gen3 will not understand this at the moment (DM-35031). 

192 wcs = self.butler.get("raw_header_wcs", self.dataIds["raw"]) 

193 self.assertEqual(wcs, self.butler_get_data.raw_header_wcs) 

194 

195 def test_subset_raw(self): 

196 for kwargs, expect in self.butler_get_data.raw_subsets: 

197 # If one of the keyword args looks like a dimension record 

198 # subquery, pull it out into the WHERE clause. 

199 where = [] 

200 bind = {} 

201 for k, v in list(kwargs.items()): 

202 if "." in k: 

203 bindval = k.replace(".", "_") 

204 where.append(f"{k} = {bindval}") 

205 bind[bindval] = v 

206 del kwargs[k] 

207 try: 

208 subset = set( 

209 self.butler.registry.queryDatasets("raw", **kwargs, bind=bind, where=" AND ".join(where)) 

210 ) 

211 except DataIdValueError: 

212 # This means one of the dataId values does not exist. 

213 subset = {} 

214 

215 self.assertEqual(len(subset), expect, msg="Failed for kwargs: {}".format(kwargs)) 

216 

217 def test_get_linearizer(self): 

218 """Test that we can get a linearizer for good detectorIds.""" 

219 if self.butler_get_data.linearizer_type is unittest.SkipTest: 

220 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

221 

222 camera = self.butler.get("camera") 

223 for detectorId in self.butler_get_data.good_detectorIds: 

224 detector = camera[detectorId] 

225 kwargs = {"detector": detectorId} 

226 linearizer = self.butler.get("linearizer", **kwargs) 

227 self.assertEqual(linearizer.LinearityType, self.butler_get_data.linearizer_type[detectorId]) 

228 linearizer.checkDetector(detector) 

229 

230 def test_get_linearizer_bad_detectorIds(self): 

231 """Do bad detectorIds raise?""" 

232 if self.butler_get_data.linearizer_type is unittest.SkipTest: 

233 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

234 

235 for badccd in self.butler_get_data.bad_detectorIds: 

236 kwargs = {"detector": badccd} 

237 with self.assertRaises(RuntimeError): 

238 self.butler.get("linearizer", **kwargs)