Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating 

23test repositories or butlers. 

24""" 

25 

26import os 

27import shutil 

28import unittest 

29 

30import lsst.daf.butler 

31from lsst.daf.butler.tests import (makeTestRepo, makeTestCollection, addDatasetType, expandUniqueId, 

32 MetricsExample, registerMetricsExample, addDataIdValue) 

33from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir 

34 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39class ButlerUtilsTestSuite(unittest.TestCase): 

40 @classmethod 

41 def setUpClass(cls): 

42 # Repository should be re-created for each test case, but 

43 # this has a prohibitive run-time cost at present 

44 cls.root = makeTestTempDir(TESTDIR) 

45 

46 cls.creatorButler = makeTestRepo(cls.root) 

47 addDataIdValue(cls.creatorButler, "instrument", "notACam") 

48 addDataIdValue(cls.creatorButler, "instrument", "dummyCam") 

49 addDataIdValue(cls.creatorButler, "physical_filter", "k2020", band="k", instrument="notACam") 

50 addDataIdValue(cls.creatorButler, "physical_filter", "l2019", instrument="dummyCam") 

51 addDataIdValue(cls.creatorButler, "visit", 101, instrument="notACam", physical_filter="k2020") 

52 addDataIdValue(cls.creatorButler, "visit", 102, instrument="notACam", physical_filter="k2020") 

53 addDataIdValue(cls.creatorButler, "detector", 5) 

54 # Leave skymap/patch/tract undefined so that tests can assume 

55 # they're missing. 

56 

57 registerMetricsExample(cls.creatorButler) 

58 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents") 

59 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData") 

60 

61 @classmethod 

62 def tearDownClass(cls): 

63 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8 

64 # to keep the addition and removal together and make it more robust 

65 removeTestTempDir(cls.root) 

66 

67 def setUp(self): 

68 # TestCase.id() is unique for each test method 

69 self.butler = makeTestCollection(self.creatorButler, uniqueId=self.id()) 

70 

71 def testButlerValid(self): 

72 self.butler.validateConfiguration() 

73 

74 def testButlerKwargs(self): 

75 # outfile has the most obvious effects of any Butler.makeRepo keyword 

76 with safeTestTempDir(TESTDIR) as temp: 

77 path = os.path.join(temp, 'oddConfig.json') 

78 makeTestRepo(temp, {}, outfile=path) 

79 self.assertTrue(os.path.isfile(path)) 

80 

81 def _checkButlerDimension(self, dimensions, query, expected): 

82 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False)) 

83 self.assertEqual(len(result), 1) 

84 self.assertIn(dict(result[0]), expected) 

85 

86 def testButlerDimensions(self): 

87 self._checkButlerDimension({"instrument"}, 

88 "instrument='notACam'", 

89 [{"instrument": "notACam"}, {"instrument": "dummyCam"}]) 

90 self._checkButlerDimension({"visit", "instrument"}, 

91 "visit=101", 

92 [{"instrument": "notACam", "visit": 101}, 

93 {"instrument": "dummyCam", "visit": 101}]) 

94 self._checkButlerDimension({"visit", "instrument"}, 

95 "visit=102", 

96 [{"instrument": "notACam", "visit": 102}, 

97 {"instrument": "dummyCam", "visit": 102}]) 

98 self._checkButlerDimension({"detector", "instrument"}, 

99 "detector=5", 

100 [{"instrument": "notACam", "detector": 5}, 

101 {"instrument": "dummyCam", "detector": 5}]) 

102 

103 def testAddDataIdValue(self): 

104 addDataIdValue(self.butler, "visit", 1, instrument="notACam", physical_filter="k2020") 

105 self._checkButlerDimension({"visit", "instrument"}, 

106 "visit=1", 

107 [{"instrument": "notACam", "visit": 1}]) 

108 addDataIdValue(self.butler, "visit", 2, instrument="dummyCam", physical_filter="l2019") 

109 self._checkButlerDimension({"visit", "instrument"}, 

110 "visit=2", 

111 [{"instrument": "dummyCam", "visit": 2}]) 

112 

113 with self.assertRaises(ValueError): 

114 addDataIdValue(self.butler, "NotADimension", 42) 

115 with self.assertRaises(ValueError): 

116 addDataIdValue(self.butler, "detector", "nonNumeric") 

117 with self.assertRaises(ValueError): 

118 addDataIdValue(self.butler, "detector", 101, nonsenseField="string") 

119 

120 # Keywords imply different instruments 

121 with self.assertRaises(RuntimeError): 

122 addDataIdValue(self.butler, "exposure", 101, instrument="dummyCam", physical_filter="k2020") 

123 

124 # No skymap defined 

125 with self.assertRaises(RuntimeError): 

126 addDataIdValue(self.butler, "tract", 42) 

127 with self.assertRaises(RuntimeError): 

128 addDataIdValue(self.butler, "tract", 43, skymap="map") 

129 

130 def testAddDatasetType(self): 

131 # 1 for StructuredDataNoComponents, 4 for StructuredData 

132 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=True))), 5) 

133 

134 # Testing the DatasetType objects is not practical, because all tests 

135 # need a DimensionUniverse. So just check that we have the dataset 

136 # types we expect. 

137 self.butler.registry.getDatasetType("DataType1") 

138 self.butler.registry.getDatasetType("DataType2") 

139 

140 with self.assertRaises(ValueError): 

141 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray") 

142 with self.assertRaises(ValueError): 

143 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType") 

144 

145 def testRegisterMetricsExample(self): 

146 id1 = {"instrument": "notACam"} 

147 id2 = expandUniqueId(self.butler, {"visit": 101}) 

148 data = MetricsExample(summary={"answer": 42, "question": "unknown"}) 

149 

150 self.butler.put(data, "DataType1", id1) 

151 self.assertEqual(self.butler.get("DataType1", id1), data) 

152 

153 self.butler.put(data, "DataType2", id2) 

154 self.assertEqual(self.butler.get("DataType2", id2), data) 

155 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary) 

156 

157 def testRegisterMetricsExampleChained(self): 

158 """Regression test for registerMetricsExample having no effect 

159 on ChainedDatastore. 

160 """ 

161 temp = makeTestTempDir(TESTDIR) 

162 try: 

163 config = lsst.daf.butler.Config() 

164 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore" 

165 config["datastore", "datastores"] = [{ 

166 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore", 

167 }] 

168 

169 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config) 

170 butler = lsst.daf.butler.Butler(repo, run="chainedExample") 

171 registerMetricsExample(butler) 

172 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents") 

173 

174 data = MetricsExample(summary={}) 

175 # Should not raise 

176 butler.put(data, "DummyType") 

177 finally: 

178 shutil.rmtree(temp, ignore_errors=True) 

179 

180 def testUniqueButler(self): 

181 dataId = {"instrument": "notACam"} 

182 self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId) 

183 self.assertTrue(self.butler.datasetExists("DataType1", dataId)) 

184 

185 newButler = makeTestCollection(self.creatorButler) 

186 with self.assertRaises(LookupError): 

187 newButler.datasetExists("DataType1", dataId) 

188 

189 def testExpandUniqueId(self): 

190 self.assertEqual(dict(expandUniqueId(self.butler, {"instrument": "notACam"})), 

191 {"instrument": "notACam"}) 

192 self.assertIn(dict(expandUniqueId(self.butler, {"visit": 101})), 

193 [{"instrument": "notACam", "visit": 101}, 

194 {"instrument": "dummyCam", "visit": 101}]) 

195 self.assertIn(dict(expandUniqueId(self.butler, {"detector": 5})), 

196 [{"instrument": "notACam", "detector": 5}, 

197 {"instrument": "dummyCam", "detector": 5}]) 

198 self.assertIn(dict(expandUniqueId(self.butler, {"physical_filter": "k2020"})), 

199 [{"instrument": "notACam", "physical_filter": "k2020"}, 

200 {"instrument": "notACam", "physical_filter": "k2020"}]) 

201 with self.assertRaises(ValueError): 

202 expandUniqueId(self.butler, {"tract": 42}) 

203 

204 

205if __name__ == "__main__": 205 ↛ 206line 205 didn't jump to line 206, because the condition on line 205 was never true

206 unittest.main()