Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating 

23test repositories or butlers. 

24""" 

25 

26import os 

27import shutil 

28import tempfile 

29import unittest 

30 

31import lsst.daf.butler 

32from lsst.daf.butler.tests import (makeTestRepo, makeTestCollection, addDatasetType, expandUniqueId, 

33 MetricsExample, registerMetricsExample) 

34 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39class ButlerUtilsTestSuite(unittest.TestCase): 

40 @classmethod 

41 def setUpClass(cls): 

42 # Repository should be re-created for each test case, but 

43 # this has a prohibitive run-time cost at present 

44 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

45 

46 dataIds = { 

47 "instrument": ["notACam", "dummyCam"], 

48 "physical_filter": ["k2020", "l2019"], 

49 "visit": [101, 102], 

50 "detector": [5] 

51 } 

52 cls.creatorButler = makeTestRepo(cls.root, dataIds) 

53 

54 registerMetricsExample(cls.creatorButler) 

55 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents") 

56 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData") 

57 

58 @classmethod 

59 def tearDownClass(cls): 

60 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8 

61 # to keep the addition and removal together and make it more robust 

62 shutil.rmtree(cls.root, ignore_errors=True) 

63 

64 def setUp(self): 

65 self.butler = makeTestCollection(self.creatorButler) 

66 

67 def testButlerValid(self): 

68 self.butler.validateConfiguration() 

69 

70 def testButlerKwargs(self): 

71 # outfile has the most obvious effects of any Butler.makeRepo keyword 

72 temp = tempfile.mkdtemp(dir=TESTDIR) 

73 try: 

74 path = os.path.join(temp, 'oddConfig.py') 

75 makeTestRepo(self.root, {}, outfile=path, createRegistry=False) 

76 self.assertTrue(os.path.isfile(path)) 

77 finally: 

78 shutil.rmtree(temp, ignore_errors=True) 

79 

80 def _checkButlerDimension(self, dimensions, query, expected): 

81 result = [id for id in self.butler.registry.queryDimensions( 

82 dimensions, 

83 where=query, 

84 expand=False)] 

85 self.assertEqual(len(result), 1) 

86 self.assertIn(dict(result[0]), expected) 

87 

88 def testButlerDimensions(self): 

89 self. _checkButlerDimension({"instrument"}, 

90 "instrument='notACam'", 

91 [{"instrument": "notACam"}, {"instrument": "dummyCam"}]) 

92 self. _checkButlerDimension({"visit", "instrument"}, 

93 "visit=101", 

94 [{"instrument": "notACam", "visit": 101}, 

95 {"instrument": "dummyCam", "visit": 101}]) 

96 self. _checkButlerDimension({"visit", "instrument"}, 

97 "visit=102", 

98 [{"instrument": "notACam", "visit": 102}, 

99 {"instrument": "dummyCam", "visit": 102}]) 

100 self. _checkButlerDimension({"detector", "instrument"}, 

101 "detector=5", 

102 [{"instrument": "notACam", "detector": 5}, 

103 {"instrument": "dummyCam", "detector": 5}]) 

104 

105 def testAddDatasetType(self): 

106 # 1 for StructuredDataNoComponents, 4 for StructuredData 

107 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=True))), 5) 

108 

109 # Testing the DatasetType objects is not practical, because all tests 

110 # need a DimensionUniverse. So just check that we have the dataset 

111 # types we expect. 

112 self.butler.registry.getDatasetType("DataType1") 

113 self.butler.registry.getDatasetType("DataType2") 

114 

115 with self.assertRaises(ValueError): 

116 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray") 

117 with self.assertRaises(ValueError): 

118 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType") 

119 

120 def testRegisterMetricsExample(self): 

121 id1 = {"instrument": "notACam"} 

122 id2 = expandUniqueId(self.butler, {"visit": 101}) 

123 data = MetricsExample(summary={"answer": 42, "question": "unknown"}) 

124 

125 self.butler.put(data, "DataType1", id1) 

126 self.assertEqual(self.butler.get("DataType1", id1), data) 

127 

128 self.butler.put(data, "DataType2", id2) 

129 self.assertEqual(self.butler.get("DataType2", id2), data) 

130 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary) 

131 

132 def testRegisterMetricsExampleChained(self): 

133 """Regression test for registerMetricsExample having no effect 

134 on ChainedDatastore. 

135 """ 

136 temp = tempfile.mkdtemp(dir=TESTDIR) 

137 try: 

138 config = lsst.daf.butler.Config() 

139 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore" 

140 config["datastore", "datastores"] = [{ 

141 "cls": "lsst.daf.butler.datastores.posixDatastore.PosixDatastore", 

142 }] 

143 

144 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config) 

145 butler = lsst.daf.butler.Butler(repo, run="chainedExample") 

146 registerMetricsExample(butler) 

147 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents") 

148 

149 data = MetricsExample(summary={}) 

150 # Should not raise 

151 butler.put(data, "DummyType") 

152 finally: 

153 shutil.rmtree(temp, ignore_errors=True) 

154 

155 def testUniqueButler(self): 

156 dataId = {"instrument": "notACam"} 

157 self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId) 

158 self.assertTrue(self.butler.datasetExists("DataType1", dataId)) 

159 

160 newButler = makeTestCollection(self.creatorButler) 

161 with self.assertRaises(LookupError): 

162 newButler.datasetExists("DataType1", dataId) 

163 

164 def testExpandUniqueId(self): 

165 self.assertEqual(dict(expandUniqueId(self.butler, {"instrument": "notACam"})), 

166 {"instrument": "notACam"}) 

167 self.assertIn(dict(expandUniqueId(self.butler, {"visit": 101})), 

168 [{"instrument": "notACam", "visit": 101}, 

169 {"instrument": "dummyCam", "visit": 101}]) 

170 self.assertIn(dict(expandUniqueId(self.butler, {"detector": 5})), 

171 [{"instrument": "notACam", "detector": 5}, 

172 {"instrument": "dummyCam", "detector": 5}]) 

173 self.assertIn(dict(expandUniqueId(self.butler, {"physical_filter": "k2020"})), 

174 [{"instrument": "notACam", "physical_filter": "k2020"}, 

175 {"instrument": "notACam", "physical_filter": "k2020"}]) 

176 with self.assertRaises(ValueError): 

177 expandUniqueId(self.butler, {"tract": 42}) 

178 

179 

180if __name__ == "__main__": 180 ↛ 181line 180 didn't jump to line 181, because the condition on line 180 was never true

181 unittest.main()