Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating 

23test repositories or butlers. 

24""" 

25 

26import os 

27import shutil 

28import tempfile 

29import unittest 

30 

31import lsst.daf.butler 

32from lsst.daf.butler.tests import (makeTestRepo, makeTestCollection, addDatasetType, expandUniqueId, 

33 MetricsExample, registerMetricsExample) 

34 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39class ButlerUtilsTestSuite(unittest.TestCase): 

40 @classmethod 

41 def setUpClass(cls): 

42 # Repository should be re-created for each test case, but 

43 # this has a prohibitive run-time cost at present 

44 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

45 

46 dataIds = { 

47 "instrument": ["notACam", "dummyCam"], 

48 "physical_filter": ["k2020", "l2019"], 

49 "visit": [101, 102], 

50 "detector": [5] 

51 } 

52 cls.creatorButler = makeTestRepo(cls.root, dataIds) 

53 

54 registerMetricsExample(cls.creatorButler) 

55 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents") 

56 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData") 

57 

58 @classmethod 

59 def tearDownClass(cls): 

60 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8 

61 # to keep the addition and removal together and make it more robust 

62 shutil.rmtree(cls.root, ignore_errors=True) 

63 

64 def setUp(self): 

65 self.butler = makeTestCollection(self.creatorButler) 

66 

67 def testButlerValid(self): 

68 self.butler.validateConfiguration() 

69 

70 def testButlerKwargs(self): 

71 # outfile has the most obvious effects of any Butler.makeRepo keyword 

72 temp = tempfile.mkdtemp(dir=TESTDIR) 

73 try: 

74 path = os.path.join(temp, 'oddConfig.py') 

75 makeTestRepo(self.root, {}, outfile=path, createRegistry=False) 

76 self.assertTrue(os.path.isfile(path)) 

77 finally: 

78 shutil.rmtree(temp, ignore_errors=True) 

79 

80 def _checkButlerDimension(self, dimensions, query, expected): 

81 result = list(self.butler.registry.queryDataIds(dimensions, where=query)) 

82 self.assertEqual(len(result), 1) 

83 self.assertIn(dict(result[0]), expected) 

84 

85 def testButlerDimensions(self): 

86 self. _checkButlerDimension({"instrument"}, 

87 "instrument='notACam'", 

88 [{"instrument": "notACam"}, {"instrument": "dummyCam"}]) 

89 self. _checkButlerDimension({"visit", "instrument"}, 

90 "visit=101", 

91 [{"instrument": "notACam", "visit": 101}, 

92 {"instrument": "dummyCam", "visit": 101}]) 

93 self. _checkButlerDimension({"visit", "instrument"}, 

94 "visit=102", 

95 [{"instrument": "notACam", "visit": 102}, 

96 {"instrument": "dummyCam", "visit": 102}]) 

97 self. _checkButlerDimension({"detector", "instrument"}, 

98 "detector=5", 

99 [{"instrument": "notACam", "detector": 5}, 

100 {"instrument": "dummyCam", "detector": 5}]) 

101 

102 def testAddDatasetType(self): 

103 # 1 for StructuredDataNoComponents, 4 for StructuredData 

104 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=True))), 5) 

105 

106 # Testing the DatasetType objects is not practical, because all tests 

107 # need a DimensionUniverse. So just check that we have the dataset 

108 # types we expect. 

109 self.butler.registry.getDatasetType("DataType1") 

110 self.butler.registry.getDatasetType("DataType2") 

111 

112 with self.assertRaises(ValueError): 

113 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray") 

114 with self.assertRaises(ValueError): 

115 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType") 

116 

117 def testRegisterMetricsExample(self): 

118 id1 = {"instrument": "notACam"} 

119 id2 = expandUniqueId(self.butler, {"visit": 101}) 

120 data = MetricsExample(summary={"answer": 42, "question": "unknown"}) 

121 

122 self.butler.put(data, "DataType1", id1) 

123 self.assertEqual(self.butler.get("DataType1", id1), data) 

124 

125 self.butler.put(data, "DataType2", id2) 

126 self.assertEqual(self.butler.get("DataType2", id2), data) 

127 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary) 

128 

129 def testRegisterMetricsExampleChained(self): 

130 """Regression test for registerMetricsExample having no effect 

131 on ChainedDatastore. 

132 """ 

133 temp = tempfile.mkdtemp(dir=TESTDIR) 

134 try: 

135 config = lsst.daf.butler.Config() 

136 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore" 

137 config["datastore", "datastores"] = [{ 

138 "cls": "lsst.daf.butler.datastores.posixDatastore.PosixDatastore", 

139 }] 

140 

141 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config) 

142 butler = lsst.daf.butler.Butler(repo, run="chainedExample") 

143 registerMetricsExample(butler) 

144 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents") 

145 

146 data = MetricsExample(summary={}) 

147 # Should not raise 

148 butler.put(data, "DummyType") 

149 finally: 

150 shutil.rmtree(temp, ignore_errors=True) 

151 

152 def testUniqueButler(self): 

153 dataId = {"instrument": "notACam"} 

154 self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId) 

155 self.assertTrue(self.butler.datasetExists("DataType1", dataId)) 

156 

157 newButler = makeTestCollection(self.creatorButler) 

158 with self.assertRaises(LookupError): 

159 newButler.datasetExists("DataType1", dataId) 

160 

161 def testExpandUniqueId(self): 

162 self.assertEqual(dict(expandUniqueId(self.butler, {"instrument": "notACam"})), 

163 {"instrument": "notACam"}) 

164 self.assertIn(dict(expandUniqueId(self.butler, {"visit": 101})), 

165 [{"instrument": "notACam", "visit": 101}, 

166 {"instrument": "dummyCam", "visit": 101}]) 

167 self.assertIn(dict(expandUniqueId(self.butler, {"detector": 5})), 

168 [{"instrument": "notACam", "detector": 5}, 

169 {"instrument": "dummyCam", "detector": 5}]) 

170 self.assertIn(dict(expandUniqueId(self.butler, {"physical_filter": "k2020"})), 

171 [{"instrument": "notACam", "physical_filter": "k2020"}, 

172 {"instrument": "notACam", "physical_filter": "k2020"}]) 

173 with self.assertRaises(ValueError): 

174 expandUniqueId(self.butler, {"tract": 42}) 

175 

176 

177if __name__ == "__main__": 177 ↛ 178line 177 didn't jump to line 178, because the condition on line 177 was never true

178 unittest.main()