Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating 

23test repositories or butlers. 

24""" 

25 

26import os 

27import shutil 

28import unittest 

29 

30import lsst.daf.butler 

31from lsst.daf.butler.tests import (makeTestRepo, makeTestCollection, addDatasetType, expandUniqueId, 

32 MetricsExample, registerMetricsExample) 

33from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir 

34 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39class ButlerUtilsTestSuite(unittest.TestCase): 

40 @classmethod 

41 def setUpClass(cls): 

42 # Repository should be re-created for each test case, but 

43 # this has a prohibitive run-time cost at present 

44 cls.root = makeTestTempDir(TESTDIR) 

45 

46 dataIds = { 

47 "instrument": ["notACam", "dummyCam"], 

48 "physical_filter": ["k2020", "l2019"], 

49 "visit": [101, 102], 

50 "detector": [5] 

51 } 

52 cls.creatorButler = makeTestRepo(cls.root, dataIds) 

53 

54 registerMetricsExample(cls.creatorButler) 

55 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents") 

56 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData") 

57 

58 @classmethod 

59 def tearDownClass(cls): 

60 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8 

61 # to keep the addition and removal together and make it more robust 

62 removeTestTempDir(cls.root) 

63 

64 def setUp(self): 

65 self.butler = makeTestCollection(self.creatorButler) 

66 

67 def testButlerValid(self): 

68 self.butler.validateConfiguration() 

69 

70 def testButlerKwargs(self): 

71 # outfile has the most obvious effects of any Butler.makeRepo keyword 

72 with safeTestTempDir(TESTDIR) as temp: 

73 path = os.path.join(temp, 'oddConfig.json') 

74 makeTestRepo(temp, {}, outfile=path) 

75 self.assertTrue(os.path.isfile(path)) 

76 

77 def _checkButlerDimension(self, dimensions, query, expected): 

78 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False)) 

79 self.assertEqual(len(result), 1) 

80 self.assertIn(dict(result[0]), expected) 

81 

82 def testButlerDimensions(self): 

83 self. _checkButlerDimension({"instrument"}, 

84 "instrument='notACam'", 

85 [{"instrument": "notACam"}, {"instrument": "dummyCam"}]) 

86 self. _checkButlerDimension({"visit", "instrument"}, 

87 "visit=101", 

88 [{"instrument": "notACam", "visit": 101}, 

89 {"instrument": "dummyCam", "visit": 101}]) 

90 self. _checkButlerDimension({"visit", "instrument"}, 

91 "visit=102", 

92 [{"instrument": "notACam", "visit": 102}, 

93 {"instrument": "dummyCam", "visit": 102}]) 

94 self. _checkButlerDimension({"detector", "instrument"}, 

95 "detector=5", 

96 [{"instrument": "notACam", "detector": 5}, 

97 {"instrument": "dummyCam", "detector": 5}]) 

98 

99 def testAddDatasetType(self): 

100 # 1 for StructuredDataNoComponents, 4 for StructuredData 

101 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=True))), 5) 

102 

103 # Testing the DatasetType objects is not practical, because all tests 

104 # need a DimensionUniverse. So just check that we have the dataset 

105 # types we expect. 

106 self.butler.registry.getDatasetType("DataType1") 

107 self.butler.registry.getDatasetType("DataType2") 

108 

109 with self.assertRaises(ValueError): 

110 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray") 

111 with self.assertRaises(ValueError): 

112 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType") 

113 

114 def testRegisterMetricsExample(self): 

115 id1 = {"instrument": "notACam"} 

116 id2 = expandUniqueId(self.butler, {"visit": 101}) 

117 data = MetricsExample(summary={"answer": 42, "question": "unknown"}) 

118 

119 self.butler.put(data, "DataType1", id1) 

120 self.assertEqual(self.butler.get("DataType1", id1), data) 

121 

122 self.butler.put(data, "DataType2", id2) 

123 self.assertEqual(self.butler.get("DataType2", id2), data) 

124 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary) 

125 

126 def testRegisterMetricsExampleChained(self): 

127 """Regression test for registerMetricsExample having no effect 

128 on ChainedDatastore. 

129 """ 

130 temp = makeTestTempDir(TESTDIR) 

131 try: 

132 config = lsst.daf.butler.Config() 

133 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore" 

134 config["datastore", "datastores"] = [{ 

135 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore", 

136 }] 

137 

138 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config) 

139 butler = lsst.daf.butler.Butler(repo, run="chainedExample") 

140 registerMetricsExample(butler) 

141 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents") 

142 

143 data = MetricsExample(summary={}) 

144 # Should not raise 

145 butler.put(data, "DummyType") 

146 finally: 

147 shutil.rmtree(temp, ignore_errors=True) 

148 

149 def testUniqueButler(self): 

150 dataId = {"instrument": "notACam"} 

151 self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId) 

152 self.assertTrue(self.butler.datasetExists("DataType1", dataId)) 

153 

154 newButler = makeTestCollection(self.creatorButler) 

155 with self.assertRaises(LookupError): 

156 newButler.datasetExists("DataType1", dataId) 

157 

158 def testExpandUniqueId(self): 

159 self.assertEqual(dict(expandUniqueId(self.butler, {"instrument": "notACam"})), 

160 {"instrument": "notACam"}) 

161 self.assertIn(dict(expandUniqueId(self.butler, {"visit": 101})), 

162 [{"instrument": "notACam", "visit": 101}, 

163 {"instrument": "dummyCam", "visit": 101}]) 

164 self.assertIn(dict(expandUniqueId(self.butler, {"detector": 5})), 

165 [{"instrument": "notACam", "detector": 5}, 

166 {"instrument": "dummyCam", "detector": 5}]) 

167 self.assertIn(dict(expandUniqueId(self.butler, {"physical_filter": "k2020"})), 

168 [{"instrument": "notACam", "physical_filter": "k2020"}, 

169 {"instrument": "notACam", "physical_filter": "k2020"}]) 

170 with self.assertRaises(ValueError): 

171 expandUniqueId(self.butler, {"tract": 42}) 

172 

173 

174if __name__ == "__main__": 174 ↛ 175line 174 didn't jump to line 175, because the condition on line 174 was never true

175 unittest.main()