Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating 

23test repositories or butlers. 

24""" 

25 

26import os 

27import shutil 

28import unittest 

29 

30import lsst.daf.butler 

31from lsst.daf.butler.tests import (makeTestRepo, makeTestCollection, addDatasetType, expandUniqueId, 

32 MetricsExample, registerMetricsExample, addDataIdValue) 

33from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir 

34 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39class ButlerUtilsTestSuite(unittest.TestCase): 

40 @classmethod 

41 def setUpClass(cls): 

42 # Repository should be re-created for each test case, but 

43 # this has a prohibitive run-time cost at present 

44 cls.root = makeTestTempDir(TESTDIR) 

45 

46 cls.creatorButler = makeTestRepo(cls.root) 

47 addDataIdValue(cls.creatorButler, "instrument", "notACam") 

48 addDataIdValue(cls.creatorButler, "instrument", "dummyCam") 

49 addDataIdValue(cls.creatorButler, "physical_filter", "k2020", band="k", instrument="notACam") 

50 addDataIdValue(cls.creatorButler, "physical_filter", "l2019", instrument="dummyCam") 

51 addDataIdValue(cls.creatorButler, "visit", 101, instrument="notACam", physical_filter="k2020") 

52 addDataIdValue(cls.creatorButler, "visit", 102, instrument="notACam", physical_filter="k2020") 

53 addDataIdValue(cls.creatorButler, "detector", 5) 

54 # Leave skymap/patch/tract undefined so that tests can assume 

55 # they're missing. 

56 

57 registerMetricsExample(cls.creatorButler) 

58 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents") 

59 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData") 

60 

61 @classmethod 

62 def tearDownClass(cls): 

63 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8 

64 # to keep the addition and removal together and make it more robust 

65 removeTestTempDir(cls.root) 

66 

67 def setUp(self): 

68 self.butler = makeTestCollection(self.creatorButler) 

69 

70 def testButlerValid(self): 

71 self.butler.validateConfiguration() 

72 

73 def testButlerKwargs(self): 

74 # outfile has the most obvious effects of any Butler.makeRepo keyword 

75 with safeTestTempDir(TESTDIR) as temp: 

76 path = os.path.join(temp, 'oddConfig.json') 

77 makeTestRepo(temp, {}, outfile=path) 

78 self.assertTrue(os.path.isfile(path)) 

79 

80 def _checkButlerDimension(self, dimensions, query, expected): 

81 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False)) 

82 self.assertEqual(len(result), 1) 

83 self.assertIn(dict(result[0]), expected) 

84 

85 def testButlerDimensions(self): 

86 self._checkButlerDimension({"instrument"}, 

87 "instrument='notACam'", 

88 [{"instrument": "notACam"}, {"instrument": "dummyCam"}]) 

89 self._checkButlerDimension({"visit", "instrument"}, 

90 "visit=101", 

91 [{"instrument": "notACam", "visit": 101}, 

92 {"instrument": "dummyCam", "visit": 101}]) 

93 self._checkButlerDimension({"visit", "instrument"}, 

94 "visit=102", 

95 [{"instrument": "notACam", "visit": 102}, 

96 {"instrument": "dummyCam", "visit": 102}]) 

97 self._checkButlerDimension({"detector", "instrument"}, 

98 "detector=5", 

99 [{"instrument": "notACam", "detector": 5}, 

100 {"instrument": "dummyCam", "detector": 5}]) 

101 

102 def testAddDataIdValue(self): 

103 addDataIdValue(self.butler, "visit", 1, instrument="notACam", physical_filter="k2020") 

104 self._checkButlerDimension({"visit", "instrument"}, 

105 "visit=1", 

106 [{"instrument": "notACam", "visit": 1}]) 

107 addDataIdValue(self.butler, "visit", 2, instrument="dummyCam", physical_filter="l2019") 

108 self._checkButlerDimension({"visit", "instrument"}, 

109 "visit=2", 

110 [{"instrument": "dummyCam", "visit": 2}]) 

111 

112 with self.assertRaises(ValueError): 

113 addDataIdValue(self.butler, "NotADimension", 42) 

114 with self.assertRaises(ValueError): 

115 addDataIdValue(self.butler, "detector", "nonNumeric") 

116 with self.assertRaises(ValueError): 

117 addDataIdValue(self.butler, "detector", 101, nonsenseField="string") 

118 

119 # Keywords imply different instruments 

120 with self.assertRaises(RuntimeError): 

121 addDataIdValue(self.butler, "exposure", 101, instrument="dummyCam", physical_filter="k2020") 

122 

123 # No skymap defined 

124 with self.assertRaises(RuntimeError): 

125 addDataIdValue(self.butler, "tract", 42) 

126 with self.assertRaises(RuntimeError): 

127 addDataIdValue(self.butler, "tract", 43, skymap="map") 

128 

129 def testAddDatasetType(self): 

130 # 1 for StructuredDataNoComponents, 4 for StructuredData 

131 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=True))), 5) 

132 

133 # Testing the DatasetType objects is not practical, because all tests 

134 # need a DimensionUniverse. So just check that we have the dataset 

135 # types we expect. 

136 self.butler.registry.getDatasetType("DataType1") 

137 self.butler.registry.getDatasetType("DataType2") 

138 

139 with self.assertRaises(ValueError): 

140 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray") 

141 with self.assertRaises(ValueError): 

142 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType") 

143 

144 def testRegisterMetricsExample(self): 

145 id1 = {"instrument": "notACam"} 

146 id2 = expandUniqueId(self.butler, {"visit": 101}) 

147 data = MetricsExample(summary={"answer": 42, "question": "unknown"}) 

148 

149 self.butler.put(data, "DataType1", id1) 

150 self.assertEqual(self.butler.get("DataType1", id1), data) 

151 

152 self.butler.put(data, "DataType2", id2) 

153 self.assertEqual(self.butler.get("DataType2", id2), data) 

154 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary) 

155 

156 def testRegisterMetricsExampleChained(self): 

157 """Regression test for registerMetricsExample having no effect 

158 on ChainedDatastore. 

159 """ 

160 temp = makeTestTempDir(TESTDIR) 

161 try: 

162 config = lsst.daf.butler.Config() 

163 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore" 

164 config["datastore", "datastores"] = [{ 

165 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore", 

166 }] 

167 

168 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config) 

169 butler = lsst.daf.butler.Butler(repo, run="chainedExample") 

170 registerMetricsExample(butler) 

171 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents") 

172 

173 data = MetricsExample(summary={}) 

174 # Should not raise 

175 butler.put(data, "DummyType") 

176 finally: 

177 shutil.rmtree(temp, ignore_errors=True) 

178 

179 def testUniqueButler(self): 

180 dataId = {"instrument": "notACam"} 

181 self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId) 

182 self.assertTrue(self.butler.datasetExists("DataType1", dataId)) 

183 

184 newButler = makeTestCollection(self.creatorButler) 

185 with self.assertRaises(LookupError): 

186 newButler.datasetExists("DataType1", dataId) 

187 

188 def testExpandUniqueId(self): 

189 self.assertEqual(dict(expandUniqueId(self.butler, {"instrument": "notACam"})), 

190 {"instrument": "notACam"}) 

191 self.assertIn(dict(expandUniqueId(self.butler, {"visit": 101})), 

192 [{"instrument": "notACam", "visit": 101}, 

193 {"instrument": "dummyCam", "visit": 101}]) 

194 self.assertIn(dict(expandUniqueId(self.butler, {"detector": 5})), 

195 [{"instrument": "notACam", "detector": 5}, 

196 {"instrument": "dummyCam", "detector": 5}]) 

197 self.assertIn(dict(expandUniqueId(self.butler, {"physical_filter": "k2020"})), 

198 [{"instrument": "notACam", "physical_filter": "k2020"}, 

199 {"instrument": "notACam", "physical_filter": "k2020"}]) 

200 with self.assertRaises(ValueError): 

201 expandUniqueId(self.butler, {"tract": 42}) 

202 

203 

204if __name__ == "__main__": 204 ↛ 205line 204 didn't jump to line 205, because the condition on line 204 was never true

205 unittest.main()