Coverage for tests/test_testRepo.py: 23%

116 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-05 01:26 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating 

23test repositories or butlers. 

24""" 

25 

26import os 

27import shutil 

28import unittest 

29 

30import lsst.daf.butler 

31from lsst.daf.butler.tests import ( 

32 MetricsExample, 

33 addDataIdValue, 

34 addDatasetType, 

35 expandUniqueId, 

36 makeTestCollection, 

37 makeTestRepo, 

38 registerMetricsExample, 

39) 

40from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir 

41 

42TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

43 

44 

45class ButlerTestRepoTestCase(unittest.TestCase): 

46 """Simpler test than below without setUpClass getting in the way.""" 

47 

48 def setUp(self): 

49 self.root = makeTestTempDir(TESTDIR) 

50 

51 def tearDown(self): 

52 removeTestTempDir(self.root) 

53 

54 def testMakeTestRepo(self): 

55 dataIds = { 

56 "instrument": ["DummyCam"], 

57 "physical_filter": ["d-r"], 

58 "exposure": [42, 43, 44], 

59 "visit": [42, 43, 44], 

60 } 

61 

62 butler = makeTestRepo(self.root, dataIds) 

63 

64 records = list(butler.registry.queryDimensionRecords("visit")) 

65 self.assertEqual(len(records), 3) 

66 

67 

68class ButlerUtilsTestSuite(unittest.TestCase): 

69 """Test the butler test utilities.""" 

70 

71 @classmethod 

72 def setUpClass(cls): 

73 # Repository should be re-created for each test case, but 

74 # this has a prohibitive run-time cost at present 

75 cls.root = makeTestTempDir(TESTDIR) 

76 

77 cls.creatorButler = makeTestRepo(cls.root) 

78 addDataIdValue(cls.creatorButler, "instrument", "notACam") 

79 addDataIdValue(cls.creatorButler, "instrument", "dummyCam") 

80 addDataIdValue(cls.creatorButler, "physical_filter", "k2020", band="k", instrument="notACam") 

81 addDataIdValue(cls.creatorButler, "physical_filter", "l2019", instrument="dummyCam") 

82 addDataIdValue(cls.creatorButler, "visit", 101, instrument="notACam", physical_filter="k2020") 

83 addDataIdValue(cls.creatorButler, "visit", 102, instrument="notACam", physical_filter="k2020") 

84 addDataIdValue(cls.creatorButler, "detector", 5) 

85 # Leave skymap/patch/tract undefined so that tests can assume 

86 # they're missing. 

87 

88 registerMetricsExample(cls.creatorButler) 

89 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents") 

90 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData") 

91 

92 @classmethod 

93 def tearDownClass(cls): 

94 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8 

95 # to keep the addition and removal together and make it more robust 

96 removeTestTempDir(cls.root) 

97 

98 def setUp(self): 

99 # TestCase.id() is unique for each test method 

100 self.butler = makeTestCollection(self.creatorButler, uniqueId=self.id()) 

101 

102 def testButlerValid(self): 

103 self.butler.validateConfiguration() 

104 

105 def testButlerKwargs(self): 

106 # outfile has the most obvious effects of any Butler.makeRepo keyword 

107 with safeTestTempDir(TESTDIR) as temp: 

108 path = os.path.join(temp, "oddConfig.json") 

109 makeTestRepo(temp, {}, outfile=path) 

110 self.assertTrue(os.path.isfile(path)) 

111 

112 def _checkButlerDimension(self, dimensions, query, expected): 

113 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False)) 

114 self.assertEqual(len(result), 1) 

115 self.assertIn(result[0].byName(), expected) 

116 

117 def testButlerDimensions(self): 

118 self._checkButlerDimension( 

119 {"instrument"}, "instrument='notACam'", [{"instrument": "notACam"}, {"instrument": "dummyCam"}] 

120 ) 

121 self._checkButlerDimension( 

122 {"visit", "instrument"}, 

123 "visit=101", 

124 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}], 

125 ) 

126 self._checkButlerDimension( 

127 {"visit", "instrument"}, 

128 "visit=102", 

129 [{"instrument": "notACam", "visit": 102}, {"instrument": "dummyCam", "visit": 102}], 

130 ) 

131 self._checkButlerDimension( 

132 {"detector", "instrument"}, 

133 "detector=5", 

134 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}], 

135 ) 

136 

137 def testAddDataIdValue(self): 

138 addDataIdValue(self.butler, "visit", 1, instrument="notACam", physical_filter="k2020") 

139 self._checkButlerDimension( 

140 {"visit", "instrument"}, "visit=1", [{"instrument": "notACam", "visit": 1}] 

141 ) 

142 addDataIdValue(self.butler, "visit", 2, instrument="dummyCam", physical_filter="l2019") 

143 self._checkButlerDimension( 

144 {"visit", "instrument"}, "visit=2", [{"instrument": "dummyCam", "visit": 2}] 

145 ) 

146 

147 with self.assertRaises(ValueError): 

148 addDataIdValue(self.butler, "NotADimension", 42) 

149 with self.assertRaises(ValueError): 

150 addDataIdValue(self.butler, "detector", "nonNumeric") 

151 with self.assertRaises(ValueError): 

152 addDataIdValue(self.butler, "detector", 101, nonsenseField="string") 

153 

154 # Keywords imply different instruments 

155 with self.assertRaises(RuntimeError): 

156 addDataIdValue(self.butler, "exposure", 101, instrument="dummyCam", physical_filter="k2020") 

157 

158 # No skymap defined 

159 with self.assertRaises(RuntimeError): 

160 addDataIdValue(self.butler, "tract", 42) 

161 # Didn't create skymap "map" first. 

162 with self.assertRaises(RuntimeError): 

163 addDataIdValue(self.butler, "tract", 43, skymap="map") 

164 

165 def testAddDatasetType(self): 

166 # 1 for StructuredDataNoComponents, 1 for StructuredData (components 

167 # not included). 

168 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=False))), 2) 

169 

170 # Testing the DatasetType objects is not practical, because all tests 

171 # need a DimensionUniverse. So just check that we have the dataset 

172 # types we expect. 

173 self.butler.registry.getDatasetType("DataType1") 

174 self.butler.registry.getDatasetType("DataType2") 

175 

176 with self.assertRaises(ValueError): 

177 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray") 

178 with self.assertRaises(ValueError): 

179 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType") 

180 

181 def testRegisterMetricsExample(self): 

182 id1 = {"instrument": "notACam"} 

183 id2 = expandUniqueId(self.butler, {"visit": 101}) 

184 data = MetricsExample(summary={"answer": 42, "question": "unknown"}) 

185 

186 self.butler.put(data, "DataType1", id1) 

187 self.assertEqual(self.butler.get("DataType1", id1), data) 

188 

189 self.butler.put(data, "DataType2", id2) 

190 self.assertEqual(self.butler.get("DataType2", id2), data) 

191 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary) 

192 

193 def testRegisterMetricsExampleChained(self): 

194 """Regression test for registerMetricsExample having no effect 

195 on ChainedDatastore. 

196 """ 

197 temp = makeTestTempDir(TESTDIR) 

198 try: 

199 config = lsst.daf.butler.Config() 

200 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore" 

201 config["datastore", "datastores"] = [ 

202 { 

203 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore", 

204 } 

205 ] 

206 

207 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config) 

208 butler = lsst.daf.butler.Butler(repo, run="chainedExample") 

209 registerMetricsExample(butler) 

210 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents") 

211 

212 data = MetricsExample(summary={}) 

213 # Should not raise 

214 butler.put(data, "DummyType") 

215 finally: 

216 shutil.rmtree(temp, ignore_errors=True) 

217 

218 def testUniqueButler(self): 

219 dataId = {"instrument": "notACam"} 

220 ref = self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId) 

221 self.assertTrue(self.butler.exists("DataType1", dataId)) 

222 self.assertTrue(self.butler.exists(ref)) 

223 

224 newButler = makeTestCollection(self.creatorButler) 

225 

226 # Can not be found in the new default collection. 

227 self.assertFalse(newButler.exists("DataType1", dataId)) 

228 

229 # The ref does exist in the new butler though. 

230 self.assertTrue(newButler.exists(ref)) 

231 

232 def testExpandUniqueId(self): 

233 self.assertEqual( 

234 dict(expandUniqueId(self.butler, {"instrument": "notACam"})), {"instrument": "notACam"} 

235 ) 

236 self.assertIn( 

237 dict(expandUniqueId(self.butler, {"visit": 101})), 

238 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}], 

239 ) 

240 self.assertIn( 

241 dict(expandUniqueId(self.butler, {"detector": 5})), 

242 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}], 

243 ) 

244 self.assertIn( 

245 dict(expandUniqueId(self.butler, {"physical_filter": "k2020"})), 

246 [ 

247 {"instrument": "notACam", "physical_filter": "k2020"}, 

248 {"instrument": "notACam", "physical_filter": "k2020"}, 

249 ], 

250 ) 

251 with self.assertRaises(ValueError): 

252 expandUniqueId(self.butler, {"tract": 42}) 

253 

254 

255if __name__ == "__main__": 

256 unittest.main()