Coverage for tests/test_testRepo.py: 22%

117 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-30 02:51 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating 

29test repositories or butlers. 

30""" 

31 

32import os 

33import shutil 

34import unittest 

35 

36import lsst.daf.butler 

37from lsst.daf.butler.tests import ( 

38 MetricsExample, 

39 addDataIdValue, 

40 addDatasetType, 

41 expandUniqueId, 

42 makeTestCollection, 

43 makeTestRepo, 

44 registerMetricsExample, 

45) 

46from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir 

47 

48TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

49 

50 

51class ButlerTestRepoTestCase(unittest.TestCase): 

52 """Simpler test than below without setUpClass getting in the way.""" 

53 

54 def setUp(self): 

55 self.root = makeTestTempDir(TESTDIR) 

56 

57 def tearDown(self): 

58 removeTestTempDir(self.root) 

59 

60 def testMakeTestRepo(self): 

61 dataIds = { 

62 "instrument": ["DummyCam"], 

63 "physical_filter": ["d-r"], 

64 "exposure": [42, 43, 44], 

65 "visit": [42, 43, 44], 

66 } 

67 

68 butler = makeTestRepo(self.root, dataIds) 

69 

70 records = list(butler.registry.queryDimensionRecords("visit")) 

71 self.assertEqual(len(records), 3) 

72 

73 

74class ButlerUtilsTestSuite(unittest.TestCase): 

75 """Test the butler test utilities.""" 

76 

77 @classmethod 

78 def setUpClass(cls): 

79 # Repository should be re-created for each test case, but 

80 # this has a prohibitive run-time cost at present 

81 cls.root = makeTestTempDir(TESTDIR) 

82 

83 cls.creatorButler = makeTestRepo(cls.root) 

84 addDataIdValue(cls.creatorButler, "instrument", "notACam") 

85 addDataIdValue(cls.creatorButler, "instrument", "dummyCam") 

86 addDataIdValue(cls.creatorButler, "physical_filter", "k2020", band="k", instrument="notACam") 

87 addDataIdValue(cls.creatorButler, "physical_filter", "l2019", instrument="dummyCam") 

88 addDataIdValue(cls.creatorButler, "visit", 101, instrument="notACam", physical_filter="k2020") 

89 addDataIdValue(cls.creatorButler, "visit", 102, instrument="notACam", physical_filter="k2020") 

90 addDataIdValue(cls.creatorButler, "detector", 5) 

91 # Leave skymap/patch/tract undefined so that tests can assume 

92 # they're missing. 

93 

94 registerMetricsExample(cls.creatorButler) 

95 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents") 

96 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData") 

97 

98 @classmethod 

99 def tearDownClass(cls): 

100 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8 

101 # to keep the addition and removal together and make it more robust 

102 removeTestTempDir(cls.root) 

103 

104 def setUp(self): 

105 # TestCase.id() is unique for each test method 

106 self.butler = makeTestCollection(self.creatorButler, uniqueId=self.id()) 

107 

108 def testButlerValid(self): 

109 self.butler.validateConfiguration() 

110 

111 def testButlerKwargs(self): 

112 # outfile has the most obvious effects of any Butler.makeRepo keyword 

113 with safeTestTempDir(TESTDIR) as temp: 

114 path = os.path.join(temp, "oddConfig.json") 

115 makeTestRepo(temp, {}, outfile=path) 

116 self.assertTrue(os.path.isfile(path)) 

117 

118 def _checkButlerDimension(self, dimensions, query, expected): 

119 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False)) 

120 self.assertEqual(len(result), 1) 

121 self.assertIn(result[0].required, expected) 

122 

123 def testButlerDimensions(self): 

124 self._checkButlerDimension( 

125 {"instrument"}, "instrument='notACam'", [{"instrument": "notACam"}, {"instrument": "dummyCam"}] 

126 ) 

127 self._checkButlerDimension( 

128 {"visit", "instrument"}, 

129 "visit=101", 

130 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}], 

131 ) 

132 self._checkButlerDimension( 

133 {"visit", "instrument"}, 

134 "visit=102", 

135 [{"instrument": "notACam", "visit": 102}, {"instrument": "dummyCam", "visit": 102}], 

136 ) 

137 self._checkButlerDimension( 

138 {"detector", "instrument"}, 

139 "detector=5", 

140 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}], 

141 ) 

142 

143 def testAddDataIdValue(self): 

144 addDataIdValue(self.butler, "visit", 1, instrument="notACam", physical_filter="k2020") 

145 self._checkButlerDimension( 

146 {"visit", "instrument"}, "visit=1", [{"instrument": "notACam", "visit": 1}] 

147 ) 

148 addDataIdValue(self.butler, "visit", 2, instrument="dummyCam", physical_filter="l2019") 

149 self._checkButlerDimension( 

150 {"visit", "instrument"}, "visit=2", [{"instrument": "dummyCam", "visit": 2}] 

151 ) 

152 

153 with self.assertRaises(ValueError): 

154 addDataIdValue(self.butler, "NotADimension", 42) 

155 with self.assertRaises(ValueError): 

156 addDataIdValue(self.butler, "detector", "nonNumeric") 

157 with self.assertRaises(ValueError): 

158 addDataIdValue(self.butler, "detector", 101, nonsenseField="string") 

159 

160 # Keywords imply different instruments 

161 with self.assertRaises(RuntimeError): 

162 addDataIdValue(self.butler, "exposure", 101, instrument="dummyCam", physical_filter="k2020") 

163 

164 # No skymap defined 

165 with self.assertRaises(RuntimeError): 

166 addDataIdValue(self.butler, "tract", 42) 

167 # Didn't create skymap "map" first. 

168 with self.assertRaises(RuntimeError): 

169 addDataIdValue(self.butler, "tract", 43, skymap="map") 

170 

171 def testAddDatasetType(self): 

172 # 1 for StructuredDataNoComponents, 1 for StructuredData (components 

173 # not included). 

174 with self.assertWarns(FutureWarning): 

175 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=False))), 2) 

176 

177 # Testing the DatasetType objects is not practical, because all tests 

178 # need a DimensionUniverse. So just check that we have the dataset 

179 # types we expect. 

180 self.butler.get_dataset_type("DataType1") 

181 self.butler.get_dataset_type("DataType2") 

182 

183 with self.assertRaises(ValueError): 

184 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray") 

185 with self.assertRaises(ValueError): 

186 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType") 

187 

188 def testRegisterMetricsExample(self): 

189 id1 = {"instrument": "notACam"} 

190 id2 = expandUniqueId(self.butler, {"visit": 101}) 

191 data = MetricsExample(summary={"answer": 42, "question": "unknown"}) 

192 

193 self.butler.put(data, "DataType1", id1) 

194 self.assertEqual(self.butler.get("DataType1", id1), data) 

195 

196 self.butler.put(data, "DataType2", id2) 

197 self.assertEqual(self.butler.get("DataType2", id2), data) 

198 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary) 

199 

200 def testRegisterMetricsExampleChained(self): 

201 """Regression test for registerMetricsExample having no effect 

202 on ChainedDatastore. 

203 """ 

204 temp = makeTestTempDir(TESTDIR) 

205 try: 

206 config = lsst.daf.butler.Config() 

207 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore" 

208 config["datastore", "datastores"] = [ 

209 { 

210 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore", 

211 } 

212 ] 

213 

214 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config) 

215 butler = lsst.daf.butler.Butler.from_config(repo, run="chainedExample") 

216 registerMetricsExample(butler) 

217 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents") 

218 

219 data = MetricsExample(summary={}) 

220 # Should not raise 

221 butler.put(data, "DummyType") 

222 finally: 

223 shutil.rmtree(temp, ignore_errors=True) 

224 

225 def testUniqueButler(self): 

226 dataId = {"instrument": "notACam"} 

227 ref = self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId) 

228 self.assertTrue(self.butler.exists("DataType1", dataId)) 

229 self.assertTrue(self.butler.exists(ref)) 

230 

231 newButler = makeTestCollection(self.creatorButler) 

232 

233 # Can not be found in the new default collection. 

234 self.assertFalse(newButler.exists("DataType1", dataId)) 

235 

236 # The ref does exist in the new butler though. 

237 self.assertTrue(newButler.exists(ref)) 

238 

239 def testExpandUniqueId(self): 

240 self.assertEqual( 

241 expandUniqueId(self.butler, {"instrument": "notACam"}).required, {"instrument": "notACam"} 

242 ) 

243 self.assertIn( 

244 expandUniqueId(self.butler, {"visit": 101}).required, 

245 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}], 

246 ) 

247 self.assertIn( 

248 expandUniqueId(self.butler, {"detector": 5}).required, 

249 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}], 

250 ) 

251 self.assertIn( 

252 expandUniqueId(self.butler, {"physical_filter": "k2020"}).required, 

253 [ 

254 {"instrument": "notACam", "physical_filter": "k2020"}, 

255 {"instrument": "notACam", "physical_filter": "k2020"}, 

256 ], 

257 ) 

258 with self.assertRaises(ValueError): 

259 expandUniqueId(self.butler, {"tract": 42}) 

260 

261 

262if __name__ == "__main__": 

263 unittest.main()