Coverage for tests / test_testRepo.py: 23%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating 

29test repositories or butlers. 

30""" 

31 

32import os 

33import shutil 

34import unittest 

35 

36import lsst.daf.butler 

37from lsst.daf.butler.tests import ( 

38 MetricsExample, 

39 addDataIdValue, 

40 addDatasetType, 

41 expandUniqueId, 

42 makeTestCollection, 

43 makeTestRepo, 

44 registerMetricsExample, 

45) 

46from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir 

47 

48TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

49 

50 

51class ButlerTestRepoTestCase(unittest.TestCase): 

52 """Simpler test than below without setUpClass getting in the way.""" 

53 

54 def setUp(self): 

55 self.root = makeTestTempDir(TESTDIR) 

56 

57 def tearDown(self): 

58 removeTestTempDir(self.root) 

59 

60 def testMakeTestRepo(self): 

61 dataIds = { 

62 "instrument": ["DummyCam"], 

63 "physical_filter": ["d-r"], 

64 "exposure": [42, 43, 44], 

65 "visit": [42, 43, 44], 

66 } 

67 

68 butler = makeTestRepo(self.root, dataIds) 

69 self.enterContext(butler) 

70 

71 records = list(butler.registry.queryDimensionRecords("visit")) 

72 self.assertEqual(len(records), 3) 

73 

74 

75class ButlerUtilsTestSuite(unittest.TestCase): 

76 """Test the butler test utilities.""" 

77 

78 @classmethod 

79 def setUpClass(cls): 

80 # Repository should be re-created for each test case, but 

81 # this has a prohibitive run-time cost at present 

82 cls.root = makeTestTempDir(TESTDIR) 

83 

84 cls.creatorButler = makeTestRepo(cls.root) 

85 cls.enterClassContext(cls.creatorButler) 

86 addDataIdValue(cls.creatorButler, "instrument", "notACam") 

87 addDataIdValue(cls.creatorButler, "instrument", "dummyCam") 

88 addDataIdValue(cls.creatorButler, "physical_filter", "k2020", band="k", instrument="notACam") 

89 addDataIdValue(cls.creatorButler, "physical_filter", "l2019", instrument="dummyCam") 

90 addDataIdValue(cls.creatorButler, "visit", 101, instrument="notACam", physical_filter="k2020") 

91 addDataIdValue(cls.creatorButler, "visit", 102, instrument="notACam", physical_filter="k2020") 

92 addDataIdValue(cls.creatorButler, "detector", 5) 

93 # Leave skymap/patch/tract undefined so that tests can assume 

94 # they're missing. 

95 

96 registerMetricsExample(cls.creatorButler) 

97 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents") 

98 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData") 

99 

100 @classmethod 

101 def tearDownClass(cls): 

102 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8 

103 # to keep the addition and removal together and make it more robust 

104 removeTestTempDir(cls.root) 

105 

106 def setUp(self): 

107 # TestCase.id() is unique for each test method 

108 self.butler = makeTestCollection(self.creatorButler, uniqueId=self.id()) 

109 

110 def testButlerValid(self): 

111 self.butler.validateConfiguration() 

112 

113 def testButlerKwargs(self): 

114 # outfile has the most obvious effects of any Butler.makeRepo keyword 

115 with safeTestTempDir(TESTDIR) as temp: 

116 path = os.path.join(temp, "oddConfig.json") 

117 butler = makeTestRepo(temp, {}, outfile=path) 

118 self.enterContext(butler) 

119 self.assertTrue(os.path.isfile(path)) 

120 

121 def _checkButlerDimension(self, dimensions, query, expected): 

122 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False)) 

123 self.assertEqual(len(result), 1) 

124 self.assertIn(result[0].required, expected) 

125 

126 def testButlerDimensions(self): 

127 self._checkButlerDimension( 

128 {"instrument"}, "instrument='notACam'", [{"instrument": "notACam"}, {"instrument": "dummyCam"}] 

129 ) 

130 self._checkButlerDimension( 

131 {"visit", "instrument"}, 

132 "visit=101", 

133 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}], 

134 ) 

135 self._checkButlerDimension( 

136 {"visit", "instrument"}, 

137 "visit=102", 

138 [{"instrument": "notACam", "visit": 102}, {"instrument": "dummyCam", "visit": 102}], 

139 ) 

140 self._checkButlerDimension( 

141 {"detector", "instrument"}, 

142 "detector=5", 

143 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}], 

144 ) 

145 

146 def testAddDataIdValue(self): 

147 addDataIdValue(self.butler, "visit", 1, instrument="notACam", physical_filter="k2020") 

148 self._checkButlerDimension( 

149 {"visit", "instrument"}, "visit=1", [{"instrument": "notACam", "visit": 1}] 

150 ) 

151 addDataIdValue(self.butler, "visit", 2, instrument="dummyCam", physical_filter="l2019") 

152 self._checkButlerDimension( 

153 {"visit", "instrument"}, "visit=2", [{"instrument": "dummyCam", "visit": 2}] 

154 ) 

155 

156 with self.assertRaises(ValueError): 

157 addDataIdValue(self.butler, "NotADimension", 42) 

158 with self.assertRaises(ValueError): 

159 addDataIdValue(self.butler, "detector", "nonNumeric") 

160 with self.assertRaises(ValueError): 

161 addDataIdValue(self.butler, "detector", 101, nonsenseField="string") 

162 

163 # Keywords imply different instruments 

164 with self.assertRaises(RuntimeError): 

165 addDataIdValue(self.butler, "exposure", 101, instrument="dummyCam", physical_filter="k2020") 

166 

167 # No skymap defined 

168 with self.assertRaises(RuntimeError): 

169 addDataIdValue(self.butler, "tract", 42) 

170 # Didn't create skymap "map" first. 

171 with self.assertRaises(RuntimeError): 

172 addDataIdValue(self.butler, "tract", 43, skymap="map") 

173 

174 def testAddDatasetType(self): 

175 # 1 for StructuredDataNoComponents, 1 for StructuredData (components 

176 # not included). 

177 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes())), 2) 

178 

179 # Testing the DatasetType objects is not practical, because all tests 

180 # need a DimensionUniverse. So just check that we have the dataset 

181 # types we expect. 

182 self.butler.get_dataset_type("DataType1") 

183 self.butler.get_dataset_type("DataType2") 

184 

185 with self.assertRaises(ValueError): 

186 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray") 

187 with self.assertRaises(ValueError): 

188 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType") 

189 

190 def testRegisterMetricsExample(self): 

191 id1 = {"instrument": "notACam"} 

192 id2 = expandUniqueId(self.butler, {"visit": 101}) 

193 data = MetricsExample(summary={"answer": 42, "question": "unknown"}) 

194 

195 self.butler.put(data, "DataType1", id1) 

196 self.assertEqual(self.butler.get("DataType1", id1), data) 

197 

198 self.butler.put(data, "DataType2", id2) 

199 self.assertEqual(self.butler.get("DataType2", id2), data) 

200 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary) 

201 

202 def testRegisterMetricsExampleChained(self): 

203 """Regression test for registerMetricsExample having no effect 

204 on ChainedDatastore. 

205 """ 

206 temp = makeTestTempDir(TESTDIR) 

207 try: 

208 config = lsst.daf.butler.Config() 

209 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore" 

210 config["datastore", "datastores"] = [ 

211 { 

212 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore", 

213 } 

214 ] 

215 

216 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config) 

217 butler = lsst.daf.butler.Butler.from_config(repo, run="chainedExample") 

218 self.enterContext(butler) 

219 registerMetricsExample(butler) 

220 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents") 

221 

222 data = MetricsExample(summary={}) 

223 # Should not raise 

224 butler.put(data, "DummyType") 

225 finally: 

226 shutil.rmtree(temp, ignore_errors=True) 

227 

228 def testUniqueButler(self): 

229 dataId = {"instrument": "notACam"} 

230 ref = self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId) 

231 self.assertTrue(self.butler.exists("DataType1", dataId)) 

232 self.assertTrue(self.butler.exists(ref)) 

233 

234 newButler = makeTestCollection(self.creatorButler) 

235 

236 # Can not be found in the new default collection. 

237 self.assertFalse(newButler.exists("DataType1", dataId)) 

238 

239 # The ref does exist in the new butler though. 

240 self.assertTrue(newButler.exists(ref)) 

241 

242 def testExpandUniqueId(self): 

243 self.assertEqual( 

244 expandUniqueId(self.butler, {"instrument": "notACam"}).required, {"instrument": "notACam"} 

245 ) 

246 self.assertIn( 

247 expandUniqueId(self.butler, {"visit": 101}).required, 

248 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}], 

249 ) 

250 self.assertIn( 

251 expandUniqueId(self.butler, {"detector": 5}).required, 

252 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}], 

253 ) 

254 self.assertIn( 

255 expandUniqueId(self.butler, {"physical_filter": "k2020"}).required, 

256 [ 

257 {"instrument": "notACam", "physical_filter": "k2020"}, 

258 {"instrument": "notACam", "physical_filter": "k2020"}, 

259 ], 

260 ) 

261 with self.assertRaises(ValueError): 

262 expandUniqueId(self.butler, {"tract": 42}) 

263 

264 

265if __name__ == "__main__": 

266 unittest.main()