Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23import pickle 

24 

25from lsst.daf.butler import ( 

26 DataCoordinate, 

27 DatasetType, 

28 DatasetRef, 

29 DimensionUniverse, 

30 StorageClass, 

31 StorageClassFactory, 

32) 

33 

34"""Tests for datasets module. 

35""" 

36 

37 

38class DatasetTypeTestCase(unittest.TestCase): 

39 """Test for DatasetType. 

40 """ 

41 def setUp(self): 

42 self.universe = DimensionUniverse() 

43 

44 def testConstructor(self): 

45 """Test construction preserves values. 

46 

47 Note that construction doesn't check for valid storageClass. 

48 This can only be verified for a particular schema. 

49 """ 

50 datasetTypeName = "test" 

51 storageClass = StorageClass("test_StructuredData") 

52 dimensions = self.universe.extract(("instrument", "visit")) 

53 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

54 self.assertEqual(datasetType.name, datasetTypeName) 

55 self.assertEqual(datasetType.storageClass, storageClass) 

56 self.assertEqual(datasetType.dimensions, dimensions) 

57 

58 def testConstructor2(self): 

59 """Test construction from StorageClass name. 

60 """ 

61 datasetTypeName = "test" 

62 storageClass = StorageClass("test_constructor2") 

63 StorageClassFactory().registerStorageClass(storageClass) 

64 dimensions = self.universe.extract(("instrument", "visit")) 

65 datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2") 

66 self.assertEqual(datasetType.name, datasetTypeName) 

67 self.assertEqual(datasetType.storageClass, storageClass) 

68 self.assertEqual(datasetType.dimensions, dimensions) 

69 

70 def testNameValidation(self): 

71 """Test that dataset type names only contain certain characters 

72 in certain positions. 

73 """ 

74 dimensions = self.universe.extract(("instrument", "visit")) 

75 storageClass = StorageClass("test_StructuredData") 

76 goodNames = ("a", "A", "z1", "Z1", "a_1B", "A_1b") 

77 badNames = ("1", "_", "a%b", "B+Z", "T[0]") 

78 for name in goodNames: 

79 self.assertEqual(DatasetType(name, dimensions, storageClass).name, name) 

80 for suffix in goodNames: 

81 full = f"{name}.{suffix}" 

82 self.assertEqual(DatasetType(full, dimensions, storageClass).name, full) 

83 for suffix in badNames: 

84 full = f"{name}.{suffix}" 

85 with self.subTest(full=full): 

86 with self.assertRaises(ValueError): 

87 DatasetType(full, dimensions, storageClass) 

88 for name in badNames: 

89 with self.subTest(name=name): 

90 with self.assertRaises(ValueError): 

91 DatasetType(name, dimensions, storageClass) 

92 

93 def testEquality(self): 

94 storageA = StorageClass("test_a") 

95 storageB = StorageClass("test_b") 

96 dimensionsA = self.universe.extract(["instrument"]) 

97 dimensionsB = self.universe.extract(["skymap"]) 

98 self.assertEqual(DatasetType("a", dimensionsA, storageA,), 

99 DatasetType("a", dimensionsA, storageA,)) 

100 self.assertEqual(DatasetType("a", dimensionsA, "test_a",), 

101 DatasetType("a", dimensionsA, storageA,)) 

102 self.assertEqual(DatasetType("a", dimensionsA, storageA,), 

103 DatasetType("a", dimensionsA, "test_a",)) 

104 self.assertEqual(DatasetType("a", dimensionsA, "test_a",), 

105 DatasetType("a", dimensionsA, "test_a",)) 

106 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

107 DatasetType("b", dimensionsA, storageA,)) 

108 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

109 DatasetType("b", dimensionsA, "test_a",)) 

110 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

111 DatasetType("a", dimensionsA, storageB,)) 

112 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

113 DatasetType("a", dimensionsA, "test_b",)) 

114 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

115 DatasetType("a", dimensionsB, storageA,)) 

116 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

117 DatasetType("a", dimensionsB, "test_a",)) 

118 

119 def testHashability(self): 

120 """Test `DatasetType.__hash__`. 

121 

122 This test is performed by checking that `DatasetType` entries can 

123 be inserted into a `set` and that unique values of its 

124 (`name`, `storageClass`, `dimensions`) parameters result in separate 

125 entries (and equal ones don't). 

126 

127 This does not check for uniformity of hashing or the actual values 

128 of the hash function. 

129 """ 

130 types = [] 

131 unique = 0 

132 storageC = StorageClass("test_c") 

133 storageD = StorageClass("test_d") 

134 for name in ["a", "b"]: 

135 for storageClass in [storageC, storageD]: 

136 for dimensions in [("instrument", ), ("skymap", )]: 

137 datasetType = DatasetType(name, self.universe.extract(dimensions), storageClass) 

138 datasetTypeCopy = DatasetType(name, self.universe.extract(dimensions), storageClass) 

139 types.extend((datasetType, datasetTypeCopy)) 

140 unique += 1 # datasetType should always equal its copy 

141 self.assertEqual(len(set(types)), unique) # all other combinations are unique 

142 

143 # also check that hashes of instances constructed with StorageClass 

144 # name matches hashes of instances constructed with instances 

145 dimensions = self.universe.extract(["instrument"]) 

146 self.assertEqual(hash(DatasetType("a", dimensions, storageC)), 

147 hash(DatasetType("a", dimensions, "test_c"))) 

148 self.assertEqual(hash(DatasetType("a", dimensions, "test_c")), 

149 hash(DatasetType("a", dimensions, "test_c"))) 

150 self.assertNotEqual(hash(DatasetType("a", dimensions, storageC)), 

151 hash(DatasetType("a", dimensions, "test_d"))) 

152 self.assertNotEqual(hash(DatasetType("a", dimensions, storageD)), 

153 hash(DatasetType("a", dimensions, "test_c"))) 

154 self.assertNotEqual(hash(DatasetType("a", dimensions, "test_c")), 

155 hash(DatasetType("a", dimensions, "test_d"))) 

156 

157 def testPickle(self): 

158 """Test pickle support. 

159 """ 

160 storageClass = StorageClass("test_pickle") 

161 datasetTypeName = "test" 

162 dimensions = self.universe.extract(("instrument", "visit")) 

163 # Un-pickling requires that storage class is registered with factory. 

164 StorageClassFactory().registerStorageClass(storageClass) 

165 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

166 datasetTypeOut = pickle.loads(pickle.dumps(datasetType)) 

167 self.assertIsInstance(datasetTypeOut, DatasetType) 

168 self.assertEqual(datasetType.name, datasetTypeOut.name) 

169 self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names) 

170 self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass) 

171 

172 def test_composites(self): 

173 """Test components within composite DatasetTypes.""" 

174 storageClassA = StorageClass("compA") 

175 storageClassB = StorageClass("compB") 

176 storageClass = StorageClass("test_composite", components={"compA": storageClassA, 

177 "compB": storageClassB}) 

178 self.assertTrue(storageClass.isComposite()) 

179 self.assertFalse(storageClassA.isComposite()) 

180 self.assertFalse(storageClassB.isComposite()) 

181 

182 dimensions = self.universe.extract(("instrument", "visit")) 

183 

184 datasetTypeComposite = DatasetType("composite", dimensions, storageClass) 

185 datasetTypeComponentA = DatasetType("composite.compA", dimensions, storageClassA) 

186 datasetTypeComponentB = DatasetType("composite.compB", dimensions, storageClassB) 

187 

188 self.assertTrue(datasetTypeComposite.isComposite()) 

189 self.assertFalse(datasetTypeComponentA.isComposite()) 

190 self.assertTrue(datasetTypeComponentB.isComponent()) 

191 self.assertFalse(datasetTypeComposite.isComponent()) 

192 

193 self.assertEqual(datasetTypeComposite.name, "composite") 

194 self.assertEqual(datasetTypeComponentA.name, "composite.compA") 

195 self.assertEqual(datasetTypeComponentB.component(), "compB") 

196 self.assertEqual(datasetTypeComposite.nameAndComponent(), ("composite", None)) 

197 self.assertEqual(datasetTypeComponentA.nameAndComponent(), ("composite", "compA")) 

198 

199 

200class DatasetRefTestCase(unittest.TestCase): 

201 """Test for DatasetRef. 

202 """ 

203 

204 def setUp(self): 

205 self.universe = DimensionUniverse() 

206 datasetTypeName = "test" 

207 self.componentStorageClass1 = StorageClass("Component1") 

208 self.componentStorageClass2 = StorageClass("Component2") 

209 self.parentStorageClass = StorageClass("Parent", components={"a": self.componentStorageClass1, 

210 "b": self.componentStorageClass2}) 

211 dimensions = self.universe.extract(("instrument", "visit")) 

212 self.dataId = dict(instrument="DummyCam", visit=42) 

213 self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass) 

214 

215 def testConstructor(self): 

216 """Test that construction preserves and validates values. 

217 """ 

218 # Construct an unresolved ref. 

219 ref = DatasetRef(self.datasetType, self.dataId) 

220 self.assertEqual(ref.datasetType, self.datasetType) 

221 self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), 

222 msg=ref.dataId) 

223 self.assertIsInstance(ref.dataId, DataCoordinate) 

224 self.assertIsNone(ref.components) 

225 # Constructing an unresolved ref with run and/or components should 

226 # fail. 

227 run = "somerun" 

228 with self.assertRaises(ValueError): 

229 DatasetRef(self.datasetType, self.dataId, run=run) 

230 components = { 

231 "a": DatasetRef(self.datasetType.makeComponentDatasetType("a"), self.dataId, id=2, run=run) 

232 } 

233 with self.assertRaises(ValueError): 

234 DatasetRef(self.datasetType, self.dataId, components=components) 

235 # Passing a data ID that is missing dimensions should fail. 

236 with self.assertRaises(KeyError): 

237 DatasetRef(self.datasetType, {"instrument": "DummyCam"}) 

238 # Constructing a resolved ref should preserve run and components, 

239 # as well as everything else. 

240 ref = DatasetRef(self.datasetType, self.dataId, id=1, run=run, components=components) 

241 self.assertEqual(ref.datasetType, self.datasetType) 

242 self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), 

243 msg=ref.dataId) 

244 self.assertIsInstance(ref.dataId, DataCoordinate) 

245 self.assertEqual(ref.id, 1) 

246 self.assertEqual(ref.run, run) 

247 self.assertEqual(ref.components, components) 

248 # Constructing a resolved ref with bad component storage classes 

249 # should fail. 

250 with self.assertRaises(ValueError): 

251 DatasetRef(self.datasetType, self.dataId, id=1, run=run, components={"b": components["a"]}) 

252 # Constructing a resolved ref with unresolved components should fail. 

253 with self.assertRaises(ValueError): 

254 DatasetRef(self.datasetType, self.dataId, id=1, run=run, 

255 components={"a": components["a"].unresolved()}) 

256 # Constructing a resolved ref with bad component names should fail. 

257 with self.assertRaises(ValueError): 

258 DatasetRef(self.datasetType, self.dataId, id=1, run=run, 

259 components={"c": components["a"]}) 

260 

261 def testResolving(self): 

262 ref = DatasetRef(self.datasetType, self.dataId, id=1, run="somerun") 

263 unresolvedRef = ref.unresolved() 

264 self.assertIsNotNone(ref.id) 

265 self.assertIsNone(unresolvedRef.id) 

266 self.assertIsNone(unresolvedRef.run) 

267 self.assertIsNone(unresolvedRef.components) 

268 self.assertNotEqual(ref, unresolvedRef) 

269 self.assertEqual(ref.unresolved(), unresolvedRef) 

270 self.assertEqual(ref.datasetType, unresolvedRef.datasetType) 

271 self.assertEqual(ref.dataId, unresolvedRef.dataId) 

272 reresolvedRef = unresolvedRef.resolved(id=1, run="somerun") 

273 self.assertEqual(ref, reresolvedRef) 

274 self.assertEqual(reresolvedRef.unresolved(), unresolvedRef) 

275 self.assertIsNotNone(reresolvedRef.run) 

276 self.assertIsNotNone(reresolvedRef.components) 

277 

278 def testPickle(self): 

279 ref = DatasetRef(self.datasetType, self.dataId, id=1, run="somerun") 

280 s = pickle.dumps(ref) 

281 self.assertEqual(pickle.loads(s), ref) 

282 

283 

284if __name__ == "__main__": 284 ↛ 285line 284 didn't jump to line 285, because the condition on line 284 was never true

285 unittest.main()