Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23import pickle 

24import copy 

25 

26from lsst.daf.butler import ( 

27 DataCoordinate, 

28 DatasetType, 

29 DatasetRef, 

30 DimensionUniverse, 

31 StorageClass, 

32 StorageClassFactory, 

33) 

34 

35"""Tests for datasets module. 

36""" 

37 

38 

39class DatasetTypeTestCase(unittest.TestCase): 

40 """Test for DatasetType. 

41 """ 

42 def setUp(self): 

43 self.universe = DimensionUniverse() 

44 

45 def testConstructor(self): 

46 """Test construction preserves values. 

47 

48 Note that construction doesn't check for valid storageClass. 

49 This can only be verified for a particular schema. 

50 """ 

51 datasetTypeName = "test" 

52 storageClass = StorageClass("test_StructuredData") 

53 dimensions = self.universe.extract(("instrument", "visit")) 

54 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

55 self.assertEqual(datasetType.name, datasetTypeName) 

56 self.assertEqual(datasetType.storageClass, storageClass) 

57 self.assertEqual(datasetType.dimensions, dimensions) 

58 

59 with self.assertRaises(ValueError, msg="Construct component without parent storage class"): 

60 DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), 

61 dimensions, storageClass) 

62 with self.assertRaises(ValueError, msg="Construct non-component with parent storage class"): 

63 DatasetType(datasetTypeName, 

64 dimensions, storageClass, parentStorageClass="NotAllowed") 

65 

66 def testConstructor2(self): 

67 """Test construction from StorageClass name. 

68 """ 

69 datasetTypeName = "test" 

70 storageClass = StorageClass("test_constructor2") 

71 StorageClassFactory().registerStorageClass(storageClass) 

72 dimensions = self.universe.extract(("instrument", "visit")) 

73 datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2") 

74 self.assertEqual(datasetType.name, datasetTypeName) 

75 self.assertEqual(datasetType.storageClass, storageClass) 

76 self.assertEqual(datasetType.dimensions, dimensions) 

77 

78 def testNameValidation(self): 

79 """Test that dataset type names only contain certain characters 

80 in certain positions. 

81 """ 

82 dimensions = self.universe.extract(("instrument", "visit")) 

83 goodNames = ("a", "A", "z1", "Z1", "a_1B", "A_1b") 

84 badNames = ("1", "_", "a%b", "B+Z", "T[0]") 

85 

86 # Construct storage class with all the good names included as 

87 # components so that we can test internal consistency 

88 storageClass = StorageClass("test_StructuredData", 

89 components={n: StorageClass("component") for n in goodNames}) 

90 

91 for name in goodNames: 

92 composite = DatasetType(name, dimensions, storageClass) 

93 self.assertEqual(composite.name, name) 

94 for suffix in goodNames: 

95 full = DatasetType.nameWithComponent(name, suffix) 

96 component = composite.makeComponentDatasetType(suffix) 

97 self.assertEqual(component.name, full) 

98 self.assertEqual(component.parentStorageClass.name, "test_StructuredData") 

99 for suffix in badNames: 

100 full = DatasetType.nameWithComponent(name, suffix) 

101 with self.subTest(full=full): 

102 with self.assertRaises(ValueError): 

103 DatasetType(full, dimensions, storageClass) 

104 for name in badNames: 

105 with self.subTest(name=name): 

106 with self.assertRaises(ValueError): 

107 DatasetType(name, dimensions, storageClass) 

108 

109 def testEquality(self): 

110 storageA = StorageClass("test_a") 

111 storageB = StorageClass("test_b") 

112 parent = StorageClass("test") 

113 dimensionsA = self.universe.extract(["instrument"]) 

114 dimensionsB = self.universe.extract(["skymap"]) 

115 self.assertEqual(DatasetType("a", dimensionsA, storageA,), 

116 DatasetType("a", dimensionsA, storageA,)) 

117 self.assertEqual(DatasetType("a", dimensionsA, "test_a",), 

118 DatasetType("a", dimensionsA, storageA,)) 

119 self.assertEqual(DatasetType("a", dimensionsA, storageA,), 

120 DatasetType("a", dimensionsA, "test_a",)) 

121 self.assertEqual(DatasetType("a", dimensionsA, "test_a",), 

122 DatasetType("a", dimensionsA, "test_a",)) 

123 self.assertEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent), 

124 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent)) 

125 self.assertEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"), 

126 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent")) 

127 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

128 DatasetType("b", dimensionsA, storageA,)) 

129 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

130 DatasetType("b", dimensionsA, "test_a",)) 

131 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

132 DatasetType("a", dimensionsA, storageB,)) 

133 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

134 DatasetType("a", dimensionsA, "test_b",)) 

135 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

136 DatasetType("a", dimensionsB, storageA,)) 

137 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,), 

138 DatasetType("a", dimensionsB, "test_a",)) 

139 self.assertNotEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageA), 

140 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageB)) 

141 self.assertNotEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageA"), 

142 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageB")) 

143 

144 def testParentPlaceholder(self): 

145 """Test that a parent placeholder can be replaced.""" 

146 storageComp = StorageClass("component") 

147 storageParent = StorageClass("Parent") 

148 dimensions = self.universe.extract(["instrument"]) 

149 component = DatasetType("a.b", dimensions, storageComp, 

150 parentStorageClass=DatasetType.PlaceholderParentStorageClass) 

151 self.assertIsNotNone(component.parentStorageClass) 

152 

153 with self.assertRaises(ValueError): 

154 component.finalizeParentStorageClass("parent") 

155 

156 component.finalizeParentStorageClass(storageParent) 

157 self.assertEqual(component.parentStorageClass, storageParent) 

158 

159 component = DatasetType("a.b", dimensions, storageComp, 

160 parentStorageClass=storageParent) 

161 

162 with self.assertRaises(ValueError): 

163 # Can not replace unless a placeholder 

164 component.finalizeParentStorageClass(storageComp) 

165 

166 datasetType = DatasetType("a", dimensions, storageParent) 

167 with self.assertRaises(ValueError): 

168 # Can not add parent if not component 

169 datasetType.finalizeParentStorageClass(storageComp) 

170 

171 def testHashability(self): 

172 """Test `DatasetType.__hash__`. 

173 

174 This test is performed by checking that `DatasetType` entries can 

175 be inserted into a `set` and that unique values of its 

176 (`name`, `storageClass`, `dimensions`) parameters result in separate 

177 entries (and equal ones don't). 

178 

179 This does not check for uniformity of hashing or the actual values 

180 of the hash function. 

181 """ 

182 types = [] 

183 unique = 0 

184 storageC = StorageClass("test_c") 

185 storageD = StorageClass("test_d") 

186 for name in ["a", "b"]: 

187 for storageClass in [storageC, storageD]: 

188 for dimensions in [("instrument", ), ("skymap", )]: 

189 datasetType = DatasetType(name, self.universe.extract(dimensions), storageClass) 

190 datasetTypeCopy = DatasetType(name, self.universe.extract(dimensions), storageClass) 

191 types.extend((datasetType, datasetTypeCopy)) 

192 unique += 1 # datasetType should always equal its copy 

193 self.assertEqual(len(set(types)), unique) # all other combinations are unique 

194 

195 # also check that hashes of instances constructed with StorageClass 

196 # name matches hashes of instances constructed with instances 

197 dimensions = self.universe.extract(["instrument"]) 

198 self.assertEqual(hash(DatasetType("a", dimensions, storageC)), 

199 hash(DatasetType("a", dimensions, "test_c"))) 

200 self.assertEqual(hash(DatasetType("a", dimensions, "test_c")), 

201 hash(DatasetType("a", dimensions, "test_c"))) 

202 self.assertNotEqual(hash(DatasetType("a", dimensions, storageC)), 

203 hash(DatasetType("a", dimensions, "test_d"))) 

204 self.assertNotEqual(hash(DatasetType("a", dimensions, storageD)), 

205 hash(DatasetType("a", dimensions, "test_c"))) 

206 self.assertNotEqual(hash(DatasetType("a", dimensions, "test_c")), 

207 hash(DatasetType("a", dimensions, "test_d"))) 

208 

209 def testDeepCopy(self): 

210 """Test that we can copy a dataset type.""" 

211 storageClass = StorageClass("test_copy") 

212 datasetTypeName = "test" 

213 dimensions = self.universe.extract(("instrument", "visit")) 

214 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

215 dcopy = copy.deepcopy(datasetType) 

216 self.assertEqual(dcopy, datasetType) 

217 

218 # And again with a composite 

219 componentStorageClass = StorageClass("copy_component") 

220 componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), 

221 dimensions, componentStorageClass, 

222 parentStorageClass=storageClass) 

223 dcopy = copy.deepcopy(componentDatasetType) 

224 self.assertEqual(dcopy, componentDatasetType) 

225 

226 def testPickle(self): 

227 """Test pickle support. 

228 """ 

229 storageClass = StorageClass("test_pickle") 

230 datasetTypeName = "test" 

231 dimensions = self.universe.extract(("instrument", "visit")) 

232 # Un-pickling requires that storage class is registered with factory. 

233 StorageClassFactory().registerStorageClass(storageClass) 

234 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

235 datasetTypeOut = pickle.loads(pickle.dumps(datasetType)) 

236 self.assertIsInstance(datasetTypeOut, DatasetType) 

237 self.assertEqual(datasetType.name, datasetTypeOut.name) 

238 self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names) 

239 self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass) 

240 self.assertIsNone(datasetTypeOut.parentStorageClass) 

241 

242 # And again with a composite 

243 componentStorageClass = StorageClass("pickle_component") 

244 StorageClassFactory().registerStorageClass(componentStorageClass) 

245 componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), 

246 dimensions, componentStorageClass, 

247 parentStorageClass=storageClass) 

248 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) 

249 self.assertIsInstance(datasetTypeOut, DatasetType) 

250 self.assertEqual(componentDatasetType.name, datasetTypeOut.name) 

251 self.assertEqual(componentDatasetType.dimensions.names, datasetTypeOut.dimensions.names) 

252 self.assertEqual(componentDatasetType.storageClass, datasetTypeOut.storageClass) 

253 self.assertEqual(componentDatasetType.parentStorageClass, datasetTypeOut.parentStorageClass) 

254 self.assertEqual(datasetTypeOut.parentStorageClass.name, 

255 storageClass.name) 

256 self.assertEqual(datasetTypeOut, componentDatasetType) 

257 

258 # Now with a string and not a real storage class to test that 

259 # pickling doesn't force the StorageClass to be resolved 

260 componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), 

261 dimensions, "StrangeComponent", 

262 parentStorageClass="UnknownParent") 

263 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) 

264 self.assertEqual(datasetTypeOut, componentDatasetType) 

265 self.assertEqual(datasetTypeOut._parentStorageClassName, 

266 componentDatasetType._parentStorageClassName) 

267 

268 # Now with a storage class that is created by the factory 

269 factoryStorageClassClass = StorageClassFactory.makeNewStorageClass("ParentClass") 

270 factoryComponentStorageClassClass = StorageClassFactory.makeNewStorageClass("ComponentClass") 

271 componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), 

272 dimensions, factoryComponentStorageClassClass(), 

273 parentStorageClass=factoryStorageClassClass()) 

274 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) 

275 self.assertEqual(datasetTypeOut, componentDatasetType) 

276 self.assertEqual(datasetTypeOut._parentStorageClassName, 

277 componentDatasetType._parentStorageClassName) 

278 

279 def test_composites(self): 

280 """Test components within composite DatasetTypes.""" 

281 storageClassA = StorageClass("compA") 

282 storageClassB = StorageClass("compB") 

283 storageClass = StorageClass("test_composite", components={"compA": storageClassA, 

284 "compB": storageClassB}) 

285 self.assertTrue(storageClass.isComposite()) 

286 self.assertFalse(storageClassA.isComposite()) 

287 self.assertFalse(storageClassB.isComposite()) 

288 

289 dimensions = self.universe.extract(("instrument", "visit")) 

290 

291 datasetTypeComposite = DatasetType("composite", dimensions, storageClass) 

292 datasetTypeComponentA = datasetTypeComposite.makeComponentDatasetType("compA") 

293 datasetTypeComponentB = datasetTypeComposite.makeComponentDatasetType("compB") 

294 

295 self.assertTrue(datasetTypeComposite.isComposite()) 

296 self.assertFalse(datasetTypeComponentA.isComposite()) 

297 self.assertTrue(datasetTypeComponentB.isComponent()) 

298 self.assertFalse(datasetTypeComposite.isComponent()) 

299 

300 self.assertEqual(datasetTypeComposite.name, "composite") 

301 self.assertEqual(datasetTypeComponentA.name, "composite.compA") 

302 self.assertEqual(datasetTypeComponentB.component(), "compB") 

303 self.assertEqual(datasetTypeComposite.nameAndComponent(), ("composite", None)) 

304 self.assertEqual(datasetTypeComponentA.nameAndComponent(), ("composite", "compA")) 

305 

306 self.assertEqual(datasetTypeComponentA.parentStorageClass, storageClass) 

307 self.assertEqual(datasetTypeComponentB.parentStorageClass, storageClass) 

308 self.assertIsNone(datasetTypeComposite.parentStorageClass) 

309 

310 

311class DatasetRefTestCase(unittest.TestCase): 

312 """Test for DatasetRef. 

313 """ 

314 

315 def setUp(self): 

316 self.universe = DimensionUniverse() 

317 datasetTypeName = "test" 

318 self.componentStorageClass1 = StorageClass("Component1") 

319 self.componentStorageClass2 = StorageClass("Component2") 

320 self.parentStorageClass = StorageClass("Parent", components={"a": self.componentStorageClass1, 

321 "b": self.componentStorageClass2}) 

322 dimensions = self.universe.extract(("instrument", "visit")) 

323 self.dataId = dict(instrument="DummyCam", visit=42) 

324 self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass) 

325 

326 def testConstructor(self): 

327 """Test that construction preserves and validates values. 

328 """ 

329 # Construct an unresolved ref. 

330 ref = DatasetRef(self.datasetType, self.dataId) 

331 self.assertEqual(ref.datasetType, self.datasetType) 

332 self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), 

333 msg=ref.dataId) 

334 self.assertIsInstance(ref.dataId, DataCoordinate) 

335 # Constructing an unresolved ref with run and/or components should 

336 # fail. 

337 run = "somerun" 

338 with self.assertRaises(ValueError): 

339 DatasetRef(self.datasetType, self.dataId, run=run) 

340 # Passing a data ID that is missing dimensions should fail. 

341 with self.assertRaises(KeyError): 

342 DatasetRef(self.datasetType, {"instrument": "DummyCam"}) 

343 # Constructing a resolved ref should preserve run as well as everything 

344 # else. 

345 ref = DatasetRef(self.datasetType, self.dataId, id=1, run=run) 

346 self.assertEqual(ref.datasetType, self.datasetType) 

347 self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), 

348 msg=ref.dataId) 

349 self.assertIsInstance(ref.dataId, DataCoordinate) 

350 self.assertEqual(ref.id, 1) 

351 self.assertEqual(ref.run, run) 

352 

353 def testResolving(self): 

354 ref = DatasetRef(self.datasetType, self.dataId, id=1, run="somerun") 

355 unresolvedRef = ref.unresolved() 

356 self.assertIsNotNone(ref.id) 

357 self.assertIsNone(unresolvedRef.id) 

358 self.assertIsNone(unresolvedRef.run) 

359 self.assertNotEqual(ref, unresolvedRef) 

360 self.assertEqual(ref.unresolved(), unresolvedRef) 

361 self.assertEqual(ref.datasetType, unresolvedRef.datasetType) 

362 self.assertEqual(ref.dataId, unresolvedRef.dataId) 

363 reresolvedRef = unresolvedRef.resolved(id=1, run="somerun") 

364 self.assertEqual(ref, reresolvedRef) 

365 self.assertEqual(reresolvedRef.unresolved(), unresolvedRef) 

366 self.assertIsNotNone(reresolvedRef.run) 

367 

368 def testPickle(self): 

369 ref = DatasetRef(self.datasetType, self.dataId, id=1, run="somerun") 

370 s = pickle.dumps(ref) 

371 self.assertEqual(pickle.loads(s), ref) 

372 

373 

374if __name__ == "__main__": 374 ↛ 375line 374 didn't jump to line 375, because the condition on line 374 was never true

375 unittest.main()