Coverage for tests/test_datasets.py: 8%

294 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-15 09:13 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import copy 

23import pickle 

24import unittest 

25import uuid 

26 

27from lsst.daf.butler import ( 

28 DataCoordinate, 

29 DatasetRef, 

30 DatasetType, 

31 DimensionUniverse, 

32 FileDataset, 

33 StorageClass, 

34 StorageClassFactory, 

35) 

36 

37"""Tests for datasets module. 

38""" 

39 

40 

41class DatasetTypeTestCase(unittest.TestCase): 

42 """Test for DatasetType.""" 

43 

44 def setUp(self): 

45 self.universe = DimensionUniverse() 

46 

47 def testConstructor(self): 

48 """Test construction preserves values. 

49 

50 Note that construction doesn't check for valid storageClass. 

51 This can only be verified for a particular schema. 

52 """ 

53 datasetTypeName = "test" 

54 storageClass = StorageClass("test_StructuredData") 

55 dimensions = self.universe.extract(("visit", "instrument")) 

56 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

57 self.assertEqual(datasetType.name, datasetTypeName) 

58 self.assertEqual(datasetType.storageClass, storageClass) 

59 self.assertEqual(datasetType.dimensions, dimensions) 

60 

61 with self.assertRaises(ValueError, msg="Construct component without parent storage class"): 

62 DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), dimensions, storageClass) 

63 with self.assertRaises(ValueError, msg="Construct non-component with parent storage class"): 

64 DatasetType(datasetTypeName, dimensions, storageClass, parentStorageClass="NotAllowed") 

65 

66 def testConstructor2(self): 

67 """Test construction from StorageClass name.""" 

68 datasetTypeName = "test" 

69 storageClass = StorageClass("test_constructor2") 

70 StorageClassFactory().registerStorageClass(storageClass) 

71 dimensions = self.universe.extract(("instrument", "visit")) 

72 datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2") 

73 self.assertEqual(datasetType.name, datasetTypeName) 

74 self.assertEqual(datasetType.storageClass, storageClass) 

75 self.assertEqual(datasetType.dimensions, dimensions) 

76 

77 def testNameValidation(self): 

78 """Test that dataset type names only contain certain characters 

79 in certain positions. 

80 """ 

81 dimensions = self.universe.extract(("instrument", "visit")) 

82 goodNames = ("a", "A", "z1", "Z1", "a_1B", "A_1b", "_a") 

83 badNames = ("1", "a%b", "B+Z", "T[0]") 

84 

85 # Construct storage class with all the good names included as 

86 # components so that we can test internal consistency 

87 storageClass = StorageClass( 

88 "test_StructuredData", components={n: StorageClass("component") for n in goodNames} 

89 ) 

90 

91 for name in goodNames: 

92 composite = DatasetType(name, dimensions, storageClass) 

93 self.assertEqual(composite.name, name) 

94 for suffix in goodNames: 

95 full = DatasetType.nameWithComponent(name, suffix) 

96 component = composite.makeComponentDatasetType(suffix) 

97 self.assertEqual(component.name, full) 

98 self.assertEqual(component.parentStorageClass.name, "test_StructuredData") 

99 for suffix in badNames: 

100 full = DatasetType.nameWithComponent(name, suffix) 

101 with self.subTest(full=full): 

102 with self.assertRaises(ValueError): 

103 DatasetType(full, dimensions, storageClass) 

104 for name in badNames: 

105 with self.subTest(name=name): 

106 with self.assertRaises(ValueError): 

107 DatasetType(name, dimensions, storageClass) 

108 

109 def testEquality(self): 

110 storageA = StorageClass("test_a") 

111 storageB = StorageClass("test_b") 

112 parent = StorageClass("test") 

113 dimensionsA = self.universe.extract(["instrument"]) 

114 dimensionsB = self.universe.extract(["skymap"]) 

115 self.assertEqual( 

116 DatasetType( 

117 "a", 

118 dimensionsA, 

119 storageA, 

120 ), 

121 DatasetType( 

122 "a", 

123 dimensionsA, 

124 storageA, 

125 ), 

126 ) 

127 self.assertEqual( 

128 DatasetType( 

129 "a", 

130 dimensionsA, 

131 "test_a", 

132 ), 

133 DatasetType( 

134 "a", 

135 dimensionsA, 

136 storageA, 

137 ), 

138 ) 

139 self.assertEqual( 

140 DatasetType( 

141 "a", 

142 dimensionsA, 

143 storageA, 

144 ), 

145 DatasetType( 

146 "a", 

147 dimensionsA, 

148 "test_a", 

149 ), 

150 ) 

151 self.assertEqual( 

152 DatasetType( 

153 "a", 

154 dimensionsA, 

155 "test_a", 

156 ), 

157 DatasetType( 

158 "a", 

159 dimensionsA, 

160 "test_a", 

161 ), 

162 ) 

163 self.assertEqual( 

164 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent), 

165 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent), 

166 ) 

167 self.assertEqual( 

168 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"), 

169 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"), 

170 ) 

171 self.assertNotEqual( 

172 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent", isCalibration=True), 

173 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent", isCalibration=False), 

174 ) 

175 self.assertNotEqual( 

176 DatasetType( 

177 "a", 

178 dimensionsA, 

179 storageA, 

180 ), 

181 DatasetType( 

182 "b", 

183 dimensionsA, 

184 storageA, 

185 ), 

186 ) 

187 self.assertNotEqual( 

188 DatasetType( 

189 "a", 

190 dimensionsA, 

191 storageA, 

192 ), 

193 DatasetType( 

194 "b", 

195 dimensionsA, 

196 "test_a", 

197 ), 

198 ) 

199 self.assertNotEqual( 

200 DatasetType( 

201 "a", 

202 dimensionsA, 

203 storageA, 

204 ), 

205 DatasetType( 

206 "a", 

207 dimensionsA, 

208 storageB, 

209 ), 

210 ) 

211 self.assertNotEqual( 

212 DatasetType( 

213 "a", 

214 dimensionsA, 

215 storageA, 

216 ), 

217 DatasetType( 

218 "a", 

219 dimensionsA, 

220 "test_b", 

221 ), 

222 ) 

223 self.assertNotEqual( 

224 DatasetType( 

225 "a", 

226 dimensionsA, 

227 storageA, 

228 ), 

229 DatasetType( 

230 "a", 

231 dimensionsB, 

232 storageA, 

233 ), 

234 ) 

235 self.assertNotEqual( 

236 DatasetType( 

237 "a", 

238 dimensionsA, 

239 storageA, 

240 ), 

241 DatasetType( 

242 "a", 

243 dimensionsB, 

244 "test_a", 

245 ), 

246 ) 

247 self.assertNotEqual( 

248 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageA), 

249 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageB), 

250 ) 

251 self.assertNotEqual( 

252 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageA"), 

253 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageB"), 

254 ) 

255 

256 def testCompatibility(self): 

257 storageA = StorageClass("test_a", pytype=set, converters={"list": "builtins.set"}) 

258 storageB = StorageClass("test_b", pytype=list) 

259 storageC = StorageClass("test_c", pytype=dict) 

260 self.assertTrue(storageA.can_convert(storageB)) 

261 dimensionsA = self.universe.extract(["instrument"]) 

262 

263 dA = DatasetType("a", dimensionsA, storageA) 

264 dA2 = DatasetType("a", dimensionsA, storageB) 

265 self.assertNotEqual(dA, dA2) 

266 self.assertTrue(dA.is_compatible_with(dA)) 

267 self.assertTrue(dA.is_compatible_with(dA2)) 

268 self.assertFalse(dA2.is_compatible_with(dA)) 

269 

270 dA3 = DatasetType("a", dimensionsA, storageC) 

271 self.assertFalse(dA.is_compatible_with(dA3)) 

272 

273 def testOverrideStorageClass(self): 

274 storageA = StorageClass("test_a", pytype=list, converters={"dict": "builtins.list"}) 

275 storageB = StorageClass("test_b", pytype=dict) 

276 dimensions = self.universe.extract(["instrument"]) 

277 

278 dA = DatasetType("a", dimensions, storageA) 

279 dB = dA.overrideStorageClass(storageB) 

280 self.assertNotEqual(dA, dB) 

281 self.assertEqual(dB.storageClass, storageB) 

282 

283 round_trip = dB.overrideStorageClass(storageA) 

284 self.assertEqual(round_trip, dA) 

285 

286 # Check that parents move over. 

287 parent = StorageClass("composite", components={"a": storageA, "c": storageA}) 

288 dP = DatasetType("comp", dimensions, parent) 

289 dP_A = dP.makeComponentDatasetType("a") 

290 print(dP_A) 

291 dp_B = dP_A.overrideStorageClass(storageB) 

292 self.assertEqual(dp_B.storageClass, storageB) 

293 self.assertEqual(dp_B.parentStorageClass, parent) 

294 

295 def testJson(self): 

296 storageA = StorageClass("test_a") 

297 dimensionsA = self.universe.extract(["instrument"]) 

298 self.assertEqual( 

299 DatasetType( 

300 "a", 

301 dimensionsA, 

302 storageA, 

303 ), 

304 DatasetType.from_json( 

305 DatasetType( 

306 "a", 

307 dimensionsA, 

308 storageA, 

309 ).to_json(), 

310 self.universe, 

311 ), 

312 ) 

313 self.assertEqual( 

314 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"), 

315 DatasetType.from_json( 

316 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent").to_json(), 

317 self.universe, 

318 ), 

319 ) 

320 

321 def testSorting(self): 

322 """Can we sort a DatasetType""" 

323 storage = StorageClass("test_a") 

324 dimensions = self.universe.extract(["instrument"]) 

325 

326 d_a = DatasetType("a", dimensions, storage) 

327 d_f = DatasetType("f", dimensions, storage) 

328 d_p = DatasetType("p", dimensions, storage) 

329 

330 sort = sorted([d_p, d_f, d_a]) 

331 self.assertEqual(sort, [d_a, d_f, d_p]) 

332 

333 # Now with strings 

334 with self.assertRaises(TypeError): 

335 sort = sorted(["z", d_p, "c", d_f, d_a, "d"]) 

336 

337 def testHashability(self): 

338 """Test `DatasetType.__hash__`. 

339 

340 This test is performed by checking that `DatasetType` entries can 

341 be inserted into a `set` and that unique values of its 

342 (`name`, `storageClass`, `dimensions`) parameters result in separate 

343 entries (and equal ones don't). 

344 

345 This does not check for uniformity of hashing or the actual values 

346 of the hash function. 

347 """ 

348 types = [] 

349 unique = 0 

350 storageC = StorageClass("test_c") 

351 storageD = StorageClass("test_d") 

352 for name in ["a", "b"]: 

353 for storageClass in [storageC, storageD]: 

354 for dimensions in [("instrument",), ("skymap",)]: 

355 datasetType = DatasetType(name, self.universe.extract(dimensions), storageClass) 

356 datasetTypeCopy = DatasetType(name, self.universe.extract(dimensions), storageClass) 

357 types.extend((datasetType, datasetTypeCopy)) 

358 unique += 1 # datasetType should always equal its copy 

359 self.assertEqual(len(set(types)), unique) # all other combinations are unique 

360 

361 # also check that hashes of instances constructed with StorageClass 

362 # name matches hashes of instances constructed with instances 

363 dimensions = self.universe.extract(["instrument"]) 

364 self.assertEqual( 

365 hash(DatasetType("a", dimensions, storageC)), hash(DatasetType("a", dimensions, "test_c")) 

366 ) 

367 self.assertEqual( 

368 hash(DatasetType("a", dimensions, "test_c")), hash(DatasetType("a", dimensions, "test_c")) 

369 ) 

370 self.assertNotEqual( 

371 hash(DatasetType("a", dimensions, storageC)), hash(DatasetType("a", dimensions, "test_d")) 

372 ) 

373 self.assertNotEqual( 

374 hash(DatasetType("a", dimensions, storageD)), hash(DatasetType("a", dimensions, "test_c")) 

375 ) 

376 self.assertNotEqual( 

377 hash(DatasetType("a", dimensions, "test_c")), hash(DatasetType("a", dimensions, "test_d")) 

378 ) 

379 

380 def testDeepCopy(self): 

381 """Test that we can copy a dataset type.""" 

382 storageClass = StorageClass("test_copy") 

383 datasetTypeName = "test" 

384 dimensions = self.universe.extract(("instrument", "visit")) 

385 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

386 dcopy = copy.deepcopy(datasetType) 

387 self.assertEqual(dcopy, datasetType) 

388 

389 # Now with calibration flag set 

390 datasetType = DatasetType(datasetTypeName, dimensions, storageClass, isCalibration=True) 

391 dcopy = copy.deepcopy(datasetType) 

392 self.assertEqual(dcopy, datasetType) 

393 self.assertTrue(dcopy.isCalibration()) 

394 

395 # And again with a composite 

396 componentStorageClass = StorageClass("copy_component") 

397 componentDatasetType = DatasetType( 

398 DatasetType.nameWithComponent(datasetTypeName, "comp"), 

399 dimensions, 

400 componentStorageClass, 

401 parentStorageClass=storageClass, 

402 ) 

403 dcopy = copy.deepcopy(componentDatasetType) 

404 self.assertEqual(dcopy, componentDatasetType) 

405 

406 def testPickle(self): 

407 """Test pickle support.""" 

408 storageClass = StorageClass("test_pickle") 

409 datasetTypeName = "test" 

410 dimensions = self.universe.extract(("instrument", "visit")) 

411 # Un-pickling requires that storage class is registered with factory. 

412 StorageClassFactory().registerStorageClass(storageClass) 

413 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

414 datasetTypeOut = pickle.loads(pickle.dumps(datasetType)) 

415 self.assertIsInstance(datasetTypeOut, DatasetType) 

416 self.assertEqual(datasetType.name, datasetTypeOut.name) 

417 self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names) 

418 self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass) 

419 self.assertIsNone(datasetTypeOut.parentStorageClass) 

420 self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration()) 

421 self.assertFalse(datasetTypeOut.isCalibration()) 

422 

423 datasetType = DatasetType(datasetTypeName, dimensions, storageClass, isCalibration=True) 

424 datasetTypeOut = pickle.loads(pickle.dumps(datasetType)) 

425 self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration()) 

426 self.assertTrue(datasetTypeOut.isCalibration()) 

427 

428 # And again with a composite 

429 componentStorageClass = StorageClass("pickle_component") 

430 StorageClassFactory().registerStorageClass(componentStorageClass) 

431 componentDatasetType = DatasetType( 

432 DatasetType.nameWithComponent(datasetTypeName, "comp"), 

433 dimensions, 

434 componentStorageClass, 

435 parentStorageClass=storageClass, 

436 ) 

437 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) 

438 self.assertIsInstance(datasetTypeOut, DatasetType) 

439 self.assertEqual(componentDatasetType.name, datasetTypeOut.name) 

440 self.assertEqual(componentDatasetType.dimensions.names, datasetTypeOut.dimensions.names) 

441 self.assertEqual(componentDatasetType.storageClass, datasetTypeOut.storageClass) 

442 self.assertEqual(componentDatasetType.parentStorageClass, datasetTypeOut.parentStorageClass) 

443 self.assertEqual(datasetTypeOut.parentStorageClass.name, storageClass.name) 

444 self.assertEqual(datasetTypeOut, componentDatasetType) 

445 

446 # Now with a string and not a real storage class to test that 

447 # pickling doesn't force the StorageClass to be resolved 

448 componentDatasetType = DatasetType( 

449 DatasetType.nameWithComponent(datasetTypeName, "comp"), 

450 dimensions, 

451 "StrangeComponent", 

452 parentStorageClass="UnknownParent", 

453 ) 

454 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) 

455 self.assertEqual(datasetTypeOut, componentDatasetType) 

456 self.assertEqual(datasetTypeOut._parentStorageClassName, componentDatasetType._parentStorageClassName) 

457 

458 # Now with a storage class that is created by the factory 

459 factoryStorageClassClass = StorageClassFactory.makeNewStorageClass("ParentClass") 

460 factoryComponentStorageClassClass = StorageClassFactory.makeNewStorageClass("ComponentClass") 

461 componentDatasetType = DatasetType( 

462 DatasetType.nameWithComponent(datasetTypeName, "comp"), 

463 dimensions, 

464 factoryComponentStorageClassClass(), 

465 parentStorageClass=factoryStorageClassClass(), 

466 ) 

467 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) 

468 self.assertEqual(datasetTypeOut, componentDatasetType) 

469 self.assertEqual(datasetTypeOut._parentStorageClassName, componentDatasetType._parentStorageClassName) 

470 

471 def test_composites(self): 

472 """Test components within composite DatasetTypes.""" 

473 storageClassA = StorageClass("compA") 

474 storageClassB = StorageClass("compB") 

475 storageClass = StorageClass( 

476 "test_composite", components={"compA": storageClassA, "compB": storageClassB} 

477 ) 

478 self.assertTrue(storageClass.isComposite()) 

479 self.assertFalse(storageClassA.isComposite()) 

480 self.assertFalse(storageClassB.isComposite()) 

481 

482 dimensions = self.universe.extract(("instrument", "visit")) 

483 

484 datasetTypeComposite = DatasetType("composite", dimensions, storageClass) 

485 datasetTypeComponentA = datasetTypeComposite.makeComponentDatasetType("compA") 

486 datasetTypeComponentB = datasetTypeComposite.makeComponentDatasetType("compB") 

487 

488 self.assertTrue(datasetTypeComposite.isComposite()) 

489 self.assertFalse(datasetTypeComponentA.isComposite()) 

490 self.assertTrue(datasetTypeComponentB.isComponent()) 

491 self.assertFalse(datasetTypeComposite.isComponent()) 

492 

493 self.assertEqual(datasetTypeComposite.name, "composite") 

494 self.assertEqual(datasetTypeComponentA.name, "composite.compA") 

495 self.assertEqual(datasetTypeComponentB.component(), "compB") 

496 self.assertEqual(datasetTypeComposite.nameAndComponent(), ("composite", None)) 

497 self.assertEqual(datasetTypeComponentA.nameAndComponent(), ("composite", "compA")) 

498 

499 self.assertEqual(datasetTypeComponentA.parentStorageClass, storageClass) 

500 self.assertEqual(datasetTypeComponentB.parentStorageClass, storageClass) 

501 self.assertIsNone(datasetTypeComposite.parentStorageClass) 

502 

503 with self.assertRaises(KeyError): 

504 datasetTypeComposite.makeComponentDatasetType("compF") 

505 

506 

507class DatasetRefTestCase(unittest.TestCase): 

508 """Test for DatasetRef.""" 

509 

510 def setUp(self): 

511 self.universe = DimensionUniverse() 

512 datasetTypeName = "test" 

513 self.componentStorageClass1 = StorageClass("Component1") 

514 self.componentStorageClass2 = StorageClass("Component2") 

515 self.parentStorageClass = StorageClass( 

516 "Parent", components={"a": self.componentStorageClass1, "b": self.componentStorageClass2} 

517 ) 

518 dimensions = self.universe.extract(("instrument", "visit")) 

519 self.dataId = dict(instrument="DummyCam", visit=42) 

520 self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass) 

521 

522 def testConstructor(self): 

523 """Test that construction preserves and validates values.""" 

524 # Constructing a ref requires a run. 

525 with self.assertRaises(TypeError): 

526 DatasetRef(self.datasetType, self.dataId, id=uuid.uuid4()) 

527 

528 # Constructing an unresolved ref with run and/or components should 

529 # issue a ref with an id. 

530 run = "somerun" 

531 ref = DatasetRef(self.datasetType, self.dataId, run=run) 

532 self.assertEqual(ref.datasetType, self.datasetType) 

533 self.assertEqual( 

534 ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), msg=ref.dataId 

535 ) 

536 self.assertIsNotNone(ref.id) 

537 

538 # Passing a data ID that is missing dimensions should fail. 

539 with self.assertRaises(KeyError): 

540 DatasetRef(self.datasetType, {"instrument": "DummyCam"}, run="run") 

541 # Constructing a resolved ref should preserve run as well as everything 

542 # else. 

543 id_ = uuid.uuid4() 

544 ref = DatasetRef(self.datasetType, self.dataId, id=id_, run=run) 

545 self.assertEqual(ref.datasetType, self.datasetType) 

546 self.assertEqual( 

547 ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), msg=ref.dataId 

548 ) 

549 self.assertIsInstance(ref.dataId, DataCoordinate) 

550 self.assertEqual(ref.id, id_) 

551 self.assertEqual(ref.run, run) 

552 

553 with self.assertRaises(ValueError): 

554 DatasetRef(self.datasetType, self.dataId, run=run, id_generation_mode=42) 

555 

556 def testSorting(self): 

557 """Can we sort a DatasetRef""" 

558 # All refs have the same run. 

559 ref1 = DatasetRef(self.datasetType, dict(instrument="DummyCam", visit=1), run="run") 

560 ref2 = DatasetRef(self.datasetType, dict(instrument="DummyCam", visit=10), run="run") 

561 ref3 = DatasetRef(self.datasetType, dict(instrument="DummyCam", visit=22), run="run") 

562 

563 # Enable detailed diff report 

564 self.maxDiff = None 

565 

566 # This will sort them on visit number 

567 sort = sorted([ref3, ref1, ref2]) 

568 self.assertEqual(sort, [ref1, ref2, ref3], msg=f"Got order: {[r.dataId for r in sort]}") 

569 

570 # Now include different runs. 

571 ref1 = DatasetRef(self.datasetType, dict(instrument="DummyCam", visit=43), run="b") 

572 self.assertEqual(ref1.run, "b") 

573 ref4 = DatasetRef(self.datasetType, dict(instrument="DummyCam", visit=10), run="b") 

574 ref2 = DatasetRef(self.datasetType, dict(instrument="DummyCam", visit=4), run="a") 

575 ref3 = DatasetRef(self.datasetType, dict(instrument="DummyCam", visit=104), run="c") 

576 

577 # This will sort them on run before visit 

578 sort = sorted([ref3, ref1, ref2, ref4]) 

579 self.assertEqual(sort, [ref2, ref4, ref1, ref3], msg=f"Got order: {[r.dataId for r in sort]}") 

580 

581 # Now with strings 

582 with self.assertRaises(TypeError): 

583 sort = sorted(["z", ref1, "c"]) 

584 

585 def testOverrideStorageClass(self): 

586 storageA = StorageClass("test_a", pytype=list) 

587 

588 ref = DatasetRef(self.datasetType, self.dataId, run="somerun") 

589 

590 ref_new = ref.overrideStorageClass(storageA) 

591 self.assertNotEqual(ref, ref_new) 

592 self.assertEqual(ref_new.datasetType.storageClass, storageA) 

593 self.assertEqual(ref_new.overrideStorageClass(ref.datasetType.storageClass), ref) 

594 self.assertTrue(ref.is_compatible_with(ref_new)) 

595 with self.assertRaises(AttributeError): 

596 ref_new.is_compatible_with(None) 

597 

598 # Check different code paths of incompatibility. 

599 ref_incompat = DatasetRef(ref.datasetType, ref.dataId, run="somerun2", id=ref.id) 

600 self.assertFalse(ref.is_compatible_with(ref_incompat)) # bad run 

601 ref_incompat = DatasetRef(ref.datasetType, ref.dataId, run="somerun") 

602 self.assertFalse(ref.is_compatible_with(ref_incompat)) # bad ID 

603 

604 incompatible_sc = StorageClass("my_int", pytype=int) 

605 with self.assertRaises(ValueError): 

606 # Do not test against "ref" because it has a default storage class 

607 # of "object" which is compatible with everything. 

608 ref_new.overrideStorageClass(incompatible_sc) 

609 

610 def testPickle(self): 

611 ref = DatasetRef(self.datasetType, self.dataId, run="somerun") 

612 s = pickle.dumps(ref) 

613 self.assertEqual(pickle.loads(s), ref) 

614 

615 def testJson(self): 

616 ref = DatasetRef(self.datasetType, self.dataId, run="somerun") 

617 s = ref.to_json() 

618 self.assertEqual(DatasetRef.from_json(s, universe=self.universe), ref) 

619 

620 def testFileDataset(self): 

621 ref = DatasetRef(self.datasetType, self.dataId, run="somerun") 

622 file_dataset = FileDataset(path="something.yaml", refs=ref) 

623 self.assertEqual(file_dataset.refs, [ref]) 

624 

625 ref2 = DatasetRef(self.datasetType, self.dataId, run="somerun2") 

626 with self.assertRaises(ValueError): 

627 FileDataset(path="other.yaml", refs=[ref, ref2]) 

628 

629 

630if __name__ == "__main__": 

631 unittest.main()