Coverage for tests / test_datasets.py: 8%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import copy 

29import os 

30import pickle 

31import unittest 

32import uuid 

33 

34from lsst.daf.butler import ( 

35 DataCoordinate, 

36 DatasetProvenance, 

37 DatasetRef, 

38 DatasetType, 

39 DimensionConfig, 

40 DimensionUniverse, 

41 FileDataset, 

42 SerializedDatasetRefContainerV1, 

43 StorageClass, 

44 StorageClassFactory, 

45 UnknownComponentError, 

46) 

47from lsst.daf.butler.datastore.stored_file_info import StoredFileInfo 

48from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ZipIndex 

49from lsst.daf.butler.formatters.yaml import YamlFormatter 

50from lsst.resources import ResourcePath 

51 

52TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

53 

54"""Tests for datasets module. 

55""" 

56 

57 

58class DatasetTypeTestCase(unittest.TestCase): 

59 """Test for DatasetType.""" 

60 

61 def setUp(self) -> None: 

62 self.universe = DimensionUniverse() 

63 

64 def testConstructor(self) -> None: 

65 """Test construction preserves values. 

66 

67 Note that construction doesn't check for valid storageClass. 

68 This can only be verified for a particular schema. 

69 """ 

70 datasetTypeName = "test" 

71 storageClass = StorageClass("test_StructuredData") 

72 dimensions = self.universe.conform(("visit", "instrument")) 

73 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

74 self.assertEqual(datasetType.name, datasetTypeName) 

75 self.assertEqual(datasetType.storageClass, storageClass) 

76 self.assertEqual(datasetType.dimensions, dimensions) 

77 

78 with self.assertRaises(ValueError, msg="Construct component without parent storage class"): 

79 DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), dimensions, storageClass) 

80 with self.assertRaises(ValueError, msg="Construct non-component with parent storage class"): 

81 DatasetType(datasetTypeName, dimensions, storageClass, parentStorageClass="NotAllowed") 

82 

83 def testConstructor2(self) -> None: 

84 """Test construction from StorageClass name.""" 

85 datasetTypeName = "test" 

86 storageClass = StorageClass("test_constructor2") 

87 StorageClassFactory().registerStorageClass(storageClass) 

88 dimensions = self.universe.conform(("instrument", "visit")) 

89 datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2") 

90 self.assertEqual(datasetType.name, datasetTypeName) 

91 self.assertEqual(datasetType.storageClass, storageClass) 

92 self.assertEqual(datasetType.dimensions, dimensions) 

93 

94 def testNameValidation(self) -> None: 

95 """Test that dataset type names only contain certain characters 

96 in certain positions. 

97 """ 

98 dimensions = self.universe.conform(("instrument", "visit")) 

99 goodNames = ("a", "A", "z1", "Z1", "a_1B", "A_1b", "_a") 

100 badNames = ("1", "a%b", "B+Z", "T[0]") 

101 

102 # Construct storage class with all the good names included as 

103 # components so that we can test internal consistency 

104 storageClass = StorageClass( 

105 "test_StructuredData", components={n: StorageClass("component") for n in goodNames} 

106 ) 

107 

108 for name in goodNames: 

109 composite = DatasetType(name, dimensions, storageClass) 

110 self.assertEqual(composite.name, name) 

111 for suffix in goodNames: 

112 full = DatasetType.nameWithComponent(name, suffix) 

113 component = composite.makeComponentDatasetType(suffix) 

114 self.assertEqual(component.name, full) 

115 assert component.parentStorageClass is not None 

116 self.assertEqual(component.parentStorageClass.name, "test_StructuredData") 

117 for suffix in badNames: 

118 full = DatasetType.nameWithComponent(name, suffix) 

119 with self.subTest(full=full): 

120 with self.assertRaises(ValueError): 

121 DatasetType(full, dimensions, storageClass) 

122 for name in badNames: 

123 with self.subTest(name=name): 

124 with self.assertRaises(ValueError): 

125 DatasetType(name, dimensions, storageClass) 

126 

127 def testEquality(self) -> None: 

128 storageA = StorageClass("test_a") 

129 storageB = StorageClass("test_b") 

130 parent = StorageClass("test") 

131 dimensionsA = self.universe.conform(["instrument"]) 

132 dimensionsB = self.universe.conform(["skymap"]) 

133 self.assertEqual( 

134 DatasetType( 

135 "a", 

136 dimensionsA, 

137 storageA, 

138 ), 

139 DatasetType( 

140 "a", 

141 dimensionsA, 

142 storageA, 

143 ), 

144 ) 

145 self.assertEqual( 

146 DatasetType( 

147 "a", 

148 dimensionsA, 

149 "test_a", 

150 ), 

151 DatasetType( 

152 "a", 

153 dimensionsA, 

154 storageA, 

155 ), 

156 ) 

157 self.assertEqual( 

158 DatasetType( 

159 "a", 

160 dimensionsA, 

161 storageA, 

162 ), 

163 DatasetType( 

164 "a", 

165 dimensionsA, 

166 "test_a", 

167 ), 

168 ) 

169 self.assertEqual( 

170 DatasetType( 

171 "a", 

172 dimensionsA, 

173 "test_a", 

174 ), 

175 DatasetType( 

176 "a", 

177 dimensionsA, 

178 "test_a", 

179 ), 

180 ) 

181 self.assertEqual( 

182 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent), 

183 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent), 

184 ) 

185 self.assertEqual( 

186 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"), 

187 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"), 

188 ) 

189 self.assertNotEqual( 

190 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent", isCalibration=True), 

191 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent", isCalibration=False), 

192 ) 

193 self.assertNotEqual( 

194 DatasetType( 

195 "a", 

196 dimensionsA, 

197 storageA, 

198 ), 

199 DatasetType( 

200 "b", 

201 dimensionsA, 

202 storageA, 

203 ), 

204 ) 

205 self.assertNotEqual( 

206 DatasetType( 

207 "a", 

208 dimensionsA, 

209 storageA, 

210 ), 

211 DatasetType( 

212 "b", 

213 dimensionsA, 

214 "test_a", 

215 ), 

216 ) 

217 self.assertNotEqual( 

218 DatasetType( 

219 "a", 

220 dimensionsA, 

221 storageA, 

222 ), 

223 DatasetType( 

224 "a", 

225 dimensionsA, 

226 storageB, 

227 ), 

228 ) 

229 self.assertNotEqual( 

230 DatasetType( 

231 "a", 

232 dimensionsA, 

233 storageA, 

234 ), 

235 DatasetType( 

236 "a", 

237 dimensionsA, 

238 "test_b", 

239 ), 

240 ) 

241 self.assertNotEqual( 

242 DatasetType( 

243 "a", 

244 dimensionsA, 

245 storageA, 

246 ), 

247 DatasetType( 

248 "a", 

249 dimensionsB, 

250 storageA, 

251 ), 

252 ) 

253 self.assertNotEqual( 

254 DatasetType( 

255 "a", 

256 dimensionsA, 

257 storageA, 

258 ), 

259 DatasetType( 

260 "a", 

261 dimensionsB, 

262 "test_a", 

263 ), 

264 ) 

265 self.assertNotEqual( 

266 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageA), 

267 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageB), 

268 ) 

269 self.assertNotEqual( 

270 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageA"), 

271 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageB"), 

272 ) 

273 

274 def testCompatibility(self) -> None: 

275 storageA = StorageClass("test_a", pytype=set, converters={"list": "builtins.set"}) 

276 storageB = StorageClass("test_b", pytype=list) 

277 storageC = StorageClass("test_c", pytype=dict) 

278 self.assertTrue(storageA.can_convert(storageB)) 

279 dimensionsA = self.universe.conform(["instrument"]) 

280 

281 dA = DatasetType("a", dimensionsA, storageA) 

282 dA2 = DatasetType("a", dimensionsA, storageB) 

283 self.assertNotEqual(dA, dA2) 

284 self.assertTrue(dA.is_compatible_with(dA)) 

285 self.assertTrue(dA.is_compatible_with(dA2)) 

286 self.assertFalse(dA2.is_compatible_with(dA)) 

287 

288 dA3 = DatasetType("a", dimensionsA, storageC) 

289 self.assertFalse(dA.is_compatible_with(dA3)) 

290 

291 def testOverrideStorageClass(self) -> None: 

292 storageA = StorageClass("test_a", pytype=list, converters={"dict": "builtins.list"}) 

293 storageB = StorageClass("test_b", pytype=dict, converters={"list": "dict"}) 

294 dimensions = self.universe.conform(["instrument"]) 

295 

296 dA = DatasetType("a", dimensions, storageA) 

297 dB = dA.overrideStorageClass(storageB) 

298 self.assertNotEqual(dA, dB) 

299 self.assertEqual(dB.storageClass, storageB) 

300 

301 round_trip = dB.overrideStorageClass(storageA) 

302 self.assertEqual(round_trip, dA) 

303 

304 # Check that parents move over. Assign a pytype to avoid using 

305 # object in later tests. 

306 parent = StorageClass("composite", pytype=tuple, components={"a": storageA, "c": storageA}) 

307 dP = DatasetType("comp", dimensions, parent) 

308 dP_A = dP.makeComponentDatasetType("a") 

309 dp_B = dP_A.overrideStorageClass(storageB) 

310 self.assertEqual(dp_B.storageClass, storageB) 

311 self.assertEqual(dp_B.parentStorageClass, parent) 

312 

313 # Check that components are checked for compatibility but parents 

314 # can be different. 

315 parent2 = StorageClass( 

316 "composite2", 

317 pytype=frozenset, 

318 components={"a": storageB, "c": storageB}, 

319 ) 

320 dP2 = DatasetType("comp", dimensions, parent2) 

321 # Components are compatible even though parents aren't. 

322 self.assertFalse(dP.is_compatible_with(dP2)) 

323 self.assertTrue(dP2.makeComponentDatasetType("a").is_compatible_with(dP_A)) 

324 

325 def testJson(self) -> None: 

326 storageA = StorageClass("test_a") 

327 dimensionsA = self.universe.conform(["instrument"]) 

328 self.assertEqual( 

329 DatasetType( 

330 "a", 

331 dimensionsA, 

332 storageA, 

333 ), 

334 DatasetType.from_json( 

335 DatasetType( 

336 "a", 

337 dimensionsA, 

338 storageA, 

339 ).to_json(), 

340 self.universe, 

341 ), 

342 ) 

343 self.assertEqual( 

344 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"), 

345 DatasetType.from_json( 

346 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent").to_json(), 

347 self.universe, 

348 ), 

349 ) 

350 

351 def testSorting(self) -> None: 

352 """Can we sort a DatasetType""" 

353 storage = StorageClass("test_a") 

354 dimensions = self.universe.conform(["instrument"]) 

355 

356 d_a = DatasetType("a", dimensions, storage) 

357 d_f = DatasetType("f", dimensions, storage) 

358 d_p = DatasetType("p", dimensions, storage) 

359 

360 sort = sorted([d_p, d_f, d_a]) 

361 self.assertEqual(sort, [d_a, d_f, d_p]) 

362 

363 # Now with strings 

364 with self.assertRaises(TypeError): 

365 sort = sorted(["z", d_p, "c", d_f, d_a, "d"]) # type: ignore [list-item] 

366 

367 def testHashability(self) -> None: 

368 """Test `DatasetType.__hash__`. 

369 

370 This test is performed by checking that `DatasetType` entries can 

371 be inserted into a `set` and that unique values of its 

372 (`name`, `storageClass`, `dimensions`) parameters result in separate 

373 entries (and equal ones don't). 

374 

375 This does not check for uniformity of hashing or the actual values 

376 of the hash function. 

377 """ 

378 types: list[DatasetType] = [] 

379 unique = 0 

380 storageC = StorageClass("test_c") 

381 storageD = StorageClass("test_d") 

382 for name in ["a", "b"]: 

383 for storageClass in [storageC, storageD]: 

384 for dims in [("instrument",), ("skymap",)]: 

385 datasetType = DatasetType(name, self.universe.conform(dims), storageClass) 

386 datasetTypeCopy = DatasetType(name, self.universe.conform(dims), storageClass) 

387 types.extend((datasetType, datasetTypeCopy)) 

388 unique += 1 # datasetType should always equal its copy 

389 self.assertEqual(len(set(types)), unique) # all other combinations are unique 

390 

391 # also check that hashes of instances constructed with StorageClass 

392 # name matches hashes of instances constructed with instances 

393 dimensions = self.universe.conform(["instrument"]) 

394 self.assertEqual( 

395 hash(DatasetType("a", dimensions, storageC)), hash(DatasetType("a", dimensions, "test_c")) 

396 ) 

397 self.assertEqual( 

398 hash(DatasetType("a", dimensions, "test_c")), hash(DatasetType("a", dimensions, "test_c")) 

399 ) 

400 self.assertNotEqual( 

401 hash(DatasetType("a", dimensions, storageC)), hash(DatasetType("a", dimensions, "test_d")) 

402 ) 

403 self.assertNotEqual( 

404 hash(DatasetType("a", dimensions, storageD)), hash(DatasetType("a", dimensions, "test_c")) 

405 ) 

406 self.assertNotEqual( 

407 hash(DatasetType("a", dimensions, "test_c")), hash(DatasetType("a", dimensions, "test_d")) 

408 ) 

409 

410 def testDeepCopy(self) -> None: 

411 """Test that we can copy a dataset type.""" 

412 storageClass = StorageClass("test_copy") 

413 datasetTypeName = "test" 

414 dimensions = self.universe.conform(("instrument", "visit")) 

415 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

416 dcopy = copy.deepcopy(datasetType) 

417 self.assertEqual(dcopy, datasetType) 

418 

419 # Now with calibration flag set 

420 datasetType = DatasetType(datasetTypeName, dimensions, storageClass, isCalibration=True) 

421 dcopy = copy.deepcopy(datasetType) 

422 self.assertEqual(dcopy, datasetType) 

423 self.assertTrue(dcopy.isCalibration()) 

424 

425 # And again with a composite 

426 componentStorageClass = StorageClass("copy_component") 

427 componentDatasetType = DatasetType( 

428 DatasetType.nameWithComponent(datasetTypeName, "comp"), 

429 dimensions, 

430 componentStorageClass, 

431 parentStorageClass=storageClass, 

432 ) 

433 dcopy = copy.deepcopy(componentDatasetType) 

434 self.assertEqual(dcopy, componentDatasetType) 

435 

436 def testPickle(self) -> None: 

437 """Test pickle support.""" 

438 storageClass = StorageClass("test_pickle") 

439 datasetTypeName = "test" 

440 dimensions = self.universe.conform(("instrument", "visit")) 

441 # Un-pickling requires that storage class is registered with factory. 

442 StorageClassFactory().registerStorageClass(storageClass) 

443 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

444 datasetTypeOut = pickle.loads(pickle.dumps(datasetType)) 

445 self.assertIsInstance(datasetTypeOut, DatasetType) 

446 self.assertEqual(datasetType.name, datasetTypeOut.name) 

447 self.assertEqual(datasetType.dimensions, datasetTypeOut.dimensions) 

448 self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass) 

449 self.assertIsNone(datasetTypeOut.parentStorageClass) 

450 self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration()) 

451 self.assertFalse(datasetTypeOut.isCalibration()) 

452 

453 datasetType = DatasetType(datasetTypeName, dimensions, storageClass, isCalibration=True) 

454 datasetTypeOut = pickle.loads(pickle.dumps(datasetType)) 

455 self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration()) 

456 self.assertTrue(datasetTypeOut.isCalibration()) 

457 

458 # And again with a composite 

459 componentStorageClass = StorageClass("pickle_component") 

460 StorageClassFactory().registerStorageClass(componentStorageClass) 

461 componentDatasetType = DatasetType( 

462 DatasetType.nameWithComponent(datasetTypeName, "comp"), 

463 dimensions, 

464 componentStorageClass, 

465 parentStorageClass=storageClass, 

466 ) 

467 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) 

468 self.assertIsInstance(datasetTypeOut, DatasetType) 

469 self.assertEqual(componentDatasetType.name, datasetTypeOut.name) 

470 self.assertEqual(componentDatasetType.dimensions.names, datasetTypeOut.dimensions.names) 

471 self.assertEqual(componentDatasetType.storageClass, datasetTypeOut.storageClass) 

472 self.assertEqual(componentDatasetType.parentStorageClass, datasetTypeOut.parentStorageClass) 

473 self.assertEqual(datasetTypeOut.parentStorageClass.name, storageClass.name) 

474 self.assertEqual(datasetTypeOut, componentDatasetType) 

475 

476 # Now with a string and not a real storage class to test that 

477 # pickling doesn't force the StorageClass to be resolved 

478 componentDatasetType = DatasetType( 

479 DatasetType.nameWithComponent(datasetTypeName, "comp"), 

480 dimensions, 

481 "StrangeComponent", 

482 parentStorageClass="UnknownParent", 

483 ) 

484 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) 

485 self.assertEqual(datasetTypeOut, componentDatasetType) 

486 self.assertEqual(datasetTypeOut._parentStorageClassName, componentDatasetType._parentStorageClassName) 

487 

488 def test_composites(self) -> None: 

489 """Test components within composite DatasetTypes.""" 

490 storageClassA = StorageClass("compA") 

491 storageClassB = StorageClass("compB") 

492 storageClass = StorageClass( 

493 "test_composite", components={"compA": storageClassA, "compB": storageClassB} 

494 ) 

495 self.assertTrue(storageClass.isComposite()) 

496 self.assertFalse(storageClassA.isComposite()) 

497 self.assertFalse(storageClassB.isComposite()) 

498 

499 dimensions = self.universe.conform(("instrument", "visit")) 

500 

501 datasetTypeComposite = DatasetType("composite", dimensions, storageClass) 

502 datasetTypeComponentA = datasetTypeComposite.makeComponentDatasetType("compA") 

503 datasetTypeComponentB = datasetTypeComposite.makeComponentDatasetType("compB") 

504 

505 self.assertTrue(datasetTypeComposite.isComposite()) 

506 self.assertFalse(datasetTypeComponentA.isComposite()) 

507 self.assertTrue(datasetTypeComponentB.isComponent()) 

508 self.assertFalse(datasetTypeComposite.isComponent()) 

509 

510 self.assertEqual(datasetTypeComposite.name, "composite") 

511 self.assertEqual(datasetTypeComponentA.name, "composite.compA") 

512 self.assertEqual(datasetTypeComponentB.component(), "compB") 

513 self.assertEqual(datasetTypeComposite.nameAndComponent(), ("composite", None)) 

514 self.assertEqual(datasetTypeComponentA.nameAndComponent(), ("composite", "compA")) 

515 

516 self.assertEqual(datasetTypeComponentA.parentStorageClass, storageClass) 

517 self.assertEqual(datasetTypeComponentB.parentStorageClass, storageClass) 

518 self.assertIsNone(datasetTypeComposite.parentStorageClass) 

519 

520 with self.assertRaises(UnknownComponentError): 

521 datasetTypeComposite.makeComponentDatasetType("compF") 

522 

523 with self.assertRaises(UnknownComponentError): 

524 datasetTypeComposite.componentTypeName("unknown") 

525 

526 

527class DatasetRefTestCase(unittest.TestCase): 

528 """Test for DatasetRef.""" 

529 

530 def setUp(self) -> None: 

531 self.universe = DimensionUniverse() 

532 datasetTypeName = "test" 

533 self.componentStorageClass1 = StorageClass("Component1") 

534 self.componentStorageClass2 = StorageClass("Component2") 

535 self.parentStorageClass = StorageClass( 

536 "Parent", components={"a": self.componentStorageClass1, "b": self.componentStorageClass2} 

537 ) 

538 sc_factory = StorageClassFactory() 

539 sc_factory.registerStorageClass(self.componentStorageClass1) 

540 sc_factory.registerStorageClass(self.componentStorageClass2) 

541 sc_factory.registerStorageClass(self.parentStorageClass) 

542 dimensions = self.universe.conform(("instrument", "visit")) 

543 self.dataId = DataCoordinate.standardize( 

544 dict(instrument="DummyCam", visit=42), universe=self.universe 

545 ) 

546 self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass) 

547 

548 def _make_datastore_records(self, ref: DatasetRef, *paths: str) -> DatasetRef: 

549 """Return an updated dataset ref with datastore records.""" 

550 opaque_table_name = "datastore_records" 

551 datastore_records = { 

552 opaque_table_name: [ 

553 StoredFileInfo( 

554 formatter="", 

555 path=path, 

556 storageClass=ref.datasetType.storageClass, 

557 component=None, 

558 checksum=None, 

559 file_size=1, 

560 ) 

561 for path in paths 

562 ] 

563 } 

564 return ref.replace(datastore_records=datastore_records) 

565 

566 def testConstructor(self) -> None: 

567 """Test that construction preserves and validates values.""" 

568 # Constructing a ref requires a run. 

569 with self.assertRaises(TypeError): 

570 DatasetRef(self.datasetType, self.dataId, id=uuid.uuid4()) # type: ignore [call-arg] 

571 

572 # Constructing an unresolved ref with run and/or components should 

573 # issue a ref with an id. 

574 run = "somerun" 

575 ref = DatasetRef(self.datasetType, self.dataId, run=run) 

576 self.assertEqual(ref.datasetType, self.datasetType) 

577 self.assertEqual( 

578 ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), msg=ref.dataId 

579 ) 

580 self.assertIsNotNone(ref.id) 

581 

582 # Passing a data ID that is missing dimensions should fail. 

583 # Create a full DataCoordinate to ensure that we are testing the 

584 # right thing. 

585 dimensions = self.universe.conform(("instrument",)) 

586 dataId = DataCoordinate.standardize(instrument="DummyCam", dimensions=dimensions) 

587 with self.assertRaises(KeyError): 

588 DatasetRef(self.datasetType, dataId, run="run") 

589 # Constructing a resolved ref should preserve run as well as everything 

590 # else. 

591 id_ = uuid.uuid4() 

592 ref = DatasetRef(self.datasetType, self.dataId, id=id_, run=run) 

593 self.assertEqual(ref.datasetType, self.datasetType) 

594 self.assertEqual( 

595 ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), msg=ref.dataId 

596 ) 

597 self.assertIsInstance(ref.dataId, DataCoordinate) 

598 self.assertEqual(ref.id, id_) 

599 self.assertEqual(ref.run, run) 

600 

601 with self.assertRaises(ValueError): 

602 DatasetRef(self.datasetType, self.dataId, run=run, id_generation_mode=42) # type: ignore 

603 

604 def testSorting(self) -> None: 

605 """Can we sort a DatasetRef""" 

606 # All refs have the same run. 

607 dimensions = self.universe.conform(("instrument", "visit")) 

608 ref1 = DatasetRef( 

609 self.datasetType, 

610 DataCoordinate.standardize(instrument="DummyCam", visit=1, dimensions=dimensions), 

611 run="run", 

612 ) 

613 ref2 = DatasetRef( 

614 self.datasetType, 

615 DataCoordinate.standardize(instrument="DummyCam", visit=10, dimensions=dimensions), 

616 run="run", 

617 ) 

618 ref3 = DatasetRef( 

619 self.datasetType, 

620 DataCoordinate.standardize(instrument="DummyCam", visit=22, dimensions=dimensions), 

621 run="run", 

622 ) 

623 

624 # Enable detailed diff report 

625 self.maxDiff = None 

626 

627 # This will sort them on visit number 

628 sort = sorted([ref3, ref1, ref2]) 

629 self.assertEqual(sort, [ref1, ref2, ref3], msg=f"Got order: {[r.dataId for r in sort]}") 

630 

631 # Now include different runs. 

632 ref1 = DatasetRef( 

633 self.datasetType, 

634 DataCoordinate.standardize(instrument="DummyCam", visit=43, dimensions=dimensions), 

635 run="b", 

636 ) 

637 self.assertEqual(ref1.run, "b") 

638 ref4 = DatasetRef( 

639 self.datasetType, 

640 DataCoordinate.standardize(instrument="DummyCam", visit=10, dimensions=dimensions), 

641 run="b", 

642 ) 

643 ref2 = DatasetRef( 

644 self.datasetType, 

645 DataCoordinate.standardize(instrument="DummyCam", visit=4, dimensions=dimensions), 

646 run="a", 

647 ) 

648 ref3 = DatasetRef( 

649 self.datasetType, 

650 DataCoordinate.standardize(instrument="DummyCam", visit=104, dimensions=dimensions), 

651 run="c", 

652 ) 

653 

654 # This will sort them on run before visit 

655 sort = sorted([ref3, ref1, ref2, ref4]) 

656 self.assertEqual(sort, [ref2, ref4, ref1, ref3], msg=f"Got order: {[r.dataId for r in sort]}") 

657 

658 # Now with strings 

659 with self.assertRaises(TypeError): 

660 sort = sorted(["z", ref1, "c"]) # type: ignore [list-item] 

661 

662 def testOverrideStorageClass(self) -> None: 

663 storageA = StorageClass("test_a", pytype=list) 

664 

665 ref = DatasetRef(self.datasetType, self.dataId, run="somerun") 

666 

667 ref_new = ref.overrideStorageClass(storageA) 

668 self.assertNotEqual(ref, ref_new) 

669 self.assertEqual(ref_new.datasetType.storageClass, storageA) 

670 self.assertEqual(ref_new.overrideStorageClass(ref.datasetType.storageClass), ref) 

671 self.assertTrue(ref.is_compatible_with(ref_new)) 

672 with self.assertRaises(AttributeError): 

673 ref_new.is_compatible_with(None) # type: ignore 

674 

675 # Check different code paths of incompatibility. 

676 ref_incompat = DatasetRef(ref.datasetType, ref.dataId, run="somerun2", id=ref.id) 

677 self.assertFalse(ref.is_compatible_with(ref_incompat)) # bad run 

678 ref_incompat = DatasetRef(ref.datasetType, ref.dataId, run="somerun") 

679 self.assertFalse(ref.is_compatible_with(ref_incompat)) # bad ID 

680 

681 incompatible_sc = StorageClass("my_int", pytype=int) 

682 with self.assertRaises(ValueError): 

683 # Do not test against "ref" because it has a default storage class 

684 # of "object" which is compatible with everything. 

685 ref_new.overrideStorageClass(incompatible_sc) 

686 

687 def testReplace(self) -> None: 

688 """Test for `DatasetRef.replace` method.""" 

689 ref = DatasetRef(self.datasetType, self.dataId, run="somerun") 

690 

691 ref2 = ref.replace(run="somerun2") 

692 self.assertEqual(ref2.run, "somerun2") 

693 self.assertIsNotNone(ref2.id) 

694 self.assertNotEqual(ref2.id, ref.id) 

695 

696 ref3 = ref.replace(run="somerun3", id=ref2.id) 

697 self.assertEqual(ref3.run, "somerun3") 

698 self.assertEqual(ref3.id, ref2.id) 

699 

700 ref4 = ref.replace(id=ref2.id) 

701 self.assertEqual(ref4.run, "somerun") 

702 self.assertEqual(ref4.id, ref2.id) 

703 

704 ref5 = ref.replace() 

705 self.assertEqual(ref5.run, "somerun") 

706 self.assertEqual(ref5, ref) 

707 

708 self.assertIsNone(ref5._datastore_records) 

709 ref5 = ref5.replace(datastore_records={}) 

710 self.assertEqual(ref5._datastore_records, {}) 

711 ref5 = ref5.replace(datastore_records=None) 

712 self.assertIsNone(ref5._datastore_records) 

713 

714 def testPickle(self) -> None: 

715 ref = DatasetRef(self.datasetType, self.dataId, run="somerun") 

716 s = pickle.dumps(ref) 

717 self.assertEqual(pickle.loads(s), ref) 

718 

719 def testJson(self) -> None: 

720 ref = DatasetRef(self.datasetType, self.dataId, run="somerun") 

721 s = ref.to_json() 

722 self.assertEqual(DatasetRef.from_json(s, universe=self.universe), ref) 

723 

724 # Also test ref with datastore records, serialization does not 

725 # preserve those. 

726 ref = self._make_datastore_records(ref, "/path1", "/path2") 

727 s = ref.to_json() 

728 ref2 = DatasetRef.from_json(s, universe=self.universe) 

729 self.assertEqual(ref2, ref) 

730 self.assertIsNone(ref2._datastore_records) 

731 

732 def testFileDataset(self) -> None: 

733 ref = DatasetRef(self.datasetType, self.dataId, run="somerun") 

734 file_dataset = FileDataset(path="something.yaml", refs=ref) 

735 self.assertEqual(file_dataset.refs, [ref]) 

736 

737 ref2 = DatasetRef(self.datasetType, self.dataId, run="somerun2") 

738 with self.assertRaises(ValueError): 

739 FileDataset(path="other.yaml", refs=[ref, ref2]) 

740 

741 serialized = file_dataset.to_simple() 

742 self.assertEqual(serialized.path, "something.yaml") 

743 self.assertEqual(len(serialized.refs), 1)