Coverage for tests/test_simpleButler.py: 15%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

268 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import tempfile 

26from typing import Any 

27import unittest 

28import uuid 

29import re 

30 

31try: 

32 import numpy as np 

33except ImportError: 

34 np = None 

35 

36import astropy.time 

37 

38from lsst.daf.butler import ( 

39 Butler, 

40 ButlerConfig, 

41 CollectionType, 

42 DatasetRef, 

43 DatasetType, 

44 Registry, 

45 Timespan, 

46) 

47from lsst.daf.butler.registry import RegistryConfig, RegistryDefaults, ConflictingDefinitionError 

48from lsst.daf.butler.tests import DatastoreMock 

49from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

50 

51 

52TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

53 

54 

55class SimpleButlerTestCase(unittest.TestCase): 

56 """Tests for butler (including import/export functionality) that should not 

57 depend on the Registry Database backend or Datastore implementation, and 

58 can instead utilize an in-memory SQLite Registry and a mocked Datastore. 

59 """ 

60 

61 datasetsManager = \ 

62 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

63 datasetsImportFile = "datasets.yaml" 

64 datasetsIdType = int 

65 

66 def setUp(self): 

67 self.root = makeTestTempDir(TESTDIR) 

68 

69 def tearDown(self): 

70 removeTestTempDir(self.root) 

71 

72 def makeButler(self, **kwargs: Any) -> Butler: 

73 """Return new Butler instance on each call. 

74 """ 

75 config = ButlerConfig() 

76 

77 # make separate temporary directory for registry of this instance 

78 tmpdir = tempfile.mkdtemp(dir=self.root) 

79 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3" 

80 config["registry", "managers", "datasets"] = self.datasetsManager 

81 config["root"] = self.root 

82 

83 # have to make a registry first 

84 registryConfig = RegistryConfig(config.get("registry")) 

85 Registry.createFromConfig(registryConfig) 

86 

87 butler = Butler(config, **kwargs) 

88 DatastoreMock.apply(butler) 

89 return butler 

90 

91 def comparableRef(self, ref: DatasetRef) -> DatasetRef: 

92 """Return a DatasetRef that can be compared to a DatasetRef from 

93 other repository. 

94 

95 For repositories that do not support round-trip of ID values this 

96 method returns unresolved DatasetRef, for round-trip-safe repos it 

97 returns unchanged ref. 

98 """ 

99 return ref if self.datasetsIdType is uuid.UUID else ref.unresolved() 

100 

101 def testReadBackwardsCompatibility(self): 

102 """Test that we can read an export file written by a previous version 

103 and commit to the daf_butler git repo. 

104 

105 Notes 

106 ----- 

107 At present this export file includes only dimension data, not datasets, 

108 which greatly limits the usefulness of this test. We should address 

109 this at some point, but I think it's best to wait for the changes to 

110 the export format required for CALIBRATION collections to land. 

111 """ 

112 butler = self.makeButler(writeable=True) 

113 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "hsc-rc2-subset.yaml")) 

114 # Spot-check a few things, but the most important test is just that 

115 # the above does not raise. 

116 self.assertGreaterEqual( 

117 set(record.id for record in butler.registry.queryDimensionRecords("detector", instrument="HSC")), 

118 set(range(104)), # should have all science CCDs; may have some focus ones. 

119 ) 

120 self.assertGreaterEqual( 

121 { 

122 (record.id, record.physical_filter) 

123 for record in butler.registry.queryDimensionRecords("visit", instrument="HSC") 

124 }, 

125 { 

126 (27136, 'HSC-Z'), 

127 (11694, 'HSC-G'), 

128 (23910, 'HSC-R'), 

129 (11720, 'HSC-Y'), 

130 (23900, 'HSC-R'), 

131 (22646, 'HSC-Y'), 

132 (1248, 'HSC-I'), 

133 (19680, 'HSC-I'), 

134 (1240, 'HSC-I'), 

135 (424, 'HSC-Y'), 

136 (19658, 'HSC-I'), 

137 (344, 'HSC-Y'), 

138 (1218, 'HSC-R'), 

139 (1190, 'HSC-Z'), 

140 (23718, 'HSC-R'), 

141 (11700, 'HSC-G'), 

142 (26036, 'HSC-G'), 

143 (23872, 'HSC-R'), 

144 (1170, 'HSC-Z'), 

145 (1876, 'HSC-Y'), 

146 } 

147 ) 

148 

149 def testDatasetTransfers(self): 

150 """Test exporting all datasets from a repo and then importing them all 

151 back in again. 

152 """ 

153 # Import data to play with. 

154 butler1 = self.makeButler(writeable=True) 

155 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

156 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

157 with tempfile.NamedTemporaryFile(mode='w', suffix=".yaml") as file: 

158 # Export all datasets. 

159 with butler1.export(filename=file.name) as exporter: 

160 exporter.saveDatasets( 

161 butler1.registry.queryDatasets(..., collections=...) 

162 ) 

163 # Import it all again. 

164 butler2 = self.makeButler(writeable=True) 

165 butler2.import_(filename=file.name) 

166 datasets1 = list(butler1.registry.queryDatasets(..., collections=...)) 

167 datasets2 = list(butler2.registry.queryDatasets(..., collections=...)) 

168 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets1)) 

169 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets2)) 

170 self.assertCountEqual( 

171 [self.comparableRef(ref) for ref in datasets1], 

172 [self.comparableRef(ref) for ref in datasets2], 

173 ) 

174 

175 def testComponentExport(self): 

176 """Test exporting component datasets and then importing them. 

177 

178 This test intentionally does not depend on whether just the component 

179 is exported and then imported vs. the full composite dataset, because 

180 I don't want it to assume more than it needs to about the 

181 implementation. 

182 """ 

183 # Import data to play with. 

184 butler1 = self.makeButler(writeable=True) 

185 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

186 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

187 with tempfile.NamedTemporaryFile(mode='w', suffix=".yaml") as file: 

188 # Export all datasets. 

189 with butler1.export(filename=file.name) as exporter: 

190 exporter.saveDatasets( 

191 butler1.registry.queryDatasets("flat.psf", collections=...) 

192 ) 

193 # Import it all again. 

194 butler2 = self.makeButler(writeable=True) 

195 butler2.import_(filename=file.name) 

196 datasets1 = list(butler1.registry.queryDatasets("flat.psf", collections=...)) 

197 datasets2 = list(butler2.registry.queryDatasets("flat.psf", collections=...)) 

198 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets1)) 

199 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets2)) 

200 self.assertCountEqual( 

201 [self.comparableRef(ref) for ref in datasets1], 

202 [self.comparableRef(ref) for ref in datasets2], 

203 ) 

204 

205 def testDatasetImportTwice(self): 

206 """Test exporting all datasets from a repo and then importing them all 

207 back in again twice. 

208 """ 

209 if self.datasetsIdType is not uuid.UUID: 

210 self.skipTest("This test can only work for UUIDs") 

211 # Import data to play with. 

212 butler1 = self.makeButler(writeable=True) 

213 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

214 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

215 with tempfile.NamedTemporaryFile(mode='w', suffix=".yaml", delete=False) as file: 

216 # Export all datasets. 

217 with butler1.export(filename=file.name) as exporter: 

218 exporter.saveDatasets( 

219 butler1.registry.queryDatasets(..., collections=...) 

220 ) 

221 butler2 = self.makeButler(writeable=True) 

222 # Import it once. 

223 butler2.import_(filename=file.name) 

224 # Import it again, but ignore all dimensions 

225 dimensions = set( 

226 dimension.name for dimension in butler2.registry.dimensions.getStaticDimensions()) 

227 butler2.import_(filename=file.name, skip_dimensions=dimensions) 

228 datasets1 = list(butler1.registry.queryDatasets(..., collections=...)) 

229 datasets2 = list(butler2.registry.queryDatasets(..., collections=...)) 

230 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets1)) 

231 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets2)) 

232 self.assertCountEqual( 

233 [self.comparableRef(ref) for ref in datasets1], 

234 [self.comparableRef(ref) for ref in datasets2], 

235 ) 

236 

237 def testDatasetImportReuseIds(self): 

238 """Test for import that should preserve dataset IDs. 

239 

240 This test assumes that dataset IDs in datasets YAML are different from 

241 what auto-incremental insert would produce. 

242 """ 

243 if self.datasetsIdType is not int: 

244 self.skipTest("This test can only work for UUIDs") 

245 # Import data to play with. 

246 butler = self.makeButler(writeable=True) 

247 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

248 filename = os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile) 

249 butler.import_(filename=filename, reuseIds=True) 

250 datasets = list(butler.registry.queryDatasets(..., collections=...)) 

251 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets)) 

252 # IDs are copied from YAML, list needs to be updated if file contents 

253 # is changed. 

254 self.assertCountEqual( 

255 [ref.id for ref in datasets], 

256 [1001, 1002, 1003, 1010, 1020, 1030, 2001, 2002, 2003, 2010, 2020, 2030, 2040], 

257 ) 

258 

259 # Try once again, it will raise 

260 with self.assertRaises(ConflictingDefinitionError): 

261 butler.import_(filename=filename, reuseIds=True) 

262 

263 def testCollectionTransfers(self): 

264 """Test exporting and then importing collections of various types. 

265 """ 

266 # Populate a registry with some datasets. 

267 butler1 = self.makeButler(writeable=True) 

268 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

269 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

270 registry1 = butler1.registry 

271 # Add some more collections. 

272 registry1.registerRun("run1") 

273 registry1.registerCollection("tag1", CollectionType.TAGGED) 

274 registry1.registerCollection("calibration1", CollectionType.CALIBRATION) 

275 registry1.registerCollection("chain1", CollectionType.CHAINED) 

276 registry1.registerCollection("chain2", CollectionType.CHAINED) 

277 registry1.setCollectionChain("chain1", ["tag1", "run1", "chain2"]) 

278 registry1.setCollectionChain("chain2", ["calibration1", "run1"]) 

279 # Associate some datasets into the TAGGED and CALIBRATION collections. 

280 flats1 = list(registry1.queryDatasets("flat", collections=...)) 

281 registry1.associate("tag1", flats1) 

282 t1 = astropy.time.Time('2020-01-01T01:00:00', format="isot", scale="tai") 

283 t2 = astropy.time.Time('2020-01-01T02:00:00', format="isot", scale="tai") 

284 t3 = astropy.time.Time('2020-01-01T03:00:00', format="isot", scale="tai") 

285 bias1a = registry1.findDataset("bias", instrument="Cam1", detector=1, collections="imported_g") 

286 bias2a = registry1.findDataset("bias", instrument="Cam1", detector=2, collections="imported_g") 

287 bias3a = registry1.findDataset("bias", instrument="Cam1", detector=3, collections="imported_g") 

288 bias2b = registry1.findDataset("bias", instrument="Cam1", detector=2, collections="imported_r") 

289 bias3b = registry1.findDataset("bias", instrument="Cam1", detector=3, collections="imported_r") 

290 registry1.certify("calibration1", [bias2a, bias3a], Timespan(t1, t2)) 

291 registry1.certify("calibration1", [bias2b], Timespan(t2, None)) 

292 registry1.certify("calibration1", [bias3b], Timespan(t2, t3)) 

293 registry1.certify("calibration1", [bias1a], Timespan.makeEmpty()) 

294 

295 with tempfile.NamedTemporaryFile(mode='w', suffix=".yaml") as file: 

296 # Export all collections, and some datasets. 

297 with butler1.export(filename=file.name) as exporter: 

298 # Sort results to put chain1 before chain2, which is 

299 # intentionally not topological order. 

300 for collection in sorted(registry1.queryCollections()): 

301 exporter.saveCollection(collection) 

302 exporter.saveDatasets(flats1) 

303 exporter.saveDatasets([bias1a, bias2a, bias2b, bias3a, bias3b]) 

304 # Import them into a new registry. 

305 butler2 = self.makeButler(writeable=True) 

306 butler2.import_(filename=file.name) 

307 registry2 = butler2.registry 

308 # Check that it all round-tripped, starting with the collections 

309 # themselves. 

310 self.assertIs(registry2.getCollectionType("run1"), CollectionType.RUN) 

311 self.assertIs(registry2.getCollectionType("tag1"), CollectionType.TAGGED) 

312 self.assertIs(registry2.getCollectionType("calibration1"), CollectionType.CALIBRATION) 

313 self.assertIs(registry2.getCollectionType("chain1"), CollectionType.CHAINED) 

314 self.assertIs(registry2.getCollectionType("chain2"), CollectionType.CHAINED) 

315 self.assertEqual( 

316 list(registry2.getCollectionChain("chain1")), 

317 ["tag1", "run1", "chain2"], 

318 ) 

319 self.assertEqual( 

320 list(registry2.getCollectionChain("chain2")), 

321 ["calibration1", "run1"], 

322 ) 

323 # Check that tag collection contents are the same. 

324 self.maxDiff = None 

325 self.assertCountEqual( 

326 [self.comparableRef(ref) for ref in registry1.queryDatasets(..., collections="tag1")], 

327 [self.comparableRef(ref) for ref in registry2.queryDatasets(..., collections="tag1")], 

328 ) 

329 # Check that calibration collection contents are the same. 

330 self.assertCountEqual( 

331 [(self.comparableRef(assoc.ref), assoc.timespan) 

332 for assoc in registry1.queryDatasetAssociations("bias", collections="calibration1")], 

333 [(self.comparableRef(assoc.ref), assoc.timespan) 

334 for assoc in registry2.queryDatasetAssociations("bias", collections="calibration1")], 

335 ) 

336 

337 def testButlerGet(self): 

338 """Test that butler.get can work with different variants.""" 

339 

340 # Import data to play with. 

341 butler = self.makeButler(writeable=True) 

342 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

343 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

344 

345 # Find the DatasetRef for a flat 

346 coll = "imported_g" 

347 flat2g = butler.registry.findDataset("flat", instrument="Cam1", detector=2, physical_filter="Cam1-G", 

348 collections=coll) 

349 

350 # Create a numpy integer to check that works fine 

351 detector_np = np.int64(2) if np else 2 

352 print(type(detector_np)) 

353 

354 # Try to get it using different variations of dataId + keyword 

355 # arguments 

356 # Note that instrument.class_name does not work 

357 variants = ( 

358 (None, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}), 

359 (None, {"instrument": "Cam1", "detector": detector_np, "physical_filter": "Cam1-G"}), 

360 ({"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}, {}), 

361 ({"instrument": "Cam1", "detector": detector_np, "physical_filter": "Cam1-G"}, {}), 

362 ({"instrument": "Cam1", "detector": 2}, {"physical_filter": "Cam1-G"}), 

363 ({"detector.full_name": "Ab"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}), 

364 ({"full_name": "Ab"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}), 

365 (None, {"full_name": "Ab", "instrument": "Cam1", "physical_filter": "Cam1-G"}), 

366 (None, {"detector": "Ab", "instrument": "Cam1", "physical_filter": "Cam1-G"}), 

367 ({"name_in_raft": "b", "raft": "A"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}), 

368 ({"name_in_raft": "b"}, {"raft": "A", "instrument": "Cam1", "physical_filter": "Cam1-G"}), 

369 (None, {"name_in_raft": "b", "raft": "A", "instrument": "Cam1", "physical_filter": "Cam1-G"}), 

370 ({"detector.name_in_raft": "b", "detector.raft": "A"}, 

371 {"instrument": "Cam1", "physical_filter": "Cam1-G"}), 

372 ({"detector.name_in_raft": "b", "detector.raft": "A", 

373 "instrument": "Cam1", "physical_filter": "Cam1-G"}, {}), 

374 ) 

375 

376 for dataId, kwds in variants: 

377 try: 

378 flat_id, _ = butler.get("flat", dataId=dataId, collections=coll, **kwds) 

379 except Exception as e: 

380 raise type(e)(f"{str(e)}: dataId={dataId}, kwds={kwds}") from e 

381 self.assertEqual(flat_id, flat2g.id, msg=f"DataId: {dataId}, kwds: {kwds}") 

382 

383 def testGetCalibration(self): 

384 """Test that `Butler.get` can be used to fetch from 

385 `~CollectionType.CALIBRATION` collections if the data ID includes 

386 extra dimensions with temporal information. 

387 """ 

388 # Import data to play with. 

389 butler = self.makeButler(writeable=True) 

390 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

391 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

392 # Certify some biases into a CALIBRATION collection. 

393 registry = butler.registry 

394 registry.registerCollection("calibs", CollectionType.CALIBRATION) 

395 t1 = astropy.time.Time('2020-01-01T01:00:00', format="isot", scale="tai") 

396 t2 = astropy.time.Time('2020-01-01T02:00:00', format="isot", scale="tai") 

397 t3 = astropy.time.Time('2020-01-01T03:00:00', format="isot", scale="tai") 

398 bias2a = registry.findDataset("bias", instrument="Cam1", detector=2, collections="imported_g") 

399 bias3a = registry.findDataset("bias", instrument="Cam1", detector=3, collections="imported_g") 

400 bias2b = registry.findDataset("bias", instrument="Cam1", detector=2, collections="imported_r") 

401 bias3b = registry.findDataset("bias", instrument="Cam1", detector=3, collections="imported_r") 

402 registry.certify("calibs", [bias2a, bias3a], Timespan(t1, t2)) 

403 registry.certify("calibs", [bias2b], Timespan(t2, None)) 

404 registry.certify("calibs", [bias3b], Timespan(t2, t3)) 

405 # Insert some exposure dimension data. 

406 registry.insertDimensionData( 

407 "exposure", 

408 { 

409 "instrument": "Cam1", 

410 "id": 3, 

411 "obs_id": "three", 

412 "timespan": Timespan(t1, t2), 

413 "physical_filter": "Cam1-G", 

414 "day_obs": 20201114, 

415 "seq_num": 55, 

416 }, 

417 { 

418 "instrument": "Cam1", 

419 "id": 4, 

420 "obs_id": "four", 

421 "timespan": Timespan(t2, t3), 

422 "physical_filter": "Cam1-G", 

423 "day_obs": 20211114, 

424 "seq_num": 42, 

425 }, 

426 ) 

427 # Get some biases from raw-like data IDs. 

428 bias2a_id, _ = butler.get("bias", {"instrument": "Cam1", "exposure": 3, "detector": 2}, 

429 collections="calibs") 

430 self.assertEqual(bias2a_id, bias2a.id) 

431 bias3b_id, _ = butler.get("bias", {"instrument": "Cam1", "exposure": 4, "detector": 3}, 

432 collections="calibs") 

433 self.assertEqual(bias3b_id, bias3b.id) 

434 

435 # Get using the kwarg form 

436 bias3b_id, _ = butler.get("bias", 

437 instrument="Cam1", exposure=4, detector=3, 

438 collections="calibs") 

439 self.assertEqual(bias3b_id, bias3b.id) 

440 

441 # Do it again but using the record information 

442 bias2a_id, _ = butler.get("bias", {"instrument": "Cam1", "exposure.obs_id": "three", 

443 "detector.full_name": "Ab"}, 

444 collections="calibs") 

445 self.assertEqual(bias2a_id, bias2a.id) 

446 bias3b_id, _ = butler.get("bias", {"exposure.obs_id": "four", 

447 "detector.full_name": "Ba"}, 

448 collections="calibs", instrument="Cam1") 

449 self.assertEqual(bias3b_id, bias3b.id) 

450 

451 # And again but this time using the alternate value rather than 

452 # the primary. 

453 bias3b_id, _ = butler.get("bias", {"exposure": "four", 

454 "detector": "Ba"}, 

455 collections="calibs", instrument="Cam1") 

456 self.assertEqual(bias3b_id, bias3b.id) 

457 

458 # And again but this time using the alternate value rather than 

459 # the primary and do it in the keyword arguments. 

460 bias3b_id, _ = butler.get("bias", 

461 exposure="four", detector="Ba", 

462 collections="calibs", instrument="Cam1") 

463 self.assertEqual(bias3b_id, bias3b.id) 

464 

465 # Now with implied record columns 

466 bias3b_id, _ = butler.get("bias", day_obs=20211114, seq_num=42, 

467 raft="B", name_in_raft="a", 

468 collections="calibs", instrument="Cam1") 

469 self.assertEqual(bias3b_id, bias3b.id) 

470 

471 def testRegistryDefaults(self): 

472 """Test that we can default the collections and some data ID keys when 

473 constructing a butler. 

474 

475 Many tests that use default run already exist in ``test_butler.py``, so 

476 that isn't tested here. And while most of this functionality is 

477 implemented in `Registry`, we test it here instead of 

478 ``daf/butler/tests/registry.py`` because it shouldn't depend on the 

479 database backend at all. 

480 """ 

481 butler = self.makeButler(writeable=True) 

482 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

483 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

484 # Need to actually set defaults later, not at construction, because 

485 # we need to import the instrument before we can use it as a default. 

486 # Don't set a default instrument value for data IDs, because 'Cam1' 

487 # should be inferred by virtue of that being the only value in the 

488 # input collections. 

489 butler.registry.defaults = RegistryDefaults(collections=["imported_g"]) 

490 # Use findDataset without collections or instrument. 

491 ref = butler.registry.findDataset("flat", detector=2, physical_filter="Cam1-G") 

492 # Do the same with Butler.get; this should ultimately invoke a lot of 

493 # the same code, so it's a bit circular, but mostly we're checking that 

494 # it works at all. 

495 dataset_id, _ = butler.get("flat", detector=2, physical_filter="Cam1-G") 

496 self.assertEqual(ref.id, dataset_id) 

497 # Query for datasets. Test defaulting the data ID in both kwargs and 

498 # in the WHERE expression. 

499 queried_refs_1 = set(butler.registry.queryDatasets("flat", detector=2, physical_filter="Cam1-G")) 

500 self.assertEqual({ref}, queried_refs_1) 

501 queried_refs_2 = set(butler.registry.queryDatasets("flat", 

502 where="detector=2 AND physical_filter='Cam1-G'")) 

503 self.assertEqual({ref}, queried_refs_2) 

504 # Query for data IDs with a dataset constraint. 

505 queried_data_ids = set(butler.registry.queryDataIds({"instrument", "detector", "physical_filter"}, 

506 datasets={"flat"}, 

507 detector=2, physical_filter="Cam1-G")) 

508 self.assertEqual({ref.dataId}, queried_data_ids) 

509 # Add another instrument to the repo, and a dataset that uses it to 

510 # the `imported_g` collection. 

511 butler.registry.insertDimensionData("instrument", {"name": "Cam2"}) 

512 camera = DatasetType( 

513 "camera", 

514 dimensions=butler.registry.dimensions["instrument"].graph, 

515 storageClass="Camera", 

516 ) 

517 butler.registry.registerDatasetType(camera) 

518 butler.registry.insertDatasets(camera, [{"instrument": "Cam2"}], run="imported_g") 

519 # Initialize a new butler with `imported_g` as its default run. 

520 # This should not have a default instrument, because there are two. 

521 # Pass run instead of collections; this should set both. 

522 butler2 = Butler(butler=butler, run="imported_g") 

523 self.assertEqual(list(butler2.registry.defaults.collections), ["imported_g"]) 

524 self.assertEqual(butler2.registry.defaults.run, "imported_g") 

525 self.assertFalse(butler2.registry.defaults.dataId) 

526 # Initialize a new butler with an instrument default explicitly given. 

527 # Set collections instead of run, which should then be None. 

528 butler3 = Butler(butler=butler, collections=["imported_g"], instrument="Cam2") 

529 self.assertEqual(list(butler3.registry.defaults.collections), ["imported_g"]) 

530 self.assertIsNone(butler3.registry.defaults.run, None) 

531 self.assertEqual(butler3.registry.defaults.dataId.byName(), {"instrument": "Cam2"}) 

532 

533 def testJson(self): 

534 """Test JSON serialization mediated by registry. 

535 """ 

536 butler = self.makeButler(writeable=True) 

537 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

538 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

539 # Need to actually set defaults later, not at construction, because 

540 # we need to import the instrument before we can use it as a default. 

541 # Don't set a default instrument value for data IDs, because 'Cam1' 

542 # should be inferred by virtue of that being the only value in the 

543 # input collections. 

544 butler.registry.defaults = RegistryDefaults(collections=["imported_g"]) 

545 # Use findDataset without collections or instrument. 

546 ref = butler.registry.findDataset("flat", detector=2, physical_filter="Cam1-G") 

547 

548 # Transform the ref and dataset type to and from JSON 

549 # and check that it can be reconstructed properly 

550 

551 # Do it with the ref and a component ref in minimal and standard form 

552 compRef = ref.makeComponentRef("wcs") 

553 

554 for test_item in (ref, ref.datasetType, compRef, compRef.datasetType): 

555 for minimal in (False, True): 

556 json_str = test_item.to_json(minimal=minimal) 

557 from_json = type(test_item).from_json(json_str, registry=butler.registry) 

558 self.assertEqual(from_json, test_item, msg=f"From JSON '{json_str}' using registry") 

559 

560 # for minimal=False case also do a test without registry 

561 if not minimal: 

562 from_json = type(test_item).from_json(json_str, universe=butler.registry.dimensions) 

563 self.assertEqual(from_json, test_item, msg=f"From JSON '{json_str}' using universe") 

564 

565 def testJsonDimensionRecords(self): 

566 # Dimension Records 

567 butler = self.makeButler(writeable=True) 

568 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "hsc-rc2-subset.yaml")) 

569 

570 for dimension in ("detector", "visit"): 

571 records = butler.registry.queryDimensionRecords(dimension, instrument="HSC") 

572 for r in records: 

573 for minimal in (True, False): 

574 json_str = r.to_json(minimal=minimal) 

575 r_json = type(r).from_json(json_str, registry=butler.registry) 

576 self.assertEqual(r_json, r) 

577 # Also check equality of each of the components as dicts 

578 self.assertEqual(r_json.toDict(), r.toDict()) 

579 

580 def testWildcardQueries(self): 

581 """Test that different collection type queries work.""" 

582 

583 # Import data to play with. 

584 butler = self.makeButler(writeable=True) 

585 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

586 

587 # Create some collections 

588 created = {"collection", "u/user/test", "coll3"} 

589 for collection in created: 

590 butler.registry.registerCollection(collection, type=CollectionType.RUN) 

591 

592 collections = butler.registry.queryCollections() 

593 self.assertEqual(set(collections), created) 

594 

595 expressions = ( 

596 ("collection", {"collection"}), 

597 (..., created), 

598 ("*", created), 

599 (("collection", "*"), created), 

600 ("u/*", {"u/user/test"}), 

601 (re.compile("u.*"), {"u/user/test"}), 

602 (re.compile(".*oll.*"), {"collection", "coll3"}), 

603 ("*oll*", {"collection", "coll3"}), 

604 ((re.compile(r".*\d$"), "u/user/test"), {"coll3", "u/user/test"}), 

605 ("*[0-9]", {"coll3"}), 

606 ) 

607 for expression, expected in expressions: 

608 result = butler.registry.queryCollections(expression) 

609 self.assertEqual(set(result), expected) 

610 

611 

612class SimpleButlerUUIDTestCase(SimpleButlerTestCase): 

613 """Same as SimpleButlerTestCase but uses UUID-based datasets manager and 

614 loads datasets from YAML file with UUIDs. 

615 """ 

616 

617 datasetsManager = \ 

618 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

619 datasetsImportFile = "datasets-uuid.yaml" 

620 datasetsIdType = uuid.UUID 

621 

622 

623class SimpleButlerMixedUUIDTestCase(SimpleButlerTestCase): 

624 """Same as SimpleButlerTestCase but uses UUID-based datasets manager and 

625 loads datasets from YAML file with integer IDs. 

626 """ 

627 

628 datasetsManager = \ 

629 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

630 datasetsImportFile = "datasets.yaml" 

631 datasetsIdType = uuid.UUID 

632 

633 

634if __name__ == "__main__": 634 ↛ 635line 634 didn't jump to line 635, because the condition on line 634 was never true

635 unittest.main()