Coverage for tests/test_simpleButler.py: 12%

271 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-04 02:04 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import json 

25import os 

26import re 

27import tempfile 

28import unittest 

29import uuid 

30from typing import Any 

31 

32try: 

33 import numpy as np 

34except ImportError: 

35 np = None 

36 

37import astropy.time 

38from lsst.daf.butler import Butler, ButlerConfig, CollectionType, DatasetRef, DatasetType, Registry, Timespan 

39from lsst.daf.butler.registry import ConflictingDefinitionError, RegistryConfig, RegistryDefaults 

40from lsst.daf.butler.tests import DatastoreMock 

41from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

42 

43TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

44 

45 

46class SimpleButlerTestCase(unittest.TestCase): 

47 """Tests for butler (including import/export functionality) that should not 

48 depend on the Registry Database backend or Datastore implementation, and 

49 can instead utilize an in-memory SQLite Registry and a mocked Datastore. 

50 """ 

51 

52 datasetsManager = ( 

53 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

54 ) 

55 datasetsImportFile = "datasets-uuid.yaml" 

56 datasetsIdType = uuid.UUID 

57 

58 def setUp(self): 

59 self.root = makeTestTempDir(TESTDIR) 

60 

61 def tearDown(self): 

62 removeTestTempDir(self.root) 

63 

64 def makeButler(self, **kwargs: Any) -> Butler: 

65 """Return new Butler instance on each call.""" 

66 config = ButlerConfig() 

67 

68 # make separate temporary directory for registry of this instance 

69 tmpdir = tempfile.mkdtemp(dir=self.root) 

70 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3" 

71 config["registry", "managers", "datasets"] = self.datasetsManager 

72 config["root"] = self.root 

73 

74 # have to make a registry first 

75 registryConfig = RegistryConfig(config.get("registry")) 

76 if self.datasetsIdType is int: 

77 with self.assertWarns(FutureWarning): 

78 Registry.createFromConfig(registryConfig) 

79 else: 

80 Registry.createFromConfig(registryConfig) 

81 

82 butler = Butler(config, **kwargs) 

83 DatastoreMock.apply(butler) 

84 return butler 

85 

86 def comparableRef(self, ref: DatasetRef) -> DatasetRef: 

87 """Return a DatasetRef that can be compared to a DatasetRef from 

88 other repository. 

89 

90 For repositories that do not support round-trip of ID values this 

91 method returns unresolved DatasetRef, for round-trip-safe repos it 

92 returns unchanged ref. 

93 """ 

94 return ref if self.datasetsIdType is uuid.UUID else ref.unresolved() 

95 

96 def testReadBackwardsCompatibility(self): 

97 """Test that we can read an export file written by a previous version 

98 and commit to the daf_butler git repo. 

99 

100 Notes 

101 ----- 

102 At present this export file includes only dimension data, not datasets, 

103 which greatly limits the usefulness of this test. We should address 

104 this at some point, but I think it's best to wait for the changes to 

105 the export format required for CALIBRATION collections to land. 

106 """ 

107 butler = self.makeButler(writeable=True) 

108 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "hsc-rc2-subset.yaml")) 

109 # Spot-check a few things, but the most important test is just that 

110 # the above does not raise. 

111 self.assertGreaterEqual( 

112 set(record.id for record in butler.registry.queryDimensionRecords("detector", instrument="HSC")), 

113 set(range(104)), # should have all science CCDs; may have some focus ones. 

114 ) 

115 self.assertGreaterEqual( 

116 { 

117 (record.id, record.physical_filter) 

118 for record in butler.registry.queryDimensionRecords("visit", instrument="HSC") 

119 }, 

120 { 

121 (27136, "HSC-Z"), 

122 (11694, "HSC-G"), 

123 (23910, "HSC-R"), 

124 (11720, "HSC-Y"), 

125 (23900, "HSC-R"), 

126 (22646, "HSC-Y"), 

127 (1248, "HSC-I"), 

128 (19680, "HSC-I"), 

129 (1240, "HSC-I"), 

130 (424, "HSC-Y"), 

131 (19658, "HSC-I"), 

132 (344, "HSC-Y"), 

133 (1218, "HSC-R"), 

134 (1190, "HSC-Z"), 

135 (23718, "HSC-R"), 

136 (11700, "HSC-G"), 

137 (26036, "HSC-G"), 

138 (23872, "HSC-R"), 

139 (1170, "HSC-Z"), 

140 (1876, "HSC-Y"), 

141 }, 

142 ) 

143 

144 def testDatasetTransfers(self): 

145 """Test exporting all datasets from a repo and then importing them all 

146 back in again. 

147 """ 

148 # Import data to play with. 

149 butler1 = self.makeButler(writeable=True) 

150 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

151 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

152 with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml") as file: 

153 # Export all datasets. 

154 with butler1.export(filename=file.name) as exporter: 

155 exporter.saveDatasets(butler1.registry.queryDatasets(..., collections=...)) 

156 # Import it all again. 

157 butler2 = self.makeButler(writeable=True) 

158 butler2.import_(filename=file.name) 

159 datasets1 = list(butler1.registry.queryDatasets(..., collections=...)) 

160 datasets2 = list(butler2.registry.queryDatasets(..., collections=...)) 

161 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets1)) 

162 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets2)) 

163 self.assertCountEqual( 

164 [self.comparableRef(ref) for ref in datasets1], 

165 [self.comparableRef(ref) for ref in datasets2], 

166 ) 

167 

168 def testImportTwice(self): 

169 """Test exporting dimension records and datasets from a repo and then 

170 importing them all back in again twice. 

171 """ 

172 if self.datasetsIdType is not uuid.UUID: 

173 self.skipTest("This test can only work for UUIDs") 

174 # Import data to play with. 

175 butler1 = self.makeButler(writeable=True) 

176 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

177 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

178 with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as file: 

179 # Export all datasets. 

180 with butler1.export(filename=file.name) as exporter: 

181 exporter.saveDatasets(butler1.registry.queryDatasets(..., collections=...)) 

182 butler2 = self.makeButler(writeable=True) 

183 # Import it once. 

184 butler2.import_(filename=file.name) 

185 # Import it again 

186 butler2.import_(filename=file.name) 

187 datasets1 = list(butler1.registry.queryDatasets(..., collections=...)) 

188 datasets2 = list(butler2.registry.queryDatasets(..., collections=...)) 

189 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets1)) 

190 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets2)) 

191 self.assertCountEqual( 

192 [self.comparableRef(ref) for ref in datasets1], 

193 [self.comparableRef(ref) for ref in datasets2], 

194 ) 

195 

196 def testDatasetImportReuseIds(self): 

197 """Test for import that should preserve dataset IDs. 

198 

199 This test assumes that dataset IDs in datasets YAML are different from 

200 what auto-incremental insert would produce. 

201 """ 

202 if self.datasetsIdType is not int: 

203 self.skipTest("This test can only work for UUIDs") 

204 # Import data to play with. 

205 butler = self.makeButler(writeable=True) 

206 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

207 filename = os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile) 

208 butler.import_(filename=filename, reuseIds=True) 

209 datasets = list(butler.registry.queryDatasets(..., collections=...)) 

210 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets)) 

211 # IDs are copied from YAML, list needs to be updated if file contents 

212 # is changed. 

213 self.assertCountEqual( 

214 [ref.id for ref in datasets], 

215 [1001, 1002, 1003, 1010, 1020, 1030, 2001, 2002, 2003, 2010, 2020, 2030, 2040], 

216 ) 

217 

218 # Try once again, it will raise 

219 with self.assertRaises(ConflictingDefinitionError): 

220 butler.import_(filename=filename, reuseIds=True) 

221 

222 def testCollectionTransfers(self): 

223 """Test exporting and then importing collections of various types.""" 

224 # Populate a registry with some datasets. 

225 butler1 = self.makeButler(writeable=True) 

226 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

227 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

228 registry1 = butler1.registry 

229 # Add some more collections. 

230 registry1.registerRun("run1") 

231 registry1.registerCollection("tag1", CollectionType.TAGGED) 

232 registry1.registerCollection("calibration1", CollectionType.CALIBRATION) 

233 registry1.registerCollection("chain1", CollectionType.CHAINED) 

234 registry1.registerCollection("chain2", CollectionType.CHAINED) 

235 registry1.setCollectionChain("chain1", ["tag1", "run1", "chain2"]) 

236 registry1.setCollectionChain("chain2", ["calibration1", "run1"]) 

237 # Associate some datasets into the TAGGED and CALIBRATION collections. 

238 flats1 = list(registry1.queryDatasets("flat", collections=...)) 

239 registry1.associate("tag1", flats1) 

240 t1 = astropy.time.Time("2020-01-01T01:00:00", format="isot", scale="tai") 

241 t2 = astropy.time.Time("2020-01-01T02:00:00", format="isot", scale="tai") 

242 t3 = astropy.time.Time("2020-01-01T03:00:00", format="isot", scale="tai") 

243 bias1a = registry1.findDataset("bias", instrument="Cam1", detector=1, collections="imported_g") 

244 bias2a = registry1.findDataset("bias", instrument="Cam1", detector=2, collections="imported_g") 

245 bias3a = registry1.findDataset("bias", instrument="Cam1", detector=3, collections="imported_g") 

246 bias2b = registry1.findDataset("bias", instrument="Cam1", detector=2, collections="imported_r") 

247 bias3b = registry1.findDataset("bias", instrument="Cam1", detector=3, collections="imported_r") 

248 registry1.certify("calibration1", [bias2a, bias3a], Timespan(t1, t2)) 

249 registry1.certify("calibration1", [bias2b], Timespan(t2, None)) 

250 registry1.certify("calibration1", [bias3b], Timespan(t2, t3)) 

251 registry1.certify("calibration1", [bias1a], Timespan.makeEmpty()) 

252 

253 with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml") as file: 

254 # Export all collections, and some datasets. 

255 with butler1.export(filename=file.name) as exporter: 

256 # Sort results to put chain1 before chain2, which is 

257 # intentionally not topological order. 

258 for collection in sorted(registry1.queryCollections()): 

259 exporter.saveCollection(collection) 

260 exporter.saveDatasets(flats1) 

261 exporter.saveDatasets([bias1a, bias2a, bias2b, bias3a, bias3b]) 

262 # Import them into a new registry. 

263 butler2 = self.makeButler(writeable=True) 

264 butler2.import_(filename=file.name) 

265 registry2 = butler2.registry 

266 # Check that it all round-tripped, starting with the collections 

267 # themselves. 

268 self.assertIs(registry2.getCollectionType("run1"), CollectionType.RUN) 

269 self.assertIs(registry2.getCollectionType("tag1"), CollectionType.TAGGED) 

270 self.assertIs(registry2.getCollectionType("calibration1"), CollectionType.CALIBRATION) 

271 self.assertIs(registry2.getCollectionType("chain1"), CollectionType.CHAINED) 

272 self.assertIs(registry2.getCollectionType("chain2"), CollectionType.CHAINED) 

273 self.assertEqual( 

274 list(registry2.getCollectionChain("chain1")), 

275 ["tag1", "run1", "chain2"], 

276 ) 

277 self.assertEqual( 

278 list(registry2.getCollectionChain("chain2")), 

279 ["calibration1", "run1"], 

280 ) 

281 # Check that tag collection contents are the same. 

282 self.maxDiff = None 

283 self.assertCountEqual( 

284 [self.comparableRef(ref) for ref in registry1.queryDatasets(..., collections="tag1")], 

285 [self.comparableRef(ref) for ref in registry2.queryDatasets(..., collections="tag1")], 

286 ) 

287 # Check that calibration collection contents are the same. 

288 self.assertCountEqual( 

289 [ 

290 (self.comparableRef(assoc.ref), assoc.timespan) 

291 for assoc in registry1.queryDatasetAssociations("bias", collections="calibration1") 

292 ], 

293 [ 

294 (self.comparableRef(assoc.ref), assoc.timespan) 

295 for assoc in registry2.queryDatasetAssociations("bias", collections="calibration1") 

296 ], 

297 ) 

298 

299 def testButlerGet(self): 

300 """Test that butler.get can work with different variants.""" 

301 

302 # Import data to play with. 

303 butler = self.makeButler(writeable=True) 

304 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

305 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

306 

307 # Find the DatasetRef for a flat 

308 coll = "imported_g" 

309 flat2g = butler.registry.findDataset( 

310 "flat", instrument="Cam1", detector=2, physical_filter="Cam1-G", collections=coll 

311 ) 

312 

313 # Create a numpy integer to check that works fine 

314 detector_np = np.int64(2) if np else 2 

315 print(type(detector_np)) 

316 

317 # Try to get it using different variations of dataId + keyword 

318 # arguments 

319 # Note that instrument.class_name does not work 

320 variants = ( 

321 (None, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}), 

322 (None, {"instrument": "Cam1", "detector": detector_np, "physical_filter": "Cam1-G"}), 

323 ({"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}, {}), 

324 ({"instrument": "Cam1", "detector": detector_np, "physical_filter": "Cam1-G"}, {}), 

325 ({"instrument": "Cam1", "detector": 2}, {"physical_filter": "Cam1-G"}), 

326 ({"detector.full_name": "Ab"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}), 

327 ({"full_name": "Ab"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}), 

328 (None, {"full_name": "Ab", "instrument": "Cam1", "physical_filter": "Cam1-G"}), 

329 (None, {"detector": "Ab", "instrument": "Cam1", "physical_filter": "Cam1-G"}), 

330 ({"name_in_raft": "b", "raft": "A"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}), 

331 ({"name_in_raft": "b"}, {"raft": "A", "instrument": "Cam1", "physical_filter": "Cam1-G"}), 

332 (None, {"name_in_raft": "b", "raft": "A", "instrument": "Cam1", "physical_filter": "Cam1-G"}), 

333 ( 

334 {"detector.name_in_raft": "b", "detector.raft": "A"}, 

335 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

336 ), 

337 ( 

338 { 

339 "detector.name_in_raft": "b", 

340 "detector.raft": "A", 

341 "instrument": "Cam1", 

342 "physical_filter": "Cam1-G", 

343 }, 

344 {}, 

345 ), 

346 # Duplicate (but valid) information. 

347 (None, {"instrument": "Cam1", "detector": 2, "raft": "A", "physical_filter": "Cam1-G"}), 

348 ({"detector": 2}, {"instrument": "Cam1", "raft": "A", "physical_filter": "Cam1-G"}), 

349 ({"raft": "A"}, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}), 

350 ({"raft": "A"}, {"instrument": "Cam1", "detector": "Ab", "physical_filter": "Cam1-G"}), 

351 ) 

352 

353 for dataId, kwds in variants: 

354 try: 

355 flat_id, _ = butler.get("flat", dataId=dataId, collections=coll, **kwds) 

356 except Exception as e: 

357 raise type(e)(f"{str(e)}: dataId={dataId}, kwds={kwds}") from e 

358 self.assertEqual(flat_id, flat2g.id, msg=f"DataId: {dataId}, kwds: {kwds}") 

359 

360 # Check that bad combinations raise. 

361 variants = ( 

362 # Inconsistent detector information. 

363 (None, {"instrument": "Cam1", "detector": 2, "raft": "B", "physical_filter": "Cam1-G"}), 

364 ({"detector": 2}, {"instrument": "Cam1", "raft": "B", "physical_filter": "Cam1-G"}), 

365 ({"detector": 12}, {"instrument": "Cam1", "raft": "B", "physical_filter": "Cam1-G"}), 

366 ({"raft": "B"}, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}), 

367 ({"raft": "B"}, {"instrument": "Cam1", "detector": "Ab", "physical_filter": "Cam1-G"}), 

368 # Under-specified. 

369 ({"raft": "B"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}), 

370 # Spurious kwargs. 

371 (None, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G", "x": "y"}), 

372 ({"x": "y"}, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}), 

373 ) 

374 for dataId, kwds in variants: 

375 with self.assertRaises((ValueError, LookupError)): 

376 butler.get("flat", dataId=dataId, collections=coll, **kwds) 

377 

378 def testGetCalibration(self): 

379 """Test that `Butler.get` can be used to fetch from 

380 `~CollectionType.CALIBRATION` collections if the data ID includes 

381 extra dimensions with temporal information. 

382 """ 

383 # Import data to play with. 

384 butler = self.makeButler(writeable=True) 

385 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

386 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

387 # Certify some biases into a CALIBRATION collection. 

388 registry = butler.registry 

389 registry.registerCollection("calibs", CollectionType.CALIBRATION) 

390 t1 = astropy.time.Time("2020-01-01T01:00:00", format="isot", scale="tai") 

391 t2 = astropy.time.Time("2020-01-01T02:00:00", format="isot", scale="tai") 

392 t3 = astropy.time.Time("2020-01-01T03:00:00", format="isot", scale="tai") 

393 bias2a = registry.findDataset("bias", instrument="Cam1", detector=2, collections="imported_g") 

394 bias3a = registry.findDataset("bias", instrument="Cam1", detector=3, collections="imported_g") 

395 bias2b = registry.findDataset("bias", instrument="Cam1", detector=2, collections="imported_r") 

396 bias3b = registry.findDataset("bias", instrument="Cam1", detector=3, collections="imported_r") 

397 registry.certify("calibs", [bias2a, bias3a], Timespan(t1, t2)) 

398 registry.certify("calibs", [bias2b], Timespan(t2, None)) 

399 registry.certify("calibs", [bias3b], Timespan(t2, t3)) 

400 # Insert some exposure dimension data. 

401 registry.insertDimensionData( 

402 "exposure", 

403 { 

404 "instrument": "Cam1", 

405 "id": 3, 

406 "obs_id": "three", 

407 "timespan": Timespan(t1, t2), 

408 "physical_filter": "Cam1-G", 

409 "day_obs": 20201114, 

410 "seq_num": 55, 

411 }, 

412 { 

413 "instrument": "Cam1", 

414 "id": 4, 

415 "obs_id": "four", 

416 "timespan": Timespan(t2, t3), 

417 "physical_filter": "Cam1-G", 

418 "day_obs": 20211114, 

419 "seq_num": 42, 

420 }, 

421 ) 

422 # Get some biases from raw-like data IDs. 

423 bias2a_id, _ = butler.get( 

424 "bias", {"instrument": "Cam1", "exposure": 3, "detector": 2}, collections="calibs" 

425 ) 

426 self.assertEqual(bias2a_id, bias2a.id) 

427 bias3b_id, _ = butler.get( 

428 "bias", {"instrument": "Cam1", "exposure": 4, "detector": 3}, collections="calibs" 

429 ) 

430 self.assertEqual(bias3b_id, bias3b.id) 

431 

432 # Get using the kwarg form 

433 bias3b_id, _ = butler.get("bias", instrument="Cam1", exposure=4, detector=3, collections="calibs") 

434 self.assertEqual(bias3b_id, bias3b.id) 

435 

436 # Do it again but using the record information 

437 bias2a_id, _ = butler.get( 

438 "bias", 

439 {"instrument": "Cam1", "exposure.obs_id": "three", "detector.full_name": "Ab"}, 

440 collections="calibs", 

441 ) 

442 self.assertEqual(bias2a_id, bias2a.id) 

443 bias3b_id, _ = butler.get( 

444 "bias", 

445 {"exposure.obs_id": "four", "detector.full_name": "Ba"}, 

446 collections="calibs", 

447 instrument="Cam1", 

448 ) 

449 self.assertEqual(bias3b_id, bias3b.id) 

450 

451 # And again but this time using the alternate value rather than 

452 # the primary. 

453 bias3b_id, _ = butler.get( 

454 "bias", {"exposure": "four", "detector": "Ba"}, collections="calibs", instrument="Cam1" 

455 ) 

456 self.assertEqual(bias3b_id, bias3b.id) 

457 

458 # And again but this time using the alternate value rather than 

459 # the primary and do it in the keyword arguments. 

460 bias3b_id, _ = butler.get( 

461 "bias", exposure="four", detector="Ba", collections="calibs", instrument="Cam1" 

462 ) 

463 self.assertEqual(bias3b_id, bias3b.id) 

464 

465 # Now with implied record columns 

466 bias3b_id, _ = butler.get( 

467 "bias", 

468 day_obs=20211114, 

469 seq_num=42, 

470 raft="B", 

471 name_in_raft="a", 

472 collections="calibs", 

473 instrument="Cam1", 

474 ) 

475 self.assertEqual(bias3b_id, bias3b.id) 

476 

477 # Allow a fully-specified dataId and unnecessary extra information 

478 # that comes from the record. 

479 bias3b_id, _ = butler.get( 

480 "bias", 

481 dataId=dict( 

482 exposure=4, 

483 day_obs=20211114, 

484 seq_num=42, 

485 detector=3, 

486 instrument="Cam1", 

487 ), 

488 collections="calibs", 

489 ) 

490 self.assertEqual(bias3b_id, bias3b.id) 

491 

492 # Extra but inconsistent record values are a problem. 

493 with self.assertRaises(ValueError): 

494 bias3b_id, _ = butler.get( 

495 "bias", 

496 exposure=3, 

497 day_obs=20211114, 

498 seq_num=42, 

499 detector=3, 

500 collections="calibs", 

501 instrument="Cam1", 

502 ) 

503 

504 # Ensure that spurious kwargs cause an exception. 

505 with self.assertRaises(ValueError): 

506 butler.get( 

507 "bias", 

508 {"exposure.obs_id": "four", "immediate": True, "detector.full_name": "Ba"}, 

509 collections="calibs", 

510 instrument="Cam1", 

511 ) 

512 

513 with self.assertRaises(ValueError): 

514 butler.get( 

515 "bias", 

516 day_obs=20211114, 

517 seq_num=42, 

518 raft="B", 

519 name_in_raft="a", 

520 collections="calibs", 

521 instrument="Cam1", 

522 immediate=True, 

523 ) 

524 

525 def testRegistryDefaults(self): 

526 """Test that we can default the collections and some data ID keys when 

527 constructing a butler. 

528 

529 Many tests that use default run already exist in ``test_butler.py``, so 

530 that isn't tested here. And while most of this functionality is 

531 implemented in `Registry`, we test it here instead of 

532 ``daf/butler/tests/registry.py`` because it shouldn't depend on the 

533 database backend at all. 

534 """ 

535 butler = self.makeButler(writeable=True) 

536 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

537 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

538 # Need to actually set defaults later, not at construction, because 

539 # we need to import the instrument before we can use it as a default. 

540 # Don't set a default instrument value for data IDs, because 'Cam1' 

541 # should be inferred by virtue of that being the only value in the 

542 # input collections. 

543 butler.registry.defaults = RegistryDefaults(collections=["imported_g"]) 

544 # Use findDataset without collections or instrument. 

545 ref = butler.registry.findDataset("flat", detector=2, physical_filter="Cam1-G") 

546 # Do the same with Butler.get; this should ultimately invoke a lot of 

547 # the same code, so it's a bit circular, but mostly we're checking that 

548 # it works at all. 

549 dataset_id, _ = butler.get("flat", detector=2, physical_filter="Cam1-G") 

550 self.assertEqual(ref.id, dataset_id) 

551 # Query for datasets. Test defaulting the data ID in both kwargs and 

552 # in the WHERE expression. 

553 queried_refs_1 = set(butler.registry.queryDatasets("flat", detector=2, physical_filter="Cam1-G")) 

554 self.assertEqual({ref}, queried_refs_1) 

555 queried_refs_2 = set( 

556 butler.registry.queryDatasets("flat", where="detector=2 AND physical_filter='Cam1-G'") 

557 ) 

558 self.assertEqual({ref}, queried_refs_2) 

559 # Query for data IDs with a dataset constraint. 

560 queried_data_ids = set( 

561 butler.registry.queryDataIds( 

562 {"instrument", "detector", "physical_filter"}, 

563 datasets={"flat"}, 

564 detector=2, 

565 physical_filter="Cam1-G", 

566 ) 

567 ) 

568 self.assertEqual({ref.dataId}, queried_data_ids) 

569 # Add another instrument to the repo, and a dataset that uses it to 

570 # the `imported_g` collection. 

571 butler.registry.insertDimensionData("instrument", {"name": "Cam2"}) 

572 camera = DatasetType( 

573 "camera", 

574 dimensions=butler.registry.dimensions["instrument"].graph, 

575 storageClass="Camera", 

576 ) 

577 butler.registry.registerDatasetType(camera) 

578 butler.registry.insertDatasets(camera, [{"instrument": "Cam2"}], run="imported_g") 

579 # Initialize a new butler with `imported_g` as its default run. 

580 # This should not have a default instrument, because there are two. 

581 # Pass run instead of collections; this should set both. 

582 butler2 = Butler(butler=butler, run="imported_g") 

583 self.assertEqual(list(butler2.registry.defaults.collections), ["imported_g"]) 

584 self.assertEqual(butler2.registry.defaults.run, "imported_g") 

585 self.assertFalse(butler2.registry.defaults.dataId) 

586 # Initialize a new butler with an instrument default explicitly given. 

587 # Set collections instead of run, which should then be None. 

588 butler3 = Butler(butler=butler, collections=["imported_g"], instrument="Cam2") 

589 self.assertEqual(list(butler3.registry.defaults.collections), ["imported_g"]) 

590 self.assertIsNone(butler3.registry.defaults.run, None) 

591 self.assertEqual(butler3.registry.defaults.dataId.byName(), {"instrument": "Cam2"}) 

592 

593 def testJson(self): 

594 """Test JSON serialization mediated by registry.""" 

595 butler = self.makeButler(writeable=True) 

596 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

597 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)) 

598 # Need to actually set defaults later, not at construction, because 

599 # we need to import the instrument before we can use it as a default. 

600 # Don't set a default instrument value for data IDs, because 'Cam1' 

601 # should be inferred by virtue of that being the only value in the 

602 # input collections. 

603 butler.registry.defaults = RegistryDefaults(collections=["imported_g"]) 

604 # Use findDataset without collections or instrument. 

605 ref = butler.registry.findDataset("flat", detector=2, physical_filter="Cam1-G") 

606 

607 # Transform the ref and dataset type to and from JSON 

608 # and check that it can be reconstructed properly 

609 

610 # Do it with the ref and a component ref in minimal and standard form 

611 compRef = ref.makeComponentRef("wcs") 

612 

613 for test_item in (ref, ref.datasetType, compRef, compRef.datasetType): 

614 for minimal in (False, True): 

615 json_str = test_item.to_json(minimal=minimal) 

616 from_json = type(test_item).from_json(json_str, registry=butler.registry) 

617 self.assertEqual(from_json, test_item, msg=f"From JSON '{json_str}' using registry") 

618 

619 # for minimal=False case also do a test without registry 

620 if not minimal: 

621 from_json = type(test_item).from_json(json_str, universe=butler.registry.dimensions) 

622 self.assertEqual(from_json, test_item, msg=f"From JSON '{json_str}' using universe") 

623 

624 def testJsonDimensionRecordsAndHtmlRepresentation(self): 

625 # Dimension Records 

626 butler = self.makeButler(writeable=True) 

627 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "hsc-rc2-subset.yaml")) 

628 

629 for dimension in ("detector", "visit"): 

630 records = butler.registry.queryDimensionRecords(dimension, instrument="HSC") 

631 for r in records: 

632 for minimal in (True, False): 

633 json_str = r.to_json(minimal=minimal) 

634 r_json = type(r).from_json(json_str, registry=butler.registry) 

635 self.assertEqual(r_json, r) 

636 # check with direct method 

637 simple = r.to_simple() 

638 fromDirect = type(simple).direct(**json.loads(json_str)) 

639 self.assertEqual(simple, fromDirect) 

640 # Also check equality of each of the components as dicts 

641 self.assertEqual(r_json.toDict(), r.toDict()) 

642 

643 # check the html representation of records 

644 r_html = r._repr_html_() 

645 self.assertTrue(isinstance(r_html, str)) 

646 self.assertIn(dimension, r_html) 

647 

648 def testWildcardQueries(self): 

649 """Test that different collection type queries work.""" 

650 

651 # Import data to play with. 

652 butler = self.makeButler(writeable=True) 

653 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) 

654 

655 # Create some collections 

656 created = {"collection", "u/user/test", "coll3"} 

657 for collection in created: 

658 butler.registry.registerCollection(collection, type=CollectionType.RUN) 

659 

660 collections = butler.registry.queryCollections() 

661 self.assertEqual(set(collections), created) 

662 

663 expressions = ( 

664 ("collection", {"collection"}), 

665 (..., created), 

666 ("*", created), 

667 (("collection", "*"), created), 

668 ("u/*", {"u/user/test"}), 

669 (re.compile("u.*"), {"u/user/test"}), 

670 (re.compile(".*oll.*"), {"collection", "coll3"}), 

671 ("*oll*", {"collection", "coll3"}), 

672 ((re.compile(r".*\d$"), "u/user/test"), {"coll3", "u/user/test"}), 

673 ("*[0-9]", {"coll3"}), 

674 ) 

675 for expression, expected in expressions: 

676 result = butler.registry.queryCollections(expression) 

677 self.assertEqual(set(result), expected) 

678 

679 

680class SimpleButlerMixedUUIDTestCase(SimpleButlerTestCase): 

681 """Same as SimpleButlerTestCase but uses UUID-based datasets manager and 

682 loads datasets from YAML file with integer IDs. 

683 """ 

684 

685 datasetsManager = ( 

686 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

687 ) 

688 datasetsImportFile = "datasets.yaml" 

689 datasetsIdType = uuid.UUID 

690 

691 

692if __name__ == "__main__": 692 ↛ 693line 692 didn't jump to line 693, because the condition on line 692 was never true

693 unittest.main()