Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["RegistryTests"] 

24 

25from abc import ABC, abstractmethod 

26import os 

27import re 

28import unittest 

29 

30import astropy.time 

31import sqlalchemy 

32from typing import Optional 

33 

34try: 

35 import numpy as np 

36except ImportError: 

37 np = None 

38 

39from ...core import ( 

40 DataCoordinate, 

41 DatasetRef, 

42 DatasetType, 

43 DimensionGraph, 

44 NamedValueSet, 

45 StorageClass, 

46 ddl, 

47 YamlRepoImportBackend 

48) 

49from .._registry import ( 

50 CollectionType, 

51 ConflictingDefinitionError, 

52 InconsistentDataIdError, 

53 Registry, 

54 RegistryConfig, 

55) 

56from ..wildcards import DatasetTypeRestriction 

57from ..interfaces import MissingCollectionError, ButlerAttributeExistsError 

58 

59 

60class RegistryTests(ABC): 

61 """Generic tests for the `Registry` class that can be subclassed to 

62 generate tests for different configurations. 

63 """ 

64 

65 collectionsManager: Optional[str] = None 

66 """Name of the collections manager class, if subclass provides value for 

67 this member then it overrides name specified in default configuration 

68 (`str`). 

69 """ 

70 

71 @classmethod 

72 @abstractmethod 

73 def getDataDir(cls) -> str: 

74 """Return the root directory containing test data YAML files. 

75 """ 

76 raise NotImplementedError() 

77 

78 def makeRegistryConfig(self) -> RegistryConfig: 

79 """Create RegistryConfig used to create a registry. 

80 

81 This method should be called by a subclass from `makeRegistry`. 

82 Returned instance will be pre-configured based on the values of class 

83 members, and default-configured for all other parametrs. Subclasses 

84 that need default configuration should just instantiate 

85 `RegistryConfig` directly. 

86 """ 

87 config = RegistryConfig() 

88 if self.collectionsManager: 

89 config["managers"]["collections"] = self.collectionsManager 

90 return config 

91 

92 @abstractmethod 

93 def makeRegistry(self) -> Registry: 

94 """Return the Registry instance to be tested. 

95 """ 

96 raise NotImplementedError() 

97 

98 def loadData(self, registry: Registry, filename: str): 

99 """Load registry test data from ``getDataDir/<filename>``, 

100 which should be a YAML import/export file. 

101 """ 

102 with open(os.path.join(self.getDataDir(), filename), 'r') as stream: 

103 backend = YamlRepoImportBackend(stream, registry) 

104 backend.register() 

105 backend.load(datastore=None) 

106 

107 def assertRowCount(self, registry: Registry, table: str, count: int): 

108 """Check the number of rows in table. 

109 """ 

110 # TODO: all tests that rely on this method should be rewritten, as it 

111 # needs to depend on Registry implementation details to have any chance 

112 # of working. 

113 sql = sqlalchemy.sql.select( 

114 [sqlalchemy.sql.func.count()] 

115 ).select_from( 

116 getattr(registry._tables, table) 

117 ) 

118 self.assertEqual(registry._db.query(sql).scalar(), count) 

119 

120 def testOpaque(self): 

121 """Tests for `Registry.registerOpaqueTable`, 

122 `Registry.insertOpaqueData`, `Registry.fetchOpaqueData`, and 

123 `Registry.deleteOpaqueData`. 

124 """ 

125 registry = self.makeRegistry() 

126 table = "opaque_table_for_testing" 

127 registry.registerOpaqueTable( 

128 table, 

129 spec=ddl.TableSpec( 

130 fields=[ 

131 ddl.FieldSpec("id", dtype=sqlalchemy.BigInteger, primaryKey=True), 

132 ddl.FieldSpec("name", dtype=sqlalchemy.String, length=16, nullable=False), 

133 ddl.FieldSpec("count", dtype=sqlalchemy.SmallInteger, nullable=True), 

134 ], 

135 ) 

136 ) 

137 rows = [ 

138 {"id": 1, "name": "one", "count": None}, 

139 {"id": 2, "name": "two", "count": 5}, 

140 {"id": 3, "name": "three", "count": 6}, 

141 ] 

142 registry.insertOpaqueData(table, *rows) 

143 self.assertCountEqual(rows, list(registry.fetchOpaqueData(table))) 

144 self.assertEqual(rows[0:1], list(registry.fetchOpaqueData(table, id=1))) 

145 self.assertEqual(rows[1:2], list(registry.fetchOpaqueData(table, name="two"))) 

146 self.assertEqual([], list(registry.fetchOpaqueData(table, id=1, name="two"))) 

147 registry.deleteOpaqueData(table, id=3) 

148 self.assertCountEqual(rows[:2], list(registry.fetchOpaqueData(table))) 

149 registry.deleteOpaqueData(table) 

150 self.assertEqual([], list(registry.fetchOpaqueData(table))) 

151 

152 def testDatasetType(self): 

153 """Tests for `Registry.registerDatasetType` and 

154 `Registry.getDatasetType`. 

155 """ 

156 registry = self.makeRegistry() 

157 # Check valid insert 

158 datasetTypeName = "test" 

159 storageClass = StorageClass("testDatasetType") 

160 registry.storageClasses.registerStorageClass(storageClass) 

161 dimensions = registry.dimensions.extract(("instrument", "visit")) 

162 differentDimensions = registry.dimensions.extract(("instrument", "patch")) 

163 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

164 # Inserting for the first time should return True 

165 self.assertTrue(registry.registerDatasetType(inDatasetType)) 

166 outDatasetType1 = registry.getDatasetType(datasetTypeName) 

167 self.assertEqual(outDatasetType1, inDatasetType) 

168 

169 # Re-inserting should work 

170 self.assertFalse(registry.registerDatasetType(inDatasetType)) 

171 # Except when they are not identical 

172 with self.assertRaises(ConflictingDefinitionError): 

173 nonIdenticalDatasetType = DatasetType(datasetTypeName, differentDimensions, storageClass) 

174 registry.registerDatasetType(nonIdenticalDatasetType) 

175 

176 # Template can be None 

177 datasetTypeName = "testNoneTemplate" 

178 storageClass = StorageClass("testDatasetType2") 

179 registry.storageClasses.registerStorageClass(storageClass) 

180 dimensions = registry.dimensions.extract(("instrument", "visit")) 

181 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

182 registry.registerDatasetType(inDatasetType) 

183 outDatasetType2 = registry.getDatasetType(datasetTypeName) 

184 self.assertEqual(outDatasetType2, inDatasetType) 

185 

186 allTypes = set(registry.queryDatasetTypes()) 

187 self.assertEqual(allTypes, {outDatasetType1, outDatasetType2}) 

188 

189 def testDimensions(self): 

190 """Tests for `Registry.insertDimensionData`, 

191 `Registry.syncDimensionData`, and `Registry.expandDataId`. 

192 """ 

193 registry = self.makeRegistry() 

194 dimensionName = "instrument" 

195 dimension = registry.dimensions[dimensionName] 

196 dimensionValue = {"name": "DummyCam", "visit_max": 10, "exposure_max": 10, "detector_max": 2, 

197 "class_name": "lsst.obs.base.Instrument"} 

198 registry.insertDimensionData(dimensionName, dimensionValue) 

199 # Inserting the same value twice should fail 

200 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

201 registry.insertDimensionData(dimensionName, dimensionValue) 

202 # expandDataId should retrieve the record we just inserted 

203 self.assertEqual( 

204 registry.expandDataId( 

205 instrument="DummyCam", 

206 graph=dimension.graph 

207 ).records[dimensionName].toDict(), 

208 dimensionValue 

209 ) 

210 # expandDataId should raise if there is no record with the given ID. 

211 with self.assertRaises(LookupError): 

212 registry.expandDataId({"instrument": "Unknown"}, graph=dimension.graph) 

213 # abstract_filter doesn't have a table; insert should fail. 

214 with self.assertRaises(TypeError): 

215 registry.insertDimensionData("abstract_filter", {"abstract_filter": "i"}) 

216 dimensionName2 = "physical_filter" 

217 dimension2 = registry.dimensions[dimensionName2] 

218 dimensionValue2 = {"name": "DummyCam_i", "abstract_filter": "i"} 

219 # Missing required dependency ("instrument") should fail 

220 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

221 registry.insertDimensionData(dimensionName2, dimensionValue2) 

222 # Adding required dependency should fix the failure 

223 dimensionValue2["instrument"] = "DummyCam" 

224 registry.insertDimensionData(dimensionName2, dimensionValue2) 

225 # expandDataId should retrieve the record we just inserted. 

226 self.assertEqual( 

227 registry.expandDataId( 

228 instrument="DummyCam", physical_filter="DummyCam_i", 

229 graph=dimension2.graph 

230 ).records[dimensionName2].toDict(), 

231 dimensionValue2 

232 ) 

233 # Use syncDimensionData to insert a new record successfully. 

234 dimensionName3 = "detector" 

235 dimensionValue3 = {"instrument": "DummyCam", "id": 1, "full_name": "one", 

236 "name_in_raft": "zero", "purpose": "SCIENCE"} 

237 self.assertTrue(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

238 # Sync that again. Note that one field ("raft") is NULL, and that 

239 # should be okay. 

240 self.assertFalse(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

241 # Now try that sync with the same primary key but a different value. 

242 # This should fail. 

243 with self.assertRaises(ConflictingDefinitionError): 

244 registry.syncDimensionData( 

245 dimensionName3, 

246 {"instrument": "DummyCam", "id": 1, "full_name": "one", 

247 "name_in_raft": "four", "purpose": "SCIENCE"} 

248 ) 

249 

250 @unittest.skipIf(np is None, "numpy not available.") 

251 def testNumpyDataId(self): 

252 """Test that we can use a numpy int in a dataId.""" 

253 registry = self.makeRegistry() 

254 dimensionEntries = [ 

255 ("instrument", {"instrument": "DummyCam"}), 

256 ("physical_filter", {"instrument": "DummyCam", "name": "d-r", "abstract_filter": "R"}), 

257 # Using an np.int64 here fails unless Records.fromDict is also 

258 # patched to look for numbers.Integral 

259 ("visit", {"instrument": "DummyCam", "id": 42, "name": "fortytwo", "physical_filter": "d-r"}), 

260 ] 

261 for args in dimensionEntries: 

262 registry.insertDimensionData(*args) 

263 

264 # Try a normal integer and something that looks like an int but 

265 # is not. 

266 for visit_id in (42, np.int64(42)): 

267 with self.subTest(visit_id=visit_id, id_type=type(visit_id).__name__): 

268 expanded = registry.expandDataId({"instrument": "DummyCam", "visit": visit_id}) 

269 self.assertEqual(expanded["visit"], int(visit_id)) 

270 self.assertIsInstance(expanded["visit"], int) 

271 

272 def testDataIdRelationships(self): 

273 """Test that `Registry.expandDataId` raises an exception when the given 

274 keys are inconsistent. 

275 """ 

276 registry = self.makeRegistry() 

277 self.loadData(registry, "base.yaml") 

278 # Insert a few more dimension records for the next test. 

279 registry.insertDimensionData( 

280 "exposure", 

281 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G"}, 

282 ) 

283 registry.insertDimensionData( 

284 "exposure", 

285 {"instrument": "Cam1", "id": 2, "name": "two", "physical_filter": "Cam1-G"}, 

286 ) 

287 registry.insertDimensionData( 

288 "visit_system", 

289 {"instrument": "Cam1", "id": 0, "name": "one-to-one"}, 

290 ) 

291 registry.insertDimensionData( 

292 "visit", 

293 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G", "visit_system": 0}, 

294 ) 

295 registry.insertDimensionData( 

296 "visit_definition", 

297 {"instrument": "Cam1", "visit": 1, "exposure": 1, "visit_system": 0}, 

298 ) 

299 with self.assertRaises(InconsistentDataIdError): 

300 registry.expandDataId( 

301 {"instrument": "Cam1", "visit": 1, "exposure": 2}, 

302 ) 

303 

304 def testDataset(self): 

305 """Basic tests for `Registry.insertDatasets`, `Registry.getDataset`, 

306 and `Registry.removeDatasets`. 

307 """ 

308 registry = self.makeRegistry() 

309 self.loadData(registry, "base.yaml") 

310 run = "test" 

311 registry.registerRun(run) 

312 datasetType = registry.getDatasetType("permabias") 

313 dataId = {"instrument": "Cam1", "detector": 2} 

314 ref, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

315 outRef = registry.getDataset(ref.id) 

316 self.assertIsNotNone(ref.id) 

317 self.assertEqual(ref, outRef) 

318 with self.assertRaises(ConflictingDefinitionError): 

319 registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

320 registry.removeDatasets([ref]) 

321 self.assertIsNone(registry.findDataset(datasetType, dataId, collections=[run])) 

322 

323 def testFindDataset(self): 

324 """Tests for `Registry.findDataset`. 

325 """ 

326 registry = self.makeRegistry() 

327 self.loadData(registry, "base.yaml") 

328 run = "test" 

329 datasetType = registry.getDatasetType("permabias") 

330 dataId = {"instrument": "Cam1", "detector": 4} 

331 registry.registerRun(run) 

332 inputRef, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

333 outputRef = registry.findDataset(datasetType, dataId, collections=[run]) 

334 self.assertEqual(outputRef, inputRef) 

335 # Check that retrieval with invalid dataId raises 

336 with self.assertRaises(LookupError): 

337 dataId = {"instrument": "Cam1"} # no detector 

338 registry.findDataset(datasetType, dataId, collections=run) 

339 # Check that different dataIds match to different datasets 

340 dataId1 = {"instrument": "Cam1", "detector": 1} 

341 inputRef1, = registry.insertDatasets(datasetType, dataIds=[dataId1], run=run) 

342 dataId2 = {"instrument": "Cam1", "detector": 2} 

343 inputRef2, = registry.insertDatasets(datasetType, dataIds=[dataId2], run=run) 

344 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef1) 

345 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef2) 

346 self.assertNotEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef2) 

347 self.assertNotEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef1) 

348 # Check that requesting a non-existing dataId returns None 

349 nonExistingDataId = {"instrument": "Cam1", "detector": 3} 

350 self.assertIsNone(registry.findDataset(datasetType, nonExistingDataId, collections=run)) 

351 

352 def testDatasetTypeComponentQueries(self): 

353 """Test component options when querying for dataset types. 

354 """ 

355 registry = self.makeRegistry() 

356 self.loadData(registry, "base.yaml") 

357 self.loadData(registry, "datasets.yaml") 

358 # Test querying for dataset types with different inputs. 

359 # First query for all dataset types; components should only be included 

360 # when components=True. 

361 self.assertEqual( 

362 {"permabias", "permaflat"}, 

363 NamedValueSet(registry.queryDatasetTypes()).names 

364 ) 

365 self.assertEqual( 

366 {"permabias", "permaflat"}, 

367 NamedValueSet(registry.queryDatasetTypes(components=False)).names 

368 ) 

369 self.assertLess( 

370 {"permabias", "permaflat", "permabias.wcs", "permaflat.photoCalib"}, 

371 NamedValueSet(registry.queryDatasetTypes(components=True)).names 

372 ) 

373 # Use a pattern that can match either parent or components. Again, 

374 # components are only returned if components=True. 

375 self.assertEqual( 

376 {"permabias"}, 

377 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"))).names 

378 ) 

379 self.assertEqual( 

380 {"permabias"}, 

381 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"), components=False)).names 

382 ) 

383 self.assertLess( 

384 {"permabias", "permabias.wcs"}, 

385 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"), components=True)).names 

386 ) 

387 # This pattern matches only a component. In this case we also return 

388 # that component dataset type if components=None. 

389 self.assertEqual( 

390 {"permabias.wcs"}, 

391 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"))).names 

392 ) 

393 self.assertEqual( 

394 set(), 

395 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"), components=False)).names 

396 ) 

397 self.assertEqual( 

398 {"permabias.wcs"}, 

399 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"), components=True)).names 

400 ) 

401 

402 def testComponentLookups(self): 

403 """Test searching for component datasets via their parents. 

404 """ 

405 registry = self.makeRegistry() 

406 self.loadData(registry, "base.yaml") 

407 self.loadData(registry, "datasets.yaml") 

408 # Test getting the child dataset type (which does still exist in the 

409 # Registry), and check for consistency with 

410 # DatasetRef.makeComponentRef. 

411 collection = "imported_g" 

412 parentType = registry.getDatasetType("permabias") 

413 childType = registry.getDatasetType("permabias.wcs") 

414 parentRefResolved = registry.findDataset(parentType, collections=collection, 

415 instrument="Cam1", detector=1) 

416 self.assertIsInstance(parentRefResolved, DatasetRef) 

417 self.assertEqual(childType, parentRefResolved.makeComponentRef("wcs").datasetType) 

418 # Search for a single dataset with findDataset. 

419 childRef1 = registry.findDataset("permabias.wcs", collections=collection, 

420 dataId=parentRefResolved.dataId) 

421 self.assertEqual(childRef1, parentRefResolved.makeComponentRef("wcs")) 

422 # Search for detector data IDs constrained by component dataset 

423 # existence with queryDimensions. 

424 dataIds = set(registry.queryDimensions( 

425 ["detector"], 

426 datasets=["permabias.wcs"], 

427 collections=collection, 

428 expand=False, 

429 )) 

430 self.assertEqual( 

431 dataIds, 

432 { 

433 DataCoordinate.standardize(instrument="Cam1", detector=d, graph=parentType.dimensions) 

434 for d in (1, 2, 3) 

435 } 

436 ) 

437 # Search for multiple datasets of a single type with queryDatasets. 

438 childRefs2 = set(registry.queryDatasets( 

439 "permabias.wcs", 

440 collections=collection, 

441 expand=False, 

442 )) 

443 self.assertEqual( 

444 {ref.unresolved() for ref in childRefs2}, 

445 {DatasetRef(childType, dataId) for dataId in dataIds} 

446 ) 

447 

448 def testCollections(self): 

449 """Tests for registry methods that manage collections. 

450 """ 

451 registry = self.makeRegistry() 

452 self.loadData(registry, "base.yaml") 

453 self.loadData(registry, "datasets.yaml") 

454 run1 = "imported_g" 

455 run2 = "imported_r" 

456 datasetType = "permabias" 

457 # Find some datasets via their run's collection. 

458 dataId1 = {"instrument": "Cam1", "detector": 1} 

459 ref1 = registry.findDataset(datasetType, dataId1, collections=run1) 

460 self.assertIsNotNone(ref1) 

461 dataId2 = {"instrument": "Cam1", "detector": 2} 

462 ref2 = registry.findDataset(datasetType, dataId2, collections=run1) 

463 self.assertIsNotNone(ref2) 

464 # Associate those into a new collection,then look for them there. 

465 tag1 = "tag1" 

466 registry.registerCollection(tag1, type=CollectionType.TAGGED) 

467 registry.associate(tag1, [ref1, ref2]) 

468 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

469 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

470 # Disassociate one and verify that we can't it there anymore... 

471 registry.disassociate(tag1, [ref1]) 

472 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=tag1)) 

473 # ...but we can still find ref2 in tag1, and ref1 in the run. 

474 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run1), ref1) 

475 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

476 collections = set(registry.queryCollections()) 

477 self.assertEqual(collections, {run1, run2, tag1}) 

478 # Associate both refs into tag1 again; ref2 is already there, but that 

479 # should be a harmless no-op. 

480 registry.associate(tag1, [ref1, ref2]) 

481 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

482 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

483 # Get a different dataset (from a different run) that has the same 

484 # dataset type and data ID as ref2. 

485 ref2b = registry.findDataset(datasetType, dataId2, collections=run2) 

486 self.assertNotEqual(ref2, ref2b) 

487 # Attempting to associate that into tag1 should be an error. 

488 with self.assertRaises(ConflictingDefinitionError): 

489 registry.associate(tag1, [ref2b]) 

490 # That error shouldn't have messed up what we had before. 

491 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

492 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

493 # Attempt to associate the conflicting dataset again, this time with 

494 # a dataset that isn't in the collection and won't cause a conflict. 

495 # Should also fail without modifying anything. 

496 dataId3 = {"instrument": "Cam1", "detector": 3} 

497 ref3 = registry.findDataset(datasetType, dataId3, collections=run1) 

498 with self.assertRaises(ConflictingDefinitionError): 

499 registry.associate(tag1, [ref3, ref2b]) 

500 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

501 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

502 self.assertIsNone(registry.findDataset(datasetType, dataId3, collections=tag1)) 

503 # Register a chained collection that searches: 

504 # 1. 'tag1' 

505 # 2. 'run1', but only for the permaflat dataset 

506 # 3. 'run2' 

507 chain1 = "chain1" 

508 registry.registerCollection(chain1, type=CollectionType.CHAINED) 

509 self.assertIs(registry.getCollectionType(chain1), CollectionType.CHAINED) 

510 # Chained collection exists, but has no collections in it. 

511 self.assertFalse(registry.getCollectionChain(chain1)) 

512 # If we query for all collections, we should get the chained collection 

513 # only if we don't ask to flatten it (i.e. yield only its children). 

514 self.assertEqual(set(registry.queryCollections(flattenChains=False)), {tag1, run1, run2, chain1}) 

515 self.assertEqual(set(registry.queryCollections(flattenChains=True)), {tag1, run1, run2}) 

516 # Attempt to set its child collections to something circular; that 

517 # should fail. 

518 with self.assertRaises(ValueError): 

519 registry.setCollectionChain(chain1, [tag1, chain1]) 

520 # Add the child collections. 

521 registry.setCollectionChain(chain1, [tag1, (run1, "permaflat"), run2]) 

522 self.assertEqual( 

523 list(registry.getCollectionChain(chain1)), 

524 [(tag1, DatasetTypeRestriction.any), 

525 (run1, DatasetTypeRestriction.fromExpression("permaflat")), 

526 (run2, DatasetTypeRestriction.any)] 

527 ) 

528 # Searching for dataId1 or dataId2 in the chain should return ref1 and 

529 # ref2, because both are in tag1. 

530 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain1), ref1) 

531 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain1), ref2) 

532 # Now disassociate ref2 from tag1. The search (for permabias) with 

533 # dataId2 in chain1 should then: 

534 # 1. not find it in tag1 

535 # 2. not look in tag2, because it's restricted to permaflat here 

536 # 3. find a different dataset in run2 

537 registry.disassociate(tag1, [ref2]) 

538 ref2b = registry.findDataset(datasetType, dataId2, collections=chain1) 

539 self.assertNotEqual(ref2b, ref2) 

540 self.assertEqual(ref2b, registry.findDataset(datasetType, dataId2, collections=run2)) 

541 # Look in the chain for a permaflat that is in run1; should get the 

542 # same ref as if we'd searched run1 directly. 

543 dataId3 = {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"} 

544 self.assertEqual(registry.findDataset("permaflat", dataId3, collections=chain1), 

545 registry.findDataset("permaflat", dataId3, collections=run1),) 

546 # Define a new chain so we can test recursive chains. 

547 chain2 = "chain2" 

548 registry.registerCollection(chain2, type=CollectionType.CHAINED) 

549 registry.setCollectionChain(chain2, [(run2, "permabias"), chain1]) 

550 # Query for collections matching a regex. 

551 self.assertCountEqual( 

552 list(registry.queryCollections(re.compile("imported_."), flattenChains=False)), 

553 ["imported_r", "imported_g"] 

554 ) 

555 # Query for collections matching a regex or an explicit str. 

556 self.assertCountEqual( 

557 list(registry.queryCollections([re.compile("imported_."), "chain1"], flattenChains=False)), 

558 ["imported_r", "imported_g", "chain1"] 

559 ) 

560 # Search for permabias with dataId1 should find it via tag1 in chain2, 

561 # recursing, because is not in run1. 

562 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=run2)) 

563 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain2), ref1) 

564 # Search for permabias with dataId2 should find it in run2 (ref2b). 

565 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain2), ref2b) 

566 # Search for a permaflat that is in run2. That should not be found 

567 # at the front of chain2, because of the restriction to permabias 

568 # on run2 there, but it should be found in at the end of chain1. 

569 dataId4 = {"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-R2"} 

570 ref4 = registry.findDataset("permaflat", dataId4, collections=run2) 

571 self.assertIsNotNone(ref4) 

572 self.assertEqual(ref4, registry.findDataset("permaflat", dataId4, collections=chain2)) 

573 # Deleting a collection that's part of a CHAINED collection is not 

574 # allowed, and is exception-safe. 

575 with self.assertRaises(Exception): 

576 registry.removeCollection(run2) 

577 self.assertEqual(registry.getCollectionType(run2), CollectionType.RUN) 

578 with self.assertRaises(Exception): 

579 registry.removeCollection(chain1) 

580 self.assertEqual(registry.getCollectionType(chain1), CollectionType.CHAINED) 

581 # Actually remove chain2, test that it's gone by asking for its type. 

582 registry.removeCollection(chain2) 

583 with self.assertRaises(MissingCollectionError): 

584 registry.getCollectionType(chain2) 

585 # Actually remove run2 and chain1, which should work now. 

586 registry.removeCollection(chain1) 

587 registry.removeCollection(run2) 

588 with self.assertRaises(MissingCollectionError): 

589 registry.getCollectionType(run2) 

590 with self.assertRaises(MissingCollectionError): 

591 registry.getCollectionType(chain1) 

592 # Remove tag1 as well, just to test that we can remove TAGGED 

593 # collections. 

594 registry.removeCollection(tag1) 

595 with self.assertRaises(MissingCollectionError): 

596 registry.getCollectionType(tag1) 

597 

598 def testBasicTransaction(self): 

599 """Test that all operations within a single transaction block are 

600 rolled back if an exception propagates out of the block. 

601 """ 

602 registry = self.makeRegistry() 

603 storageClass = StorageClass("testDatasetType") 

604 registry.storageClasses.registerStorageClass(storageClass) 

605 with registry.transaction(): 

606 registry.insertDimensionData("instrument", {"name": "Cam1", "class_name": "A"}) 

607 with self.assertRaises(ValueError): 

608 with registry.transaction(): 

609 registry.insertDimensionData("instrument", {"name": "Cam2"}) 

610 raise ValueError("Oops, something went wrong") 

611 # Cam1 should exist 

612 self.assertEqual(registry.expandDataId(instrument="Cam1").records["instrument"].class_name, "A") 

613 # But Cam2 and Cam3 should both not exist 

614 with self.assertRaises(LookupError): 

615 registry.expandDataId(instrument="Cam2") 

616 with self.assertRaises(LookupError): 

617 registry.expandDataId(instrument="Cam3") 

618 

619 def testNestedTransaction(self): 

620 """Test that operations within a transaction block are not rolled back 

621 if an exception propagates out of an inner transaction block and is 

622 then caught. 

623 """ 

624 registry = self.makeRegistry() 

625 dimension = registry.dimensions["instrument"] 

626 dataId1 = {"instrument": "DummyCam"} 

627 dataId2 = {"instrument": "DummyCam2"} 

628 checkpointReached = False 

629 with registry.transaction(): 

630 # This should be added and (ultimately) committed. 

631 registry.insertDimensionData(dimension, dataId1) 

632 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

633 with registry.transaction(): 

634 # This does not conflict, and should succeed (but not 

635 # be committed). 

636 registry.insertDimensionData(dimension, dataId2) 

637 checkpointReached = True 

638 # This should conflict and raise, triggerring a rollback 

639 # of the previous insertion within the same transaction 

640 # context, but not the original insertion in the outer 

641 # block. 

642 registry.insertDimensionData(dimension, dataId1) 

643 self.assertTrue(checkpointReached) 

644 self.assertIsNotNone(registry.expandDataId(dataId1, graph=dimension.graph)) 

645 with self.assertRaises(LookupError): 

646 registry.expandDataId(dataId2, graph=dimension.graph) 

647 

648 def testInstrumentDimensions(self): 

649 """Test queries involving only instrument dimensions, with no joins to 

650 skymap.""" 

651 registry = self.makeRegistry() 

652 

653 # need a bunch of dimensions and datasets for test 

654 registry.insertDimensionData( 

655 "instrument", 

656 dict(name="DummyCam", visit_max=25, exposure_max=300, detector_max=6) 

657 ) 

658 registry.insertDimensionData( 

659 "physical_filter", 

660 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

661 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

662 ) 

663 registry.insertDimensionData( 

664 "detector", 

665 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in range(1, 6)] 

666 ) 

667 registry.insertDimensionData( 

668 "visit_system", 

669 dict(instrument="DummyCam", id=1, name="default"), 

670 ) 

671 registry.insertDimensionData( 

672 "visit", 

673 dict(instrument="DummyCam", id=10, name="ten", physical_filter="dummy_i", visit_system=1), 

674 dict(instrument="DummyCam", id=11, name="eleven", physical_filter="dummy_r", visit_system=1), 

675 dict(instrument="DummyCam", id=20, name="twelve", physical_filter="dummy_r", visit_system=1), 

676 ) 

677 registry.insertDimensionData( 

678 "exposure", 

679 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i"), 

680 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i"), 

681 dict(instrument="DummyCam", id=110, name="110", physical_filter="dummy_r"), 

682 dict(instrument="DummyCam", id=111, name="111", physical_filter="dummy_r"), 

683 dict(instrument="DummyCam", id=200, name="200", physical_filter="dummy_r"), 

684 dict(instrument="DummyCam", id=201, name="201", physical_filter="dummy_r"), 

685 ) 

686 registry.insertDimensionData( 

687 "visit_definition", 

688 dict(instrument="DummyCam", exposure=100, visit_system=1, visit=10), 

689 dict(instrument="DummyCam", exposure=101, visit_system=1, visit=10), 

690 dict(instrument="DummyCam", exposure=110, visit_system=1, visit=11), 

691 dict(instrument="DummyCam", exposure=111, visit_system=1, visit=11), 

692 dict(instrument="DummyCam", exposure=200, visit_system=1, visit=20), 

693 dict(instrument="DummyCam", exposure=201, visit_system=1, visit=20), 

694 ) 

695 # dataset types 

696 run1 = "test1_r" 

697 run2 = "test2_r" 

698 tagged2 = "test2_t" 

699 registry.registerRun(run1) 

700 registry.registerRun(run2) 

701 registry.registerCollection(tagged2) 

702 storageClass = StorageClass("testDataset") 

703 registry.storageClasses.registerStorageClass(storageClass) 

704 rawType = DatasetType(name="RAW", 

705 dimensions=registry.dimensions.extract(("instrument", "exposure", "detector")), 

706 storageClass=storageClass) 

707 registry.registerDatasetType(rawType) 

708 calexpType = DatasetType(name="CALEXP", 

709 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

710 storageClass=storageClass) 

711 registry.registerDatasetType(calexpType) 

712 

713 # add pre-existing datasets 

714 for exposure in (100, 101, 110, 111): 

715 for detector in (1, 2, 3): 

716 # note that only 3 of 5 detectors have datasets 

717 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

718 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run1) 

719 # exposures 100 and 101 appear in both run1 and tagged2. 

720 # 100 has different datasets in the different collections 

721 # 101 has the same dataset in both collections. 

722 if exposure == 100: 

723 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

724 if exposure in (100, 101): 

725 registry.associate(tagged2, [ref]) 

726 # Add pre-existing datasets to tagged2. 

727 for exposure in (200, 201): 

728 for detector in (3, 4, 5): 

729 # note that only 3 of 5 detectors have datasets 

730 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

731 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

732 registry.associate(tagged2, [ref]) 

733 

734 dimensions = DimensionGraph( 

735 registry.dimensions, 

736 dimensions=(rawType.dimensions.required | calexpType.dimensions.required) 

737 ) 

738 # Test that single dim string works as well as list of str 

739 rows = list(registry.queryDimensions("visit", datasets=rawType, collections=run1, expand=True)) 

740 rowsI = list(registry.queryDimensions(["visit"], datasets=rawType, collections=run1, expand=True)) 

741 self.assertEqual(rows, rowsI) 

742 # with empty expression 

743 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, expand=True)) 

744 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

745 for dataId in rows: 

746 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

747 packer1 = registry.dimensions.makePacker("visit_detector", dataId) 

748 packer2 = registry.dimensions.makePacker("exposure_detector", dataId) 

749 self.assertEqual(packer1.unpack(packer1.pack(dataId)), 

750 DataCoordinate.standardize(dataId, graph=packer1.dimensions)) 

751 self.assertEqual(packer2.unpack(packer2.pack(dataId)), 

752 DataCoordinate.standardize(dataId, graph=packer2.dimensions)) 

753 self.assertNotEqual(packer1.pack(dataId), packer2.pack(dataId)) 

754 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

755 (100, 101, 110, 111)) 

756 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11)) 

757 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

758 

759 # second collection 

760 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=tagged2)) 

761 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

762 for dataId in rows: 

763 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

764 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

765 (100, 101, 200, 201)) 

766 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 20)) 

767 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

768 

769 # with two input datasets 

770 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=[run1, tagged2])) 

771 self.assertEqual(len(set(rows)), 6*3) # 6 exposures times 3 detectors; set needed to de-dupe 

772 for dataId in rows: 

773 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

774 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

775 (100, 101, 110, 111, 200, 201)) 

776 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11, 20)) 

777 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

778 

779 # limit to single visit 

780 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

781 where="visit = 10")) 

782 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

783 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

784 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

785 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

786 

787 # more limiting expression, using link names instead of Table.column 

788 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

789 where="visit = 10 and detector > 1")) 

790 self.assertEqual(len(rows), 2*2) # 2 exposures times 2 detectors 

791 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

792 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

793 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (2, 3)) 

794 

795 # expression excludes everything 

796 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

797 where="visit > 1000")) 

798 self.assertEqual(len(rows), 0) 

799 

800 # Selecting by physical_filter, this is not in the dimensions, but it 

801 # is a part of the full expression so it should work too. 

802 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

803 where="physical_filter = 'dummy_r'")) 

804 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

805 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (110, 111)) 

806 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (11,)) 

807 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

808 

809 def testSkyMapDimensions(self): 

810 """Tests involving only skymap dimensions, no joins to instrument.""" 

811 registry = self.makeRegistry() 

812 

813 # need a bunch of dimensions and datasets for test, we want 

814 # "abstract_filter" in the test so also have to add physical_filter 

815 # dimensions 

816 registry.insertDimensionData( 

817 "instrument", 

818 dict(instrument="DummyCam") 

819 ) 

820 registry.insertDimensionData( 

821 "physical_filter", 

822 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

823 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

824 ) 

825 registry.insertDimensionData( 

826 "skymap", 

827 dict(name="DummyMap", hash="sha!".encode("utf8")) 

828 ) 

829 for tract in range(10): 

830 registry.insertDimensionData("tract", dict(skymap="DummyMap", id=tract)) 

831 registry.insertDimensionData( 

832 "patch", 

833 *[dict(skymap="DummyMap", tract=tract, id=patch, cell_x=0, cell_y=0) 

834 for patch in range(10)] 

835 ) 

836 

837 # dataset types 

838 run = "test" 

839 registry.registerRun(run) 

840 storageClass = StorageClass("testDataset") 

841 registry.storageClasses.registerStorageClass(storageClass) 

842 calexpType = DatasetType(name="deepCoadd_calexp", 

843 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

844 "abstract_filter")), 

845 storageClass=storageClass) 

846 registry.registerDatasetType(calexpType) 

847 mergeType = DatasetType(name="deepCoadd_mergeDet", 

848 dimensions=registry.dimensions.extract(("skymap", "tract", "patch")), 

849 storageClass=storageClass) 

850 registry.registerDatasetType(mergeType) 

851 measType = DatasetType(name="deepCoadd_meas", 

852 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

853 "abstract_filter")), 

854 storageClass=storageClass) 

855 registry.registerDatasetType(measType) 

856 

857 dimensions = DimensionGraph( 

858 registry.dimensions, 

859 dimensions=(calexpType.dimensions.required | mergeType.dimensions.required 

860 | measType.dimensions.required) 

861 ) 

862 

863 # add pre-existing datasets 

864 for tract in (1, 3, 5): 

865 for patch in (2, 4, 6, 7): 

866 dataId = dict(skymap="DummyMap", tract=tract, patch=patch) 

867 registry.insertDatasets(mergeType, dataIds=[dataId], run=run) 

868 for aFilter in ("i", "r"): 

869 dataId = dict(skymap="DummyMap", tract=tract, patch=patch, abstract_filter=aFilter) 

870 registry.insertDatasets(calexpType, dataIds=[dataId], run=run) 

871 

872 # with empty expression 

873 rows = list(registry.queryDimensions(dimensions, 

874 datasets=[calexpType, mergeType], collections=run)) 

875 self.assertEqual(len(rows), 3*4*2) # 4 tracts x 4 patches x 2 filters 

876 for dataId in rows: 

877 self.assertCountEqual(dataId.keys(), ("skymap", "tract", "patch", "abstract_filter")) 

878 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

879 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

880 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

881 

882 # limit to 2 tracts and 2 patches 

883 rows = list(registry.queryDimensions(dimensions, 

884 datasets=[calexpType, mergeType], collections=run, 

885 where="tract IN (1, 5) AND patch IN (2, 7)")) 

886 self.assertEqual(len(rows), 2*2*2) # 2 tracts x 2 patches x 2 filters 

887 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 5)) 

888 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 7)) 

889 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

890 

891 # limit to single filter 

892 rows = list(registry.queryDimensions(dimensions, 

893 datasets=[calexpType, mergeType], collections=run, 

894 where="abstract_filter = 'i'")) 

895 self.assertEqual(len(rows), 3*4*1) # 4 tracts x 4 patches x 2 filters 

896 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

897 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

898 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i",)) 

899 

900 # expression excludes everything, specifying non-existing skymap is 

901 # not a fatal error, it's operator error 

902 rows = list(registry.queryDimensions(dimensions, 

903 datasets=[calexpType, mergeType], collections=run, 

904 where="skymap = 'Mars'")) 

905 self.assertEqual(len(rows), 0) 

906 

907 def testSpatialMatch(self): 

908 """Test involving spatial match using join tables. 

909 

910 Note that realistic test needs a reasonably-defined skypix and regions 

911 in registry tables which is hard to implement in this simple test. 

912 So we do not actually fill registry with any data and all queries will 

913 return empty result, but this is still useful for coverage of the code 

914 that generates query. 

915 """ 

916 registry = self.makeRegistry() 

917 

918 # dataset types 

919 collection = "test" 

920 registry.registerRun(name=collection) 

921 storageClass = StorageClass("testDataset") 

922 registry.storageClasses.registerStorageClass(storageClass) 

923 

924 calexpType = DatasetType(name="CALEXP", 

925 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

926 storageClass=storageClass) 

927 registry.registerDatasetType(calexpType) 

928 

929 coaddType = DatasetType(name="deepCoadd_calexp", 

930 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

931 "abstract_filter")), 

932 storageClass=storageClass) 

933 registry.registerDatasetType(coaddType) 

934 

935 dimensions = DimensionGraph( 

936 registry.dimensions, 

937 dimensions=(calexpType.dimensions.required | coaddType.dimensions.required) 

938 ) 

939 

940 # without data this should run OK but return empty set 

941 rows = list(registry.queryDimensions(dimensions, datasets=calexpType, collections=collection)) 

942 self.assertEqual(len(rows), 0) 

943 

944 def testCalibrationLabelIndirection(self): 

945 """Test that we can look up datasets with calibration_label dimensions 

946 from a data ID with exposure dimensions. 

947 """ 

948 

949 def _dt(iso_string): 

950 return astropy.time.Time(iso_string, format="iso", scale="utc") 

951 

952 registry = self.makeRegistry() 

953 

954 flat = DatasetType( 

955 "flat", 

956 registry.dimensions.extract( 

957 ["instrument", "detector", "physical_filter", "calibration_label"] 

958 ), 

959 "ImageU" 

960 ) 

961 registry.registerDatasetType(flat) 

962 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

963 registry.insertDimensionData( 

964 "physical_filter", 

965 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

966 ) 

967 registry.insertDimensionData( 

968 "detector", 

969 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in (1, 2, 3, 4, 5)] 

970 ) 

971 registry.insertDimensionData( 

972 "exposure", 

973 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i", 

974 datetime_begin=_dt("2005-12-15 02:00:00"), datetime_end=_dt("2005-12-15 03:00:00")), 

975 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i", 

976 datetime_begin=_dt("2005-12-16 02:00:00"), datetime_end=_dt("2005-12-16 03:00:00")), 

977 ) 

978 registry.insertDimensionData( 

979 "calibration_label", 

980 dict(instrument="DummyCam", name="first_night", 

981 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-15 04:00:00")), 

982 dict(instrument="DummyCam", name="second_night", 

983 datetime_begin=_dt("2005-12-16 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

984 dict(instrument="DummyCam", name="both_nights", 

985 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

986 ) 

987 # Different flats for different nights for detectors 1-3 in first 

988 # collection. 

989 run1 = "calibs1" 

990 registry.registerRun(run1) 

991 for detector in (1, 2, 3): 

992 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="first_night", 

993 physical_filter="dummy_i", detector=detector)], 

994 run=run1) 

995 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="second_night", 

996 physical_filter="dummy_i", detector=detector)], 

997 run=run1) 

998 # The same flat for both nights for detectors 3-5 (so detector 3 has 

999 # multiple valid flats) in second collection. 

1000 run2 = "calib2" 

1001 registry.registerRun(run2) 

1002 for detector in (3, 4, 5): 

1003 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="both_nights", 

1004 physical_filter="dummy_i", detector=detector)], 

1005 run=run2) 

1006 # Perform queries for individual exposure+detector combinations, which 

1007 # should always return exactly one flat. 

1008 for exposure in (100, 101): 

1009 for detector in (1, 2, 3): 

1010 with self.subTest(exposure=exposure, detector=detector): 

1011 rows = list(registry.queryDatasets("flat", collections=[run1], 

1012 instrument="DummyCam", 

1013 exposure=exposure, 

1014 detector=detector)) 

1015 self.assertEqual(len(rows), 1) 

1016 for detector in (3, 4, 5): 

1017 with self.subTest(exposure=exposure, detector=detector): 

1018 rows = registry.queryDatasets("flat", collections=[run2], 

1019 instrument="DummyCam", 

1020 exposure=exposure, 

1021 detector=detector) 

1022 self.assertEqual(len(list(rows)), 1) 

1023 for detector in (1, 2, 4, 5): 

1024 with self.subTest(exposure=exposure, detector=detector): 

1025 rows = registry.queryDatasets("flat", collections=[run1, run2], 

1026 instrument="DummyCam", 

1027 exposure=exposure, 

1028 detector=detector) 

1029 self.assertEqual(len(list(rows)), 1) 

1030 for detector in (3,): 

1031 with self.subTest(exposure=exposure, detector=detector): 

1032 rows = registry.queryDatasets("flat", collections=[run1, run2], 

1033 instrument="DummyCam", 

1034 exposure=exposure, 

1035 detector=detector) 

1036 self.assertEqual(len(list(rows)), 2) 

1037 

1038 def testAbstractFilterQuery(self): 

1039 """Test that we can run a query that just lists the known 

1040 abstract_filters. This is tricky because abstract_filter is 

1041 backed by a query against physical_filter. 

1042 """ 

1043 registry = self.makeRegistry() 

1044 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

1045 registry.insertDimensionData( 

1046 "physical_filter", 

1047 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

1048 dict(instrument="DummyCam", name="dummy_i2", abstract_filter="i"), 

1049 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

1050 ) 

1051 rows = list(registry.queryDimensions(["abstract_filter"])) 

1052 self.assertCountEqual( 

1053 rows, 

1054 [DataCoordinate.standardize(abstract_filter="i", universe=registry.dimensions), 

1055 DataCoordinate.standardize(abstract_filter="r", universe=registry.dimensions)] 

1056 ) 

1057 

1058 def testAttributeManager(self): 

1059 """Test basic functionality of attribute manager. 

1060 """ 

1061 # number of attributes with schema versions in a fresh database, 

1062 # 6 managers with 3 records per manager 

1063 VERSION_COUNT = 6 * 3 

1064 

1065 registry = self.makeRegistry() 

1066 attributes = registry._attributes 

1067 

1068 # check what get() returns for non-existing key 

1069 self.assertIsNone(attributes.get("attr")) 

1070 self.assertEqual(attributes.get("attr", ""), "") 

1071 self.assertEqual(attributes.get("attr", "Value"), "Value") 

1072 self.assertEqual(len(list(attributes.items())), VERSION_COUNT) 

1073 

1074 # cannot store empty key or value 

1075 with self.assertRaises(ValueError): 

1076 attributes.set("", "value") 

1077 with self.assertRaises(ValueError): 

1078 attributes.set("attr", "") 

1079 

1080 # set value of non-existing key 

1081 attributes.set("attr", "value") 

1082 self.assertEqual(len(list(attributes.items())), VERSION_COUNT + 1) 

1083 self.assertEqual(attributes.get("attr"), "value") 

1084 

1085 # update value of existing key 

1086 with self.assertRaises(ButlerAttributeExistsError): 

1087 attributes.set("attr", "value2") 

1088 

1089 attributes.set("attr", "value2", force=True) 

1090 self.assertEqual(len(list(attributes.items())), VERSION_COUNT + 1) 

1091 self.assertEqual(attributes.get("attr"), "value2") 

1092 

1093 # delete existing key 

1094 self.assertTrue(attributes.delete("attr")) 

1095 self.assertEqual(len(list(attributes.items())), VERSION_COUNT) 

1096 

1097 # delete non-existing key 

1098 self.assertFalse(attributes.delete("non-attr")) 

1099 

1100 # store bunch of keys and get the list back 

1101 data = [ 

1102 ("version.core", "1.2.3"), 

1103 ("version.dimensions", "3.2.1"), 

1104 ("config.managers.opaque", "ByNameOpaqueTableStorageManager"), 

1105 ] 

1106 for key, value in data: 

1107 attributes.set(key, value) 

1108 items = dict(attributes.items()) 

1109 for key, value in data: 

1110 self.assertEqual(items[key], value)