Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["RegistryTests"] 

24 

25from abc import ABC, abstractmethod 

26import os 

27import re 

28 

29import astropy.time 

30import sqlalchemy 

31from typing import Optional 

32 

33from ...core import ( 

34 DataCoordinate, 

35 DatasetRef, 

36 DatasetType, 

37 DimensionGraph, 

38 NamedValueSet, 

39 StorageClass, 

40 ddl, 

41 YamlRepoImportBackend 

42) 

43from .._registry import ( 

44 CollectionType, 

45 ConflictingDefinitionError, 

46 InconsistentDataIdError, 

47 Registry, 

48 RegistryConfig, 

49) 

50from ..wildcards import DatasetTypeRestriction 

51from ..interfaces import MissingCollectionError, ButlerAttributeExistsError 

52 

53 

54class RegistryTests(ABC): 

55 """Generic tests for the `Registry` class that can be subclassed to 

56 generate tests for different configurations. 

57 """ 

58 

59 collectionsManager: Optional[str] = None 

60 """Name of the collections manager class, if subclass provides value for 

61 this member then it overrides name specified in default configuration 

62 (`str`). 

63 """ 

64 

65 @classmethod 

66 @abstractmethod 

67 def getDataDir(cls) -> str: 

68 """Return the root directory containing test data YAML files. 

69 """ 

70 raise NotImplementedError() 

71 

72 def makeRegistryConfig(self) -> RegistryConfig: 

73 """Create RegistryConfig used to create a registry. 

74 

75 This method should be called by a subclass from `makeRegistry`. 

76 Returned instance will be pre-configured based on the values of class 

77 members, and default-configured for all other parametrs. Subclasses 

78 that need default configuration should just instantiate 

79 `RegistryConfig` directly. 

80 """ 

81 config = RegistryConfig() 

82 if self.collectionsManager: 

83 config["managers"]["collections"] = self.collectionsManager 

84 return config 

85 

86 @abstractmethod 

87 def makeRegistry(self) -> Registry: 

88 """Return the Registry instance to be tested. 

89 """ 

90 raise NotImplementedError() 

91 

92 def loadData(self, registry: Registry, filename: str): 

93 """Load registry test data from ``getDataDir/<filename>``, 

94 which should be a YAML import/export file. 

95 """ 

96 with open(os.path.join(self.getDataDir(), filename), 'r') as stream: 

97 backend = YamlRepoImportBackend(stream, registry) 

98 backend.register() 

99 backend.load(datastore=None) 

100 

101 def assertRowCount(self, registry: Registry, table: str, count: int): 

102 """Check the number of rows in table. 

103 """ 

104 # TODO: all tests that rely on this method should be rewritten, as it 

105 # needs to depend on Registry implementation details to have any chance 

106 # of working. 

107 sql = sqlalchemy.sql.select( 

108 [sqlalchemy.sql.func.count()] 

109 ).select_from( 

110 getattr(registry._tables, table) 

111 ) 

112 self.assertEqual(registry._db.query(sql).scalar(), count) 

113 

114 def testOpaque(self): 

115 """Tests for `Registry.registerOpaqueTable`, 

116 `Registry.insertOpaqueData`, `Registry.fetchOpaqueData`, and 

117 `Registry.deleteOpaqueData`. 

118 """ 

119 registry = self.makeRegistry() 

120 table = "opaque_table_for_testing" 

121 registry.registerOpaqueTable( 

122 table, 

123 spec=ddl.TableSpec( 

124 fields=[ 

125 ddl.FieldSpec("id", dtype=sqlalchemy.BigInteger, primaryKey=True), 

126 ddl.FieldSpec("name", dtype=sqlalchemy.String, length=16, nullable=False), 

127 ddl.FieldSpec("count", dtype=sqlalchemy.SmallInteger, nullable=True), 

128 ], 

129 ) 

130 ) 

131 rows = [ 

132 {"id": 1, "name": "one", "count": None}, 

133 {"id": 2, "name": "two", "count": 5}, 

134 {"id": 3, "name": "three", "count": 6}, 

135 ] 

136 registry.insertOpaqueData(table, *rows) 

137 self.assertCountEqual(rows, list(registry.fetchOpaqueData(table))) 

138 self.assertEqual(rows[0:1], list(registry.fetchOpaqueData(table, id=1))) 

139 self.assertEqual(rows[1:2], list(registry.fetchOpaqueData(table, name="two"))) 

140 self.assertEqual([], list(registry.fetchOpaqueData(table, id=1, name="two"))) 

141 registry.deleteOpaqueData(table, id=3) 

142 self.assertCountEqual(rows[:2], list(registry.fetchOpaqueData(table))) 

143 registry.deleteOpaqueData(table) 

144 self.assertEqual([], list(registry.fetchOpaqueData(table))) 

145 

146 def testDatasetType(self): 

147 """Tests for `Registry.registerDatasetType` and 

148 `Registry.getDatasetType`. 

149 """ 

150 registry = self.makeRegistry() 

151 # Check valid insert 

152 datasetTypeName = "test" 

153 storageClass = StorageClass("testDatasetType") 

154 registry.storageClasses.registerStorageClass(storageClass) 

155 dimensions = registry.dimensions.extract(("instrument", "visit")) 

156 differentDimensions = registry.dimensions.extract(("instrument", "patch")) 

157 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

158 # Inserting for the first time should return True 

159 self.assertTrue(registry.registerDatasetType(inDatasetType)) 

160 outDatasetType1 = registry.getDatasetType(datasetTypeName) 

161 self.assertEqual(outDatasetType1, inDatasetType) 

162 

163 # Re-inserting should work 

164 self.assertFalse(registry.registerDatasetType(inDatasetType)) 

165 # Except when they are not identical 

166 with self.assertRaises(ConflictingDefinitionError): 

167 nonIdenticalDatasetType = DatasetType(datasetTypeName, differentDimensions, storageClass) 

168 registry.registerDatasetType(nonIdenticalDatasetType) 

169 

170 # Template can be None 

171 datasetTypeName = "testNoneTemplate" 

172 storageClass = StorageClass("testDatasetType2") 

173 registry.storageClasses.registerStorageClass(storageClass) 

174 dimensions = registry.dimensions.extract(("instrument", "visit")) 

175 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

176 registry.registerDatasetType(inDatasetType) 

177 outDatasetType2 = registry.getDatasetType(datasetTypeName) 

178 self.assertEqual(outDatasetType2, inDatasetType) 

179 

180 allTypes = set(registry.queryDatasetTypes()) 

181 self.assertEqual(allTypes, {outDatasetType1, outDatasetType2}) 

182 

183 def testDimensions(self): 

184 """Tests for `Registry.insertDimensionData`, 

185 `Registry.syncDimensionData`, and `Registry.expandDataId`. 

186 """ 

187 registry = self.makeRegistry() 

188 dimensionName = "instrument" 

189 dimension = registry.dimensions[dimensionName] 

190 dimensionValue = {"name": "DummyCam", "visit_max": 10, "exposure_max": 10, "detector_max": 2, 

191 "class_name": "lsst.obs.base.Instrument"} 

192 registry.insertDimensionData(dimensionName, dimensionValue) 

193 # Inserting the same value twice should fail 

194 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

195 registry.insertDimensionData(dimensionName, dimensionValue) 

196 # expandDataId should retrieve the record we just inserted 

197 self.assertEqual( 

198 registry.expandDataId( 

199 instrument="DummyCam", 

200 graph=dimension.graph 

201 ).records[dimensionName].toDict(), 

202 dimensionValue 

203 ) 

204 # expandDataId should raise if there is no record with the given ID. 

205 with self.assertRaises(LookupError): 

206 registry.expandDataId({"instrument": "Unknown"}, graph=dimension.graph) 

207 # abstract_filter doesn't have a table; insert should fail. 

208 with self.assertRaises(TypeError): 

209 registry.insertDimensionData("abstract_filter", {"abstract_filter": "i"}) 

210 dimensionName2 = "physical_filter" 

211 dimension2 = registry.dimensions[dimensionName2] 

212 dimensionValue2 = {"name": "DummyCam_i", "abstract_filter": "i"} 

213 # Missing required dependency ("instrument") should fail 

214 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

215 registry.insertDimensionData(dimensionName2, dimensionValue2) 

216 # Adding required dependency should fix the failure 

217 dimensionValue2["instrument"] = "DummyCam" 

218 registry.insertDimensionData(dimensionName2, dimensionValue2) 

219 # expandDataId should retrieve the record we just inserted. 

220 self.assertEqual( 

221 registry.expandDataId( 

222 instrument="DummyCam", physical_filter="DummyCam_i", 

223 graph=dimension2.graph 

224 ).records[dimensionName2].toDict(), 

225 dimensionValue2 

226 ) 

227 # Use syncDimensionData to insert a new record successfully. 

228 dimensionName3 = "detector" 

229 dimensionValue3 = {"instrument": "DummyCam", "id": 1, "full_name": "one", 

230 "name_in_raft": "zero", "purpose": "SCIENCE"} 

231 self.assertTrue(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

232 # Sync that again. Note that one field ("raft") is NULL, and that 

233 # should be okay. 

234 self.assertFalse(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

235 # Now try that sync with the same primary key but a different value. 

236 # This should fail. 

237 with self.assertRaises(ConflictingDefinitionError): 

238 registry.syncDimensionData( 

239 dimensionName3, 

240 {"instrument": "DummyCam", "id": 1, "full_name": "one", 

241 "name_in_raft": "four", "purpose": "SCIENCE"} 

242 ) 

243 

244 def testDataIdRelationships(self): 

245 """Test that `Registry.expandDataId` raises an exception when the given 

246 keys are inconsistent. 

247 """ 

248 registry = self.makeRegistry() 

249 self.loadData(registry, "base.yaml") 

250 # Insert a few more dimension records for the next test. 

251 registry.insertDimensionData( 

252 "exposure", 

253 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G"}, 

254 ) 

255 registry.insertDimensionData( 

256 "exposure", 

257 {"instrument": "Cam1", "id": 2, "name": "two", "physical_filter": "Cam1-G"}, 

258 ) 

259 registry.insertDimensionData( 

260 "visit_system", 

261 {"instrument": "Cam1", "id": 0, "name": "one-to-one"}, 

262 ) 

263 registry.insertDimensionData( 

264 "visit", 

265 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G", "visit_system": 0}, 

266 ) 

267 registry.insertDimensionData( 

268 "visit_definition", 

269 {"instrument": "Cam1", "visit": 1, "exposure": 1, "visit_system": 0}, 

270 ) 

271 with self.assertRaises(InconsistentDataIdError): 

272 registry.expandDataId( 

273 {"instrument": "Cam1", "visit": 1, "exposure": 2}, 

274 ) 

275 

276 def testDataset(self): 

277 """Basic tests for `Registry.insertDatasets`, `Registry.getDataset`, 

278 and `Registry.removeDatasets`. 

279 """ 

280 registry = self.makeRegistry() 

281 self.loadData(registry, "base.yaml") 

282 run = "test" 

283 registry.registerRun(run) 

284 datasetType = registry.getDatasetType("permabias") 

285 dataId = {"instrument": "Cam1", "detector": 2} 

286 ref, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

287 outRef = registry.getDataset(ref.id) 

288 self.assertIsNotNone(ref.id) 

289 self.assertEqual(ref, outRef) 

290 with self.assertRaises(ConflictingDefinitionError): 

291 registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

292 registry.removeDatasets([ref]) 

293 self.assertIsNone(registry.findDataset(datasetType, dataId, collections=[run])) 

294 

295 def testFindDataset(self): 

296 """Tests for `Registry.findDataset`. 

297 """ 

298 registry = self.makeRegistry() 

299 self.loadData(registry, "base.yaml") 

300 run = "test" 

301 datasetType = registry.getDatasetType("permabias") 

302 dataId = {"instrument": "Cam1", "detector": 4} 

303 registry.registerRun(run) 

304 inputRef, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

305 outputRef = registry.findDataset(datasetType, dataId, collections=[run]) 

306 self.assertEqual(outputRef, inputRef) 

307 # Check that retrieval with invalid dataId raises 

308 with self.assertRaises(LookupError): 

309 dataId = {"instrument": "Cam1"} # no detector 

310 registry.findDataset(datasetType, dataId, collections=run) 

311 # Check that different dataIds match to different datasets 

312 dataId1 = {"instrument": "Cam1", "detector": 1} 

313 inputRef1, = registry.insertDatasets(datasetType, dataIds=[dataId1], run=run) 

314 dataId2 = {"instrument": "Cam1", "detector": 2} 

315 inputRef2, = registry.insertDatasets(datasetType, dataIds=[dataId2], run=run) 

316 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef1) 

317 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef2) 

318 self.assertNotEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef2) 

319 self.assertNotEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef1) 

320 # Check that requesting a non-existing dataId returns None 

321 nonExistingDataId = {"instrument": "Cam1", "detector": 3} 

322 self.assertIsNone(registry.findDataset(datasetType, nonExistingDataId, collections=run)) 

323 

324 def testDatasetTypeComponentQueries(self): 

325 """Test component options when querying for dataset types. 

326 """ 

327 registry = self.makeRegistry() 

328 self.loadData(registry, "base.yaml") 

329 self.loadData(registry, "datasets.yaml") 

330 # Test querying for dataset types with different inputs. 

331 # First query for all dataset types; components should only be included 

332 # when components=True. 

333 self.assertEqual( 

334 {"permabias", "permaflat"}, 

335 NamedValueSet(registry.queryDatasetTypes()).names 

336 ) 

337 self.assertEqual( 

338 {"permabias", "permaflat"}, 

339 NamedValueSet(registry.queryDatasetTypes(components=False)).names 

340 ) 

341 self.assertLess( 

342 {"permabias", "permaflat", "permabias.wcs", "permaflat.photoCalib"}, 

343 NamedValueSet(registry.queryDatasetTypes(components=True)).names 

344 ) 

345 # Use a pattern that can match either parent or components. Again, 

346 # components are only returned if components=True. 

347 self.assertEqual( 

348 {"permabias"}, 

349 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"))).names 

350 ) 

351 self.assertEqual( 

352 {"permabias"}, 

353 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"), components=False)).names 

354 ) 

355 self.assertLess( 

356 {"permabias", "permabias.wcs"}, 

357 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"), components=True)).names 

358 ) 

359 # This pattern matches only a component. In this case we also return 

360 # that component dataset type if components=None. 

361 self.assertEqual( 

362 {"permabias.wcs"}, 

363 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"))).names 

364 ) 

365 self.assertEqual( 

366 set(), 

367 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"), components=False)).names 

368 ) 

369 self.assertEqual( 

370 {"permabias.wcs"}, 

371 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"), components=True)).names 

372 ) 

373 

374 def testComponentLookups(self): 

375 """Test searching for component datasets via their parents. 

376 """ 

377 registry = self.makeRegistry() 

378 self.loadData(registry, "base.yaml") 

379 self.loadData(registry, "datasets.yaml") 

380 # Test getting the child dataset type (which does still exist in the 

381 # Registry), and check for consistency with 

382 # DatasetRef.makeComponentRef. 

383 collection = "imported_g" 

384 parentType = registry.getDatasetType("permabias") 

385 childType = registry.getDatasetType("permabias.wcs") 

386 parentRefResolved = registry.findDataset(parentType, collections=collection, 

387 instrument="Cam1", detector=1) 

388 self.assertIsInstance(parentRefResolved, DatasetRef) 

389 self.assertEqual(childType, parentRefResolved.makeComponentRef("wcs").datasetType) 

390 # Search for a single dataset with findDataset. 

391 childRef1 = registry.findDataset("permabias.wcs", collections=collection, 

392 dataId=parentRefResolved.dataId) 

393 self.assertEqual(childRef1, parentRefResolved.makeComponentRef("wcs")) 

394 # Search for detector data IDs constrained by component dataset 

395 # existence with queryDimensions. 

396 dataIds = set(registry.queryDimensions( 

397 ["detector"], 

398 datasets=["permabias.wcs"], 

399 collections=collection, 

400 expand=False, 

401 )) 

402 self.assertEqual( 

403 dataIds, 

404 { 

405 DataCoordinate.standardize(instrument="Cam1", detector=d, graph=parentType.dimensions) 

406 for d in (1, 2, 3) 

407 } 

408 ) 

409 # Search for multiple datasets of a single type with queryDatasets. 

410 childRefs2 = set(registry.queryDatasets( 

411 "permabias.wcs", 

412 collections=collection, 

413 expand=False, 

414 )) 

415 self.assertEqual( 

416 {ref.unresolved() for ref in childRefs2}, 

417 {DatasetRef(childType, dataId) for dataId in dataIds} 

418 ) 

419 

420 def testCollections(self): 

421 """Tests for registry methods that manage collections. 

422 """ 

423 registry = self.makeRegistry() 

424 self.loadData(registry, "base.yaml") 

425 self.loadData(registry, "datasets.yaml") 

426 run1 = "imported_g" 

427 run2 = "imported_r" 

428 datasetType = "permabias" 

429 # Find some datasets via their run's collection. 

430 dataId1 = {"instrument": "Cam1", "detector": 1} 

431 ref1 = registry.findDataset(datasetType, dataId1, collections=run1) 

432 self.assertIsNotNone(ref1) 

433 dataId2 = {"instrument": "Cam1", "detector": 2} 

434 ref2 = registry.findDataset(datasetType, dataId2, collections=run1) 

435 self.assertIsNotNone(ref2) 

436 # Associate those into a new collection,then look for them there. 

437 tag1 = "tag1" 

438 registry.registerCollection(tag1, type=CollectionType.TAGGED) 

439 registry.associate(tag1, [ref1, ref2]) 

440 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

441 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

442 # Disassociate one and verify that we can't it there anymore... 

443 registry.disassociate(tag1, [ref1]) 

444 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=tag1)) 

445 # ...but we can still find ref2 in tag1, and ref1 in the run. 

446 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run1), ref1) 

447 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

448 collections = set(registry.queryCollections()) 

449 self.assertEqual(collections, {run1, run2, tag1}) 

450 # Associate both refs into tag1 again; ref2 is already there, but that 

451 # should be a harmless no-op. 

452 registry.associate(tag1, [ref1, ref2]) 

453 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

454 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

455 # Get a different dataset (from a different run) that has the same 

456 # dataset type and data ID as ref2. 

457 ref2b = registry.findDataset(datasetType, dataId2, collections=run2) 

458 self.assertNotEqual(ref2, ref2b) 

459 # Attempting to associate that into tag1 should be an error. 

460 with self.assertRaises(ConflictingDefinitionError): 

461 registry.associate(tag1, [ref2b]) 

462 # That error shouldn't have messed up what we had before. 

463 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

464 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

465 # Attempt to associate the conflicting dataset again, this time with 

466 # a dataset that isn't in the collection and won't cause a conflict. 

467 # Should also fail without modifying anything. 

468 dataId3 = {"instrument": "Cam1", "detector": 3} 

469 ref3 = registry.findDataset(datasetType, dataId3, collections=run1) 

470 with self.assertRaises(ConflictingDefinitionError): 

471 registry.associate(tag1, [ref3, ref2b]) 

472 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

473 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

474 self.assertIsNone(registry.findDataset(datasetType, dataId3, collections=tag1)) 

475 # Register a chained collection that searches: 

476 # 1. 'tag1' 

477 # 2. 'run1', but only for the permaflat dataset 

478 # 3. 'run2' 

479 chain1 = "chain1" 

480 registry.registerCollection(chain1, type=CollectionType.CHAINED) 

481 self.assertIs(registry.getCollectionType(chain1), CollectionType.CHAINED) 

482 # Chained collection exists, but has no collections in it. 

483 self.assertFalse(registry.getCollectionChain(chain1)) 

484 # If we query for all collections, we should get the chained collection 

485 # only if we don't ask to flatten it (i.e. yield only its children). 

486 self.assertEqual(set(registry.queryCollections(flattenChains=False)), {tag1, run1, run2, chain1}) 

487 self.assertEqual(set(registry.queryCollections(flattenChains=True)), {tag1, run1, run2}) 

488 # Attempt to set its child collections to something circular; that 

489 # should fail. 

490 with self.assertRaises(ValueError): 

491 registry.setCollectionChain(chain1, [tag1, chain1]) 

492 # Add the child collections. 

493 registry.setCollectionChain(chain1, [tag1, (run1, "permaflat"), run2]) 

494 self.assertEqual( 

495 list(registry.getCollectionChain(chain1)), 

496 [(tag1, DatasetTypeRestriction.any), 

497 (run1, DatasetTypeRestriction.fromExpression("permaflat")), 

498 (run2, DatasetTypeRestriction.any)] 

499 ) 

500 # Searching for dataId1 or dataId2 in the chain should return ref1 and 

501 # ref2, because both are in tag1. 

502 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain1), ref1) 

503 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain1), ref2) 

504 # Now disassociate ref2 from tag1. The search (for permabias) with 

505 # dataId2 in chain1 should then: 

506 # 1. not find it in tag1 

507 # 2. not look in tag2, because it's restricted to permaflat here 

508 # 3. find a different dataset in run2 

509 registry.disassociate(tag1, [ref2]) 

510 ref2b = registry.findDataset(datasetType, dataId2, collections=chain1) 

511 self.assertNotEqual(ref2b, ref2) 

512 self.assertEqual(ref2b, registry.findDataset(datasetType, dataId2, collections=run2)) 

513 # Look in the chain for a permaflat that is in run1; should get the 

514 # same ref as if we'd searched run1 directly. 

515 dataId3 = {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"} 

516 self.assertEqual(registry.findDataset("permaflat", dataId3, collections=chain1), 

517 registry.findDataset("permaflat", dataId3, collections=run1),) 

518 # Define a new chain so we can test recursive chains. 

519 chain2 = "chain2" 

520 registry.registerCollection(chain2, type=CollectionType.CHAINED) 

521 registry.setCollectionChain(chain2, [(run2, "permabias"), chain1]) 

522 # Query for collections matching a regex. 

523 self.assertCountEqual( 

524 list(registry.queryCollections(re.compile("imported_."), flattenChains=False)), 

525 ["imported_r", "imported_g"] 

526 ) 

527 # Query for collections matching a regex or an explicit str. 

528 self.assertCountEqual( 

529 list(registry.queryCollections([re.compile("imported_."), "chain1"], flattenChains=False)), 

530 ["imported_r", "imported_g", "chain1"] 

531 ) 

532 # Search for permabias with dataId1 should find it via tag1 in chain2, 

533 # recursing, because is not in run1. 

534 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=run2)) 

535 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain2), ref1) 

536 # Search for permabias with dataId2 should find it in run2 (ref2b). 

537 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain2), ref2b) 

538 # Search for a permaflat that is in run2. That should not be found 

539 # at the front of chain2, because of the restriction to permabias 

540 # on run2 there, but it should be found in at the end of chain1. 

541 dataId4 = {"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-R2"} 

542 ref4 = registry.findDataset("permaflat", dataId4, collections=run2) 

543 self.assertIsNotNone(ref4) 

544 self.assertEqual(ref4, registry.findDataset("permaflat", dataId4, collections=chain2)) 

545 # Deleting a collection that's part of a CHAINED collection is not 

546 # allowed, and is exception-safe. 

547 with self.assertRaises(Exception): 

548 registry.removeCollection(run2) 

549 self.assertEqual(registry.getCollectionType(run2), CollectionType.RUN) 

550 with self.assertRaises(Exception): 

551 registry.removeCollection(chain1) 

552 self.assertEqual(registry.getCollectionType(chain1), CollectionType.CHAINED) 

553 # Actually remove chain2, test that it's gone by asking for its type. 

554 registry.removeCollection(chain2) 

555 with self.assertRaises(MissingCollectionError): 

556 registry.getCollectionType(chain2) 

557 # Actually remove run2 and chain1, which should work now. 

558 registry.removeCollection(chain1) 

559 registry.removeCollection(run2) 

560 with self.assertRaises(MissingCollectionError): 

561 registry.getCollectionType(run2) 

562 with self.assertRaises(MissingCollectionError): 

563 registry.getCollectionType(chain1) 

564 # Remove tag1 as well, just to test that we can remove TAGGED 

565 # collections. 

566 registry.removeCollection(tag1) 

567 with self.assertRaises(MissingCollectionError): 

568 registry.getCollectionType(tag1) 

569 

570 def testBasicTransaction(self): 

571 """Test that all operations within a single transaction block are 

572 rolled back if an exception propagates out of the block. 

573 """ 

574 registry = self.makeRegistry() 

575 storageClass = StorageClass("testDatasetType") 

576 registry.storageClasses.registerStorageClass(storageClass) 

577 with registry.transaction(): 

578 registry.insertDimensionData("instrument", {"name": "Cam1", "class_name": "A"}) 

579 with self.assertRaises(ValueError): 

580 with registry.transaction(): 

581 registry.insertDimensionData("instrument", {"name": "Cam2"}) 

582 raise ValueError("Oops, something went wrong") 

583 # Cam1 should exist 

584 self.assertEqual(registry.expandDataId(instrument="Cam1").records["instrument"].class_name, "A") 

585 # But Cam2 and Cam3 should both not exist 

586 with self.assertRaises(LookupError): 

587 registry.expandDataId(instrument="Cam2") 

588 with self.assertRaises(LookupError): 

589 registry.expandDataId(instrument="Cam3") 

590 

591 def testNestedTransaction(self): 

592 """Test that operations within a transaction block are not rolled back 

593 if an exception propagates out of an inner transaction block and is 

594 then caught. 

595 """ 

596 registry = self.makeRegistry() 

597 dimension = registry.dimensions["instrument"] 

598 dataId1 = {"instrument": "DummyCam"} 

599 dataId2 = {"instrument": "DummyCam2"} 

600 checkpointReached = False 

601 with registry.transaction(): 

602 # This should be added and (ultimately) committed. 

603 registry.insertDimensionData(dimension, dataId1) 

604 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

605 with registry.transaction(): 

606 # This does not conflict, and should succeed (but not 

607 # be committed). 

608 registry.insertDimensionData(dimension, dataId2) 

609 checkpointReached = True 

610 # This should conflict and raise, triggerring a rollback 

611 # of the previous insertion within the same transaction 

612 # context, but not the original insertion in the outer 

613 # block. 

614 registry.insertDimensionData(dimension, dataId1) 

615 self.assertTrue(checkpointReached) 

616 self.assertIsNotNone(registry.expandDataId(dataId1, graph=dimension.graph)) 

617 with self.assertRaises(LookupError): 

618 registry.expandDataId(dataId2, graph=dimension.graph) 

619 

620 def testInstrumentDimensions(self): 

621 """Test queries involving only instrument dimensions, with no joins to 

622 skymap.""" 

623 registry = self.makeRegistry() 

624 

625 # need a bunch of dimensions and datasets for test 

626 registry.insertDimensionData( 

627 "instrument", 

628 dict(name="DummyCam", visit_max=25, exposure_max=300, detector_max=6) 

629 ) 

630 registry.insertDimensionData( 

631 "physical_filter", 

632 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

633 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

634 ) 

635 registry.insertDimensionData( 

636 "detector", 

637 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in range(1, 6)] 

638 ) 

639 registry.insertDimensionData( 

640 "visit_system", 

641 dict(instrument="DummyCam", id=1, name="default"), 

642 ) 

643 registry.insertDimensionData( 

644 "visit", 

645 dict(instrument="DummyCam", id=10, name="ten", physical_filter="dummy_i", visit_system=1), 

646 dict(instrument="DummyCam", id=11, name="eleven", physical_filter="dummy_r", visit_system=1), 

647 dict(instrument="DummyCam", id=20, name="twelve", physical_filter="dummy_r", visit_system=1), 

648 ) 

649 registry.insertDimensionData( 

650 "exposure", 

651 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i"), 

652 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i"), 

653 dict(instrument="DummyCam", id=110, name="110", physical_filter="dummy_r"), 

654 dict(instrument="DummyCam", id=111, name="111", physical_filter="dummy_r"), 

655 dict(instrument="DummyCam", id=200, name="200", physical_filter="dummy_r"), 

656 dict(instrument="DummyCam", id=201, name="201", physical_filter="dummy_r"), 

657 ) 

658 registry.insertDimensionData( 

659 "visit_definition", 

660 dict(instrument="DummyCam", exposure=100, visit_system=1, visit=10), 

661 dict(instrument="DummyCam", exposure=101, visit_system=1, visit=10), 

662 dict(instrument="DummyCam", exposure=110, visit_system=1, visit=11), 

663 dict(instrument="DummyCam", exposure=111, visit_system=1, visit=11), 

664 dict(instrument="DummyCam", exposure=200, visit_system=1, visit=20), 

665 dict(instrument="DummyCam", exposure=201, visit_system=1, visit=20), 

666 ) 

667 # dataset types 

668 run1 = "test1_r" 

669 run2 = "test2_r" 

670 tagged2 = "test2_t" 

671 registry.registerRun(run1) 

672 registry.registerRun(run2) 

673 registry.registerCollection(tagged2) 

674 storageClass = StorageClass("testDataset") 

675 registry.storageClasses.registerStorageClass(storageClass) 

676 rawType = DatasetType(name="RAW", 

677 dimensions=registry.dimensions.extract(("instrument", "exposure", "detector")), 

678 storageClass=storageClass) 

679 registry.registerDatasetType(rawType) 

680 calexpType = DatasetType(name="CALEXP", 

681 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

682 storageClass=storageClass) 

683 registry.registerDatasetType(calexpType) 

684 

685 # add pre-existing datasets 

686 for exposure in (100, 101, 110, 111): 

687 for detector in (1, 2, 3): 

688 # note that only 3 of 5 detectors have datasets 

689 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

690 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run1) 

691 # exposures 100 and 101 appear in both run1 and tagged2. 

692 # 100 has different datasets in the different collections 

693 # 101 has the same dataset in both collections. 

694 if exposure == 100: 

695 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

696 if exposure in (100, 101): 

697 registry.associate(tagged2, [ref]) 

698 # Add pre-existing datasets to tagged2. 

699 for exposure in (200, 201): 

700 for detector in (3, 4, 5): 

701 # note that only 3 of 5 detectors have datasets 

702 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

703 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

704 registry.associate(tagged2, [ref]) 

705 

706 dimensions = DimensionGraph( 

707 registry.dimensions, 

708 dimensions=(rawType.dimensions.required | calexpType.dimensions.required) 

709 ) 

710 # Test that single dim string works as well as list of str 

711 rows = list(registry.queryDimensions("visit", datasets=rawType, collections=run1, expand=True)) 

712 rowsI = list(registry.queryDimensions(["visit"], datasets=rawType, collections=run1, expand=True)) 

713 self.assertEqual(rows, rowsI) 

714 # with empty expression 

715 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, expand=True)) 

716 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

717 for dataId in rows: 

718 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

719 packer1 = registry.dimensions.makePacker("visit_detector", dataId) 

720 packer2 = registry.dimensions.makePacker("exposure_detector", dataId) 

721 self.assertEqual(packer1.unpack(packer1.pack(dataId)), 

722 DataCoordinate.standardize(dataId, graph=packer1.dimensions)) 

723 self.assertEqual(packer2.unpack(packer2.pack(dataId)), 

724 DataCoordinate.standardize(dataId, graph=packer2.dimensions)) 

725 self.assertNotEqual(packer1.pack(dataId), packer2.pack(dataId)) 

726 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

727 (100, 101, 110, 111)) 

728 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11)) 

729 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

730 

731 # second collection 

732 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=tagged2)) 

733 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

734 for dataId in rows: 

735 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

736 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

737 (100, 101, 200, 201)) 

738 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 20)) 

739 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

740 

741 # with two input datasets 

742 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=[run1, tagged2])) 

743 self.assertEqual(len(set(rows)), 6*3) # 6 exposures times 3 detectors; set needed to de-dupe 

744 for dataId in rows: 

745 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

746 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

747 (100, 101, 110, 111, 200, 201)) 

748 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11, 20)) 

749 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

750 

751 # limit to single visit 

752 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

753 where="visit = 10")) 

754 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

755 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

756 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

757 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

758 

759 # more limiting expression, using link names instead of Table.column 

760 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

761 where="visit = 10 and detector > 1")) 

762 self.assertEqual(len(rows), 2*2) # 2 exposures times 2 detectors 

763 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

764 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

765 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (2, 3)) 

766 

767 # expression excludes everything 

768 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

769 where="visit > 1000")) 

770 self.assertEqual(len(rows), 0) 

771 

772 # Selecting by physical_filter, this is not in the dimensions, but it 

773 # is a part of the full expression so it should work too. 

774 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

775 where="physical_filter = 'dummy_r'")) 

776 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

777 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (110, 111)) 

778 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (11,)) 

779 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

780 

781 def testSkyMapDimensions(self): 

782 """Tests involving only skymap dimensions, no joins to instrument.""" 

783 registry = self.makeRegistry() 

784 

785 # need a bunch of dimensions and datasets for test, we want 

786 # "abstract_filter" in the test so also have to add physical_filter 

787 # dimensions 

788 registry.insertDimensionData( 

789 "instrument", 

790 dict(instrument="DummyCam") 

791 ) 

792 registry.insertDimensionData( 

793 "physical_filter", 

794 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

795 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

796 ) 

797 registry.insertDimensionData( 

798 "skymap", 

799 dict(name="DummyMap", hash="sha!".encode("utf8")) 

800 ) 

801 for tract in range(10): 

802 registry.insertDimensionData("tract", dict(skymap="DummyMap", id=tract)) 

803 registry.insertDimensionData( 

804 "patch", 

805 *[dict(skymap="DummyMap", tract=tract, id=patch, cell_x=0, cell_y=0) 

806 for patch in range(10)] 

807 ) 

808 

809 # dataset types 

810 run = "test" 

811 registry.registerRun(run) 

812 storageClass = StorageClass("testDataset") 

813 registry.storageClasses.registerStorageClass(storageClass) 

814 calexpType = DatasetType(name="deepCoadd_calexp", 

815 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

816 "abstract_filter")), 

817 storageClass=storageClass) 

818 registry.registerDatasetType(calexpType) 

819 mergeType = DatasetType(name="deepCoadd_mergeDet", 

820 dimensions=registry.dimensions.extract(("skymap", "tract", "patch")), 

821 storageClass=storageClass) 

822 registry.registerDatasetType(mergeType) 

823 measType = DatasetType(name="deepCoadd_meas", 

824 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

825 "abstract_filter")), 

826 storageClass=storageClass) 

827 registry.registerDatasetType(measType) 

828 

829 dimensions = DimensionGraph( 

830 registry.dimensions, 

831 dimensions=(calexpType.dimensions.required | mergeType.dimensions.required 

832 | measType.dimensions.required) 

833 ) 

834 

835 # add pre-existing datasets 

836 for tract in (1, 3, 5): 

837 for patch in (2, 4, 6, 7): 

838 dataId = dict(skymap="DummyMap", tract=tract, patch=patch) 

839 registry.insertDatasets(mergeType, dataIds=[dataId], run=run) 

840 for aFilter in ("i", "r"): 

841 dataId = dict(skymap="DummyMap", tract=tract, patch=patch, abstract_filter=aFilter) 

842 registry.insertDatasets(calexpType, dataIds=[dataId], run=run) 

843 

844 # with empty expression 

845 rows = list(registry.queryDimensions(dimensions, 

846 datasets=[calexpType, mergeType], collections=run)) 

847 self.assertEqual(len(rows), 3*4*2) # 4 tracts x 4 patches x 2 filters 

848 for dataId in rows: 

849 self.assertCountEqual(dataId.keys(), ("skymap", "tract", "patch", "abstract_filter")) 

850 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

851 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

852 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

853 

854 # limit to 2 tracts and 2 patches 

855 rows = list(registry.queryDimensions(dimensions, 

856 datasets=[calexpType, mergeType], collections=run, 

857 where="tract IN (1, 5) AND patch IN (2, 7)")) 

858 self.assertEqual(len(rows), 2*2*2) # 2 tracts x 2 patches x 2 filters 

859 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 5)) 

860 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 7)) 

861 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

862 

863 # limit to single filter 

864 rows = list(registry.queryDimensions(dimensions, 

865 datasets=[calexpType, mergeType], collections=run, 

866 where="abstract_filter = 'i'")) 

867 self.assertEqual(len(rows), 3*4*1) # 4 tracts x 4 patches x 2 filters 

868 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

869 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

870 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i",)) 

871 

872 # expression excludes everything, specifying non-existing skymap is 

873 # not a fatal error, it's operator error 

874 rows = list(registry.queryDimensions(dimensions, 

875 datasets=[calexpType, mergeType], collections=run, 

876 where="skymap = 'Mars'")) 

877 self.assertEqual(len(rows), 0) 

878 

879 def testSpatialMatch(self): 

880 """Test involving spatial match using join tables. 

881 

882 Note that realistic test needs a reasonably-defined skypix and regions 

883 in registry tables which is hard to implement in this simple test. 

884 So we do not actually fill registry with any data and all queries will 

885 return empty result, but this is still useful for coverage of the code 

886 that generates query. 

887 """ 

888 registry = self.makeRegistry() 

889 

890 # dataset types 

891 collection = "test" 

892 registry.registerRun(name=collection) 

893 storageClass = StorageClass("testDataset") 

894 registry.storageClasses.registerStorageClass(storageClass) 

895 

896 calexpType = DatasetType(name="CALEXP", 

897 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

898 storageClass=storageClass) 

899 registry.registerDatasetType(calexpType) 

900 

901 coaddType = DatasetType(name="deepCoadd_calexp", 

902 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

903 "abstract_filter")), 

904 storageClass=storageClass) 

905 registry.registerDatasetType(coaddType) 

906 

907 dimensions = DimensionGraph( 

908 registry.dimensions, 

909 dimensions=(calexpType.dimensions.required | coaddType.dimensions.required) 

910 ) 

911 

912 # without data this should run OK but return empty set 

913 rows = list(registry.queryDimensions(dimensions, datasets=calexpType, collections=collection)) 

914 self.assertEqual(len(rows), 0) 

915 

916 def testCalibrationLabelIndirection(self): 

917 """Test that we can look up datasets with calibration_label dimensions 

918 from a data ID with exposure dimensions. 

919 """ 

920 

921 def _dt(iso_string): 

922 return astropy.time.Time(iso_string, format="iso", scale="utc") 

923 

924 registry = self.makeRegistry() 

925 

926 flat = DatasetType( 

927 "flat", 

928 registry.dimensions.extract( 

929 ["instrument", "detector", "physical_filter", "calibration_label"] 

930 ), 

931 "ImageU" 

932 ) 

933 registry.registerDatasetType(flat) 

934 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

935 registry.insertDimensionData( 

936 "physical_filter", 

937 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

938 ) 

939 registry.insertDimensionData( 

940 "detector", 

941 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in (1, 2, 3, 4, 5)] 

942 ) 

943 registry.insertDimensionData( 

944 "exposure", 

945 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i", 

946 datetime_begin=_dt("2005-12-15 02:00:00"), datetime_end=_dt("2005-12-15 03:00:00")), 

947 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i", 

948 datetime_begin=_dt("2005-12-16 02:00:00"), datetime_end=_dt("2005-12-16 03:00:00")), 

949 ) 

950 registry.insertDimensionData( 

951 "calibration_label", 

952 dict(instrument="DummyCam", name="first_night", 

953 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-15 04:00:00")), 

954 dict(instrument="DummyCam", name="second_night", 

955 datetime_begin=_dt("2005-12-16 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

956 dict(instrument="DummyCam", name="both_nights", 

957 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

958 ) 

959 # Different flats for different nights for detectors 1-3 in first 

960 # collection. 

961 run1 = "calibs1" 

962 registry.registerRun(run1) 

963 for detector in (1, 2, 3): 

964 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="first_night", 

965 physical_filter="dummy_i", detector=detector)], 

966 run=run1) 

967 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="second_night", 

968 physical_filter="dummy_i", detector=detector)], 

969 run=run1) 

970 # The same flat for both nights for detectors 3-5 (so detector 3 has 

971 # multiple valid flats) in second collection. 

972 run2 = "calib2" 

973 registry.registerRun(run2) 

974 for detector in (3, 4, 5): 

975 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="both_nights", 

976 physical_filter="dummy_i", detector=detector)], 

977 run=run2) 

978 # Perform queries for individual exposure+detector combinations, which 

979 # should always return exactly one flat. 

980 for exposure in (100, 101): 

981 for detector in (1, 2, 3): 

982 with self.subTest(exposure=exposure, detector=detector): 

983 rows = list(registry.queryDatasets("flat", collections=[run1], 

984 instrument="DummyCam", 

985 exposure=exposure, 

986 detector=detector)) 

987 self.assertEqual(len(rows), 1) 

988 for detector in (3, 4, 5): 

989 with self.subTest(exposure=exposure, detector=detector): 

990 rows = registry.queryDatasets("flat", collections=[run2], 

991 instrument="DummyCam", 

992 exposure=exposure, 

993 detector=detector) 

994 self.assertEqual(len(list(rows)), 1) 

995 for detector in (1, 2, 4, 5): 

996 with self.subTest(exposure=exposure, detector=detector): 

997 rows = registry.queryDatasets("flat", collections=[run1, run2], 

998 instrument="DummyCam", 

999 exposure=exposure, 

1000 detector=detector) 

1001 self.assertEqual(len(list(rows)), 1) 

1002 for detector in (3,): 

1003 with self.subTest(exposure=exposure, detector=detector): 

1004 rows = registry.queryDatasets("flat", collections=[run1, run2], 

1005 instrument="DummyCam", 

1006 exposure=exposure, 

1007 detector=detector) 

1008 self.assertEqual(len(list(rows)), 2) 

1009 

1010 def testAbstractFilterQuery(self): 

1011 """Test that we can run a query that just lists the known 

1012 abstract_filters. This is tricky because abstract_filter is 

1013 backed by a query against physical_filter. 

1014 """ 

1015 registry = self.makeRegistry() 

1016 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

1017 registry.insertDimensionData( 

1018 "physical_filter", 

1019 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

1020 dict(instrument="DummyCam", name="dummy_i2", abstract_filter="i"), 

1021 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

1022 ) 

1023 rows = list(registry.queryDimensions(["abstract_filter"])) 

1024 self.assertCountEqual( 

1025 rows, 

1026 [DataCoordinate.standardize(abstract_filter="i", universe=registry.dimensions), 

1027 DataCoordinate.standardize(abstract_filter="r", universe=registry.dimensions)] 

1028 ) 

1029 

1030 def testAttributeManager(self): 

1031 """Test basic functionality of attribute manager. 

1032 """ 

1033 # number of attributes with schema versions in a fresh database 

1034 VERSION_COUNT = 0 

1035 

1036 registry = self.makeRegistry() 

1037 attributes = registry._attributes 

1038 

1039 # check what get() returns for non-existing key 

1040 self.assertIsNone(attributes.get("attr")) 

1041 self.assertEqual(attributes.get("attr", ""), "") 

1042 self.assertEqual(attributes.get("attr", "Value"), "Value") 

1043 self.assertEqual(len(list(attributes.items())), VERSION_COUNT) 

1044 

1045 # cannot store empty key or value 

1046 with self.assertRaises(ValueError): 

1047 attributes.set("", "value") 

1048 with self.assertRaises(ValueError): 

1049 attributes.set("attr", "") 

1050 

1051 # set value of non-existing key 

1052 attributes.set("attr", "value") 

1053 self.assertEqual(len(list(attributes.items())), VERSION_COUNT + 1) 

1054 self.assertEqual(attributes.get("attr"), "value") 

1055 

1056 # update value of existing key 

1057 with self.assertRaises(ButlerAttributeExistsError): 

1058 attributes.set("attr", "value2") 

1059 

1060 attributes.set("attr", "value2", force=True) 

1061 self.assertEqual(len(list(attributes.items())), VERSION_COUNT + 1) 

1062 self.assertEqual(attributes.get("attr"), "value2") 

1063 

1064 # delete existing key 

1065 self.assertTrue(attributes.delete("attr")) 

1066 self.assertEqual(len(list(attributes.items())), VERSION_COUNT) 

1067 

1068 # delete non-existing key 

1069 self.assertFalse(attributes.delete("non-attr")) 

1070 

1071 # store bunch of keys and get the list back 

1072 data = [ 

1073 ("version.core", "1.2.3"), 

1074 ("version.dimensions", "3.2.1"), 

1075 ("config.managers.opaque", "ByNameOpaqueTableStorageManager"), 

1076 ] 

1077 for key, value in data: 

1078 attributes.set(key, value) 

1079 items = dict(attributes.items()) 

1080 for key, value in data: 

1081 self.assertEqual(items[key], value)