Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["RegistryTests"] 

24 

25from abc import ABC, abstractmethod 

26import os 

27import re 

28 

29import astropy.time 

30import sqlalchemy 

31from typing import Optional 

32 

33from ...core import ( 

34 DataCoordinate, 

35 DatasetRef, 

36 DatasetType, 

37 DimensionGraph, 

38 NamedValueSet, 

39 StorageClass, 

40 ddl, 

41 YamlRepoImportBackend 

42) 

43from .._registry import ( 

44 CollectionType, 

45 ConflictingDefinitionError, 

46 ConsistentDataIds, 

47 Registry, 

48 RegistryConfig, 

49) 

50from ..wildcards import DatasetTypeRestriction 

51from ..interfaces import MissingCollectionError 

52 

53 

54class RegistryTests(ABC): 

55 """Generic tests for the `Registry` class that can be subclassed to 

56 generate tests for different configurations. 

57 """ 

58 

59 collectionsManager: Optional[str] = None 

60 """Name of the collections manager class, if subclass provides value for 

61 this member then it overrides name specified in default configuration 

62 (`str`). 

63 """ 

64 

65 @classmethod 

66 @abstractmethod 

67 def getDataDir(cls) -> str: 

68 """Return the root directory containing test data YAML files. 

69 """ 

70 raise NotImplementedError() 

71 

72 def makeRegistryConfig(self) -> RegistryConfig: 

73 """Create RegistryConfig used to create a registry. 

74 

75 This method should be called by a subclass from `makeRegistry`. 

76 Returned instance will be pre-configured based on the values of class 

77 members, and default-configured for all other parametrs. Subclasses 

78 that need default configuration should just instantiate 

79 `RegistryConfig` directly. 

80 """ 

81 config = RegistryConfig() 

82 if self.collectionsManager: 

83 config["managers"]["collections"] = self.collectionsManager 

84 return config 

85 

86 @abstractmethod 

87 def makeRegistry(self) -> Registry: 

88 """Return the Registry instance to be tested. 

89 """ 

90 raise NotImplementedError() 

91 

92 def loadData(self, registry: Registry, filename: str): 

93 """Load registry test data from ``getDataDir/<filename>``, 

94 which should be a YAML import/export file. 

95 """ 

96 with open(os.path.join(self.getDataDir(), filename), 'r') as stream: 

97 backend = YamlRepoImportBackend(stream, registry) 

98 backend.register() 

99 backend.load(datastore=None) 

100 

101 def assertRowCount(self, registry: Registry, table: str, count: int): 

102 """Check the number of rows in table. 

103 """ 

104 # TODO: all tests that rely on this method should be rewritten, as it 

105 # needs to depend on Registry implementation details to have any chance 

106 # of working. 

107 sql = sqlalchemy.sql.select( 

108 [sqlalchemy.sql.func.count()] 

109 ).select_from( 

110 getattr(registry._tables, table) 

111 ) 

112 self.assertEqual(registry._db.query(sql).scalar(), count) 

113 

114 def testOpaque(self): 

115 """Tests for `Registry.registerOpaqueTable`, 

116 `Registry.insertOpaqueData`, `Registry.fetchOpaqueData`, and 

117 `Registry.deleteOpaqueData`. 

118 """ 

119 registry = self.makeRegistry() 

120 table = "opaque_table_for_testing" 

121 registry.registerOpaqueTable( 

122 table, 

123 spec=ddl.TableSpec( 

124 fields=[ 

125 ddl.FieldSpec("id", dtype=sqlalchemy.BigInteger, primaryKey=True), 

126 ddl.FieldSpec("name", dtype=sqlalchemy.String, length=16, nullable=False), 

127 ddl.FieldSpec("count", dtype=sqlalchemy.SmallInteger, nullable=True), 

128 ], 

129 ) 

130 ) 

131 rows = [ 

132 {"id": 1, "name": "one", "count": None}, 

133 {"id": 2, "name": "two", "count": 5}, 

134 {"id": 3, "name": "three", "count": 6}, 

135 ] 

136 registry.insertOpaqueData(table, *rows) 

137 self.assertCountEqual(rows, list(registry.fetchOpaqueData(table))) 

138 self.assertEqual(rows[0:1], list(registry.fetchOpaqueData(table, id=1))) 

139 self.assertEqual(rows[1:2], list(registry.fetchOpaqueData(table, name="two"))) 

140 self.assertEqual([], list(registry.fetchOpaqueData(table, id=1, name="two"))) 

141 registry.deleteOpaqueData(table, id=3) 

142 self.assertCountEqual(rows[:2], list(registry.fetchOpaqueData(table))) 

143 registry.deleteOpaqueData(table) 

144 self.assertEqual([], list(registry.fetchOpaqueData(table))) 

145 

146 def testDatasetType(self): 

147 """Tests for `Registry.registerDatasetType` and 

148 `Registry.getDatasetType`. 

149 """ 

150 registry = self.makeRegistry() 

151 # Check valid insert 

152 datasetTypeName = "test" 

153 storageClass = StorageClass("testDatasetType") 

154 registry.storageClasses.registerStorageClass(storageClass) 

155 dimensions = registry.dimensions.extract(("instrument", "visit")) 

156 differentDimensions = registry.dimensions.extract(("instrument", "patch")) 

157 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

158 # Inserting for the first time should return True 

159 self.assertTrue(registry.registerDatasetType(inDatasetType)) 

160 outDatasetType1 = registry.getDatasetType(datasetTypeName) 

161 self.assertEqual(outDatasetType1, inDatasetType) 

162 

163 # Re-inserting should work 

164 self.assertFalse(registry.registerDatasetType(inDatasetType)) 

165 # Except when they are not identical 

166 with self.assertRaises(ConflictingDefinitionError): 

167 nonIdenticalDatasetType = DatasetType(datasetTypeName, differentDimensions, storageClass) 

168 registry.registerDatasetType(nonIdenticalDatasetType) 

169 

170 # Template can be None 

171 datasetTypeName = "testNoneTemplate" 

172 storageClass = StorageClass("testDatasetType2") 

173 registry.storageClasses.registerStorageClass(storageClass) 

174 dimensions = registry.dimensions.extract(("instrument", "visit")) 

175 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

176 registry.registerDatasetType(inDatasetType) 

177 outDatasetType2 = registry.getDatasetType(datasetTypeName) 

178 self.assertEqual(outDatasetType2, inDatasetType) 

179 

180 allTypes = set(registry.queryDatasetTypes()) 

181 self.assertEqual(allTypes, {outDatasetType1, outDatasetType2}) 

182 

183 def testDimensions(self): 

184 """Tests for `Registry.insertDimensionData`, 

185 `Registry.syncDimensionData`, and `Registry.expandDataId`. 

186 """ 

187 registry = self.makeRegistry() 

188 dimensionName = "instrument" 

189 dimension = registry.dimensions[dimensionName] 

190 dimensionValue = {"name": "DummyCam", "visit_max": 10, "exposure_max": 10, "detector_max": 2, 

191 "class_name": "lsst.obs.base.Instrument"} 

192 registry.insertDimensionData(dimensionName, dimensionValue) 

193 # Inserting the same value twice should fail 

194 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

195 registry.insertDimensionData(dimensionName, dimensionValue) 

196 # expandDataId should retrieve the record we just inserted 

197 self.assertEqual( 

198 registry.expandDataId( 

199 instrument="DummyCam", 

200 graph=dimension.graph 

201 ).records[dimensionName].toDict(), 

202 dimensionValue 

203 ) 

204 # expandDataId should raise if there is no record with the given ID. 

205 with self.assertRaises(LookupError): 

206 registry.expandDataId({"instrument": "Unknown"}, graph=dimension.graph) 

207 # abstract_filter doesn't have a table; insert should fail. 

208 with self.assertRaises(TypeError): 

209 registry.insertDimensionData("abstract_filter", {"abstract_filter": "i"}) 

210 dimensionName2 = "physical_filter" 

211 dimension2 = registry.dimensions[dimensionName2] 

212 dimensionValue2 = {"name": "DummyCam_i", "abstract_filter": "i"} 

213 # Missing required dependency ("instrument") should fail 

214 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

215 registry.insertDimensionData(dimensionName2, dimensionValue2) 

216 # Adding required dependency should fix the failure 

217 dimensionValue2["instrument"] = "DummyCam" 

218 registry.insertDimensionData(dimensionName2, dimensionValue2) 

219 # expandDataId should retrieve the record we just inserted. 

220 self.assertEqual( 

221 registry.expandDataId( 

222 instrument="DummyCam", physical_filter="DummyCam_i", 

223 graph=dimension2.graph 

224 ).records[dimensionName2].toDict(), 

225 dimensionValue2 

226 ) 

227 # Use syncDimensionData to insert a new record successfully. 

228 dimensionName3 = "detector" 

229 dimensionValue3 = {"instrument": "DummyCam", "id": 1, "full_name": "one", 

230 "name_in_raft": "zero", "purpose": "SCIENCE"} 

231 self.assertTrue(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

232 # Sync that again. Note that one field ("raft") is NULL, and that 

233 # should be okay. 

234 self.assertFalse(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

235 # Now try that sync with the same primary key but a different value. 

236 # This should fail. 

237 with self.assertRaises(ConflictingDefinitionError): 

238 registry.syncDimensionData( 

239 dimensionName3, 

240 {"instrument": "DummyCam", "id": 1, "full_name": "one", 

241 "name_in_raft": "four", "purpose": "SCIENCE"} 

242 ) 

243 

244 def testDataIdRelationships(self): 

245 """Test `Registry.relateDataId`. 

246 """ 

247 registry = self.makeRegistry() 

248 self.loadData(registry, "base.yaml") 

249 # Simple cases where the dimension key-value pairs tell us everything. 

250 self.assertEqual( 

251 registry.relateDataIds( 

252 {"instrument": "Cam1"}, 

253 {"instrument": "Cam1"}, 

254 ), 

255 ConsistentDataIds(contains=True, within=True, overlaps=True) 

256 ) 

257 self.assertEqual( 

258 registry.relateDataIds({}, {}), 

259 ConsistentDataIds(contains=True, within=True, overlaps=False) 

260 ) 

261 self.assertEqual( 

262 registry.relateDataIds({"instrument": "Cam1"}, {}), 

263 ConsistentDataIds(contains=True, within=False, overlaps=False) 

264 ) 

265 self.assertEqual( 

266 registry.relateDataIds({}, {"instrument": "Cam1"}), 

267 ConsistentDataIds(contains=False, within=True, overlaps=False) 

268 ) 

269 self.assertEqual( 

270 registry.relateDataIds( 

271 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

272 {"instrument": "Cam1"}, 

273 ), 

274 ConsistentDataIds(contains=True, within=False, overlaps=True) 

275 ) 

276 self.assertEqual( 

277 registry.relateDataIds( 

278 {"instrument": "Cam1"}, 

279 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

280 ), 

281 ConsistentDataIds(contains=False, within=True, overlaps=True) 

282 ) 

283 self.assertIsNone( 

284 registry.relateDataIds( 

285 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

286 {"instrument": "Cam1", "physical_filter": "Cam1-R1"}, 

287 ) 

288 ) 

289 # Trickier cases where we need to expand data IDs, but it's still just 

290 # required and implied dimension relationships. 

291 self.assertEqual( 

292 registry.relateDataIds( 

293 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

294 {"instrument": "Cam1", "abstract_filter": "g"}, 

295 ), 

296 ConsistentDataIds(contains=True, within=False, overlaps=True) 

297 ) 

298 self.assertEqual( 

299 registry.relateDataIds( 

300 {"instrument": "Cam1", "abstract_filter": "g"}, 

301 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

302 ), 

303 ConsistentDataIds(contains=False, within=True, overlaps=True) 

304 ) 

305 self.assertEqual( 

306 registry.relateDataIds( 

307 {"instrument": "Cam1"}, 

308 {"htm7": 131073}, 

309 ), 

310 ConsistentDataIds(contains=False, within=False, overlaps=False) 

311 ) 

312 # Trickiest cases involve spatial or temporal overlaps or non-dimension 

313 # elements that relate things (of which visit_definition is our only 

314 # current example). 

315 # 

316 # These two HTM IDs at different levels have a "contains" relationship 

317 # spatially, but there is no overlap of dimension keys. The exact 

318 # result of relateDataIds is unspecified for this case, but it's 

319 # guaranteed to be truthy (see relateDataIds docs.). 

320 self.assertTrue( 

321 registry.relateDataIds({"htm7": 131073}, {"htm9": 2097169}) 

322 ) 

323 # These two HTM IDs at different levels are disjoint spatially, which 

324 # means the data IDs are inconsistent. 

325 self.assertIsNone( 

326 registry.relateDataIds({"htm7": 131073}, {"htm9": 2097391}) 

327 ) 

328 # Insert a few more dimension records for the next test. 

329 registry.insertDimensionData( 

330 "exposure", 

331 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G"}, 

332 ) 

333 registry.insertDimensionData( 

334 "exposure", 

335 {"instrument": "Cam1", "id": 2, "name": "two", "physical_filter": "Cam1-G"}, 

336 ) 

337 registry.insertDimensionData( 

338 "visit_system", 

339 {"instrument": "Cam1", "id": 0, "name": "one-to-one"}, 

340 ) 

341 registry.insertDimensionData( 

342 "visit", 

343 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G", "visit_system": 0}, 

344 ) 

345 registry.insertDimensionData( 

346 "visit_definition", 

347 {"instrument": "Cam1", "visit": 1, "exposure": 1, "visit_system": 0}, 

348 ) 

349 self.assertEqual( 

350 registry.relateDataIds( 

351 {"instrument": "Cam1", "visit": 1}, 

352 {"instrument": "Cam1", "exposure": 1}, 

353 ), 

354 ConsistentDataIds(contains=False, within=False, overlaps=True) 

355 ) 

356 self.assertIsNone( 

357 registry.relateDataIds( 

358 {"instrument": "Cam1", "visit": 1}, 

359 {"instrument": "Cam1", "exposure": 2}, 

360 ) 

361 ) 

362 

363 def testDataset(self): 

364 """Basic tests for `Registry.insertDatasets`, `Registry.getDataset`, 

365 and `Registry.removeDatasets`. 

366 """ 

367 registry = self.makeRegistry() 

368 self.loadData(registry, "base.yaml") 

369 run = "test" 

370 registry.registerRun(run) 

371 datasetType = registry.getDatasetType("permabias") 

372 dataId = {"instrument": "Cam1", "detector": 2} 

373 ref, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

374 outRef = registry.getDataset(ref.id) 

375 self.assertIsNotNone(ref.id) 

376 self.assertEqual(ref, outRef) 

377 with self.assertRaises(ConflictingDefinitionError): 

378 registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

379 registry.removeDatasets([ref]) 

380 self.assertIsNone(registry.findDataset(datasetType, dataId, collections=[run])) 

381 

382 def testFindDataset(self): 

383 """Tests for `Registry.findDataset`. 

384 """ 

385 registry = self.makeRegistry() 

386 self.loadData(registry, "base.yaml") 

387 run = "test" 

388 datasetType = registry.getDatasetType("permabias") 

389 dataId = {"instrument": "Cam1", "detector": 4} 

390 registry.registerRun(run) 

391 inputRef, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

392 outputRef = registry.findDataset(datasetType, dataId, collections=[run]) 

393 self.assertEqual(outputRef, inputRef) 

394 # Check that retrieval with invalid dataId raises 

395 with self.assertRaises(LookupError): 

396 dataId = {"instrument": "Cam1"} # no detector 

397 registry.findDataset(datasetType, dataId, collections=run) 

398 # Check that different dataIds match to different datasets 

399 dataId1 = {"instrument": "Cam1", "detector": 1} 

400 inputRef1, = registry.insertDatasets(datasetType, dataIds=[dataId1], run=run) 

401 dataId2 = {"instrument": "Cam1", "detector": 2} 

402 inputRef2, = registry.insertDatasets(datasetType, dataIds=[dataId2], run=run) 

403 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef1) 

404 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef2) 

405 self.assertNotEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef2) 

406 self.assertNotEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef1) 

407 # Check that requesting a non-existing dataId returns None 

408 nonExistingDataId = {"instrument": "Cam1", "detector": 3} 

409 self.assertIsNone(registry.findDataset(datasetType, nonExistingDataId, collections=run)) 

410 

411 def testDatasetTypeComponentQueries(self): 

412 """Test component options when querying for dataset types. 

413 """ 

414 registry = self.makeRegistry() 

415 self.loadData(registry, "base.yaml") 

416 self.loadData(registry, "datasets.yaml") 

417 # Test querying for dataset types with different inputs. 

418 # First query for all dataset types; components should only be included 

419 # when components=True. 

420 self.assertEqual( 

421 {"permabias", "permaflat"}, 

422 NamedValueSet(registry.queryDatasetTypes()).names 

423 ) 

424 self.assertEqual( 

425 {"permabias", "permaflat"}, 

426 NamedValueSet(registry.queryDatasetTypes(components=False)).names 

427 ) 

428 self.assertLess( 

429 {"permabias", "permaflat", "permabias.wcs", "permaflat.photoCalib"}, 

430 NamedValueSet(registry.queryDatasetTypes(components=True)).names 

431 ) 

432 # Use a pattern that can match either parent or components. Again, 

433 # components are only returned if components=True. 

434 self.assertEqual( 

435 {"permabias"}, 

436 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"))).names 

437 ) 

438 self.assertEqual( 

439 {"permabias"}, 

440 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"), components=False)).names 

441 ) 

442 self.assertLess( 

443 {"permabias", "permabias.wcs"}, 

444 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"), components=True)).names 

445 ) 

446 # This pattern matches only a component. In this case we also return 

447 # that component dataset type if components=None. 

448 self.assertEqual( 

449 {"permabias.wcs"}, 

450 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"))).names 

451 ) 

452 self.assertEqual( 

453 set(), 

454 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"), components=False)).names 

455 ) 

456 self.assertEqual( 

457 {"permabias.wcs"}, 

458 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"), components=True)).names 

459 ) 

460 

461 def testComponentLookups(self): 

462 """Test searching for component datasets via their parents. 

463 """ 

464 registry = self.makeRegistry() 

465 self.loadData(registry, "base.yaml") 

466 self.loadData(registry, "datasets.yaml") 

467 # Test getting the child dataset type (which does still exist in the 

468 # Registry), and check for consistency with 

469 # DatasetRef.makeComponentRef. 

470 collection = "imported_g" 

471 parentType = registry.getDatasetType("permabias") 

472 childType = registry.getDatasetType("permabias.wcs") 

473 parentRefResolved = registry.findDataset(parentType, collections=collection, 

474 instrument="Cam1", detector=1) 

475 self.assertIsInstance(parentRefResolved, DatasetRef) 

476 self.assertEqual(childType, parentRefResolved.makeComponentRef("wcs").datasetType) 

477 # Search for a single dataset with findDataset. 

478 childRef1 = registry.findDataset("permabias.wcs", collections=collection, 

479 dataId=parentRefResolved.dataId) 

480 self.assertEqual(childRef1, parentRefResolved.makeComponentRef("wcs")) 

481 # Search for detector data IDs constrained by component dataset 

482 # existence with queryDimensions. 

483 dataIds = set(registry.queryDimensions( 

484 ["detector"], 

485 datasets=["permabias.wcs"], 

486 collections=collection, 

487 expand=False, 

488 )) 

489 self.assertEqual( 

490 dataIds, 

491 { 

492 DataCoordinate.standardize(instrument="Cam1", detector=d, graph=parentType.dimensions) 

493 for d in (1, 2, 3) 

494 } 

495 ) 

496 # Search for multiple datasets of a single type with queryDatasets. 

497 childRefs2 = set(registry.queryDatasets( 

498 "permabias.wcs", 

499 collections=collection, 

500 expand=False, 

501 )) 

502 self.assertEqual( 

503 {ref.unresolved() for ref in childRefs2}, 

504 {DatasetRef(childType, dataId) for dataId in dataIds} 

505 ) 

506 

507 def testCollections(self): 

508 """Tests for registry methods that manage collections. 

509 """ 

510 registry = self.makeRegistry() 

511 self.loadData(registry, "base.yaml") 

512 self.loadData(registry, "datasets.yaml") 

513 run1 = "imported_g" 

514 run2 = "imported_r" 

515 datasetType = "permabias" 

516 # Find some datasets via their run's collection. 

517 dataId1 = {"instrument": "Cam1", "detector": 1} 

518 ref1 = registry.findDataset(datasetType, dataId1, collections=run1) 

519 self.assertIsNotNone(ref1) 

520 dataId2 = {"instrument": "Cam1", "detector": 2} 

521 ref2 = registry.findDataset(datasetType, dataId2, collections=run1) 

522 self.assertIsNotNone(ref2) 

523 # Associate those into a new collection,then look for them there. 

524 tag1 = "tag1" 

525 registry.registerCollection(tag1, type=CollectionType.TAGGED) 

526 registry.associate(tag1, [ref1, ref2]) 

527 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

528 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

529 # Disassociate one and verify that we can't it there anymore... 

530 registry.disassociate(tag1, [ref1]) 

531 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=tag1)) 

532 # ...but we can still find ref2 in tag1, and ref1 in the run. 

533 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run1), ref1) 

534 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

535 collections = set(registry.queryCollections()) 

536 self.assertEqual(collections, {run1, run2, tag1}) 

537 # Associate both refs into tag1 again; ref2 is already there, but that 

538 # should be a harmless no-op. 

539 registry.associate(tag1, [ref1, ref2]) 

540 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

541 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

542 # Get a different dataset (from a different run) that has the same 

543 # dataset type and data ID as ref2. 

544 ref2b = registry.findDataset(datasetType, dataId2, collections=run2) 

545 self.assertNotEqual(ref2, ref2b) 

546 # Attempting to associate that into tag1 should be an error. 

547 with self.assertRaises(ConflictingDefinitionError): 

548 registry.associate(tag1, [ref2b]) 

549 # That error shouldn't have messed up what we had before. 

550 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

551 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

552 # Attempt to associate the conflicting dataset again, this time with 

553 # a dataset that isn't in the collection and won't cause a conflict. 

554 # Should also fail without modifying anything. 

555 dataId3 = {"instrument": "Cam1", "detector": 3} 

556 ref3 = registry.findDataset(datasetType, dataId3, collections=run1) 

557 with self.assertRaises(ConflictingDefinitionError): 

558 registry.associate(tag1, [ref3, ref2b]) 

559 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

560 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

561 self.assertIsNone(registry.findDataset(datasetType, dataId3, collections=tag1)) 

562 # Register a chained collection that searches: 

563 # 1. 'tag1' 

564 # 2. 'run1', but only for the permaflat dataset 

565 # 3. 'run2' 

566 chain1 = "chain1" 

567 registry.registerCollection(chain1, type=CollectionType.CHAINED) 

568 self.assertIs(registry.getCollectionType(chain1), CollectionType.CHAINED) 

569 # Chained collection exists, but has no collections in it. 

570 self.assertFalse(registry.getCollectionChain(chain1)) 

571 # If we query for all collections, we should get the chained collection 

572 # only if we don't ask to flatten it (i.e. yield only its children). 

573 self.assertEqual(set(registry.queryCollections(flattenChains=False)), {tag1, run1, run2, chain1}) 

574 self.assertEqual(set(registry.queryCollections(flattenChains=True)), {tag1, run1, run2}) 

575 # Attempt to set its child collections to something circular; that 

576 # should fail. 

577 with self.assertRaises(ValueError): 

578 registry.setCollectionChain(chain1, [tag1, chain1]) 

579 # Add the child collections. 

580 registry.setCollectionChain(chain1, [tag1, (run1, "permaflat"), run2]) 

581 self.assertEqual( 

582 list(registry.getCollectionChain(chain1)), 

583 [(tag1, DatasetTypeRestriction.any), 

584 (run1, DatasetTypeRestriction.fromExpression("permaflat")), 

585 (run2, DatasetTypeRestriction.any)] 

586 ) 

587 # Searching for dataId1 or dataId2 in the chain should return ref1 and 

588 # ref2, because both are in tag1. 

589 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain1), ref1) 

590 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain1), ref2) 

591 # Now disassociate ref2 from tag1. The search (for permabias) with 

592 # dataId2 in chain1 should then: 

593 # 1. not find it in tag1 

594 # 2. not look in tag2, because it's restricted to permaflat here 

595 # 3. find a different dataset in run2 

596 registry.disassociate(tag1, [ref2]) 

597 ref2b = registry.findDataset(datasetType, dataId2, collections=chain1) 

598 self.assertNotEqual(ref2b, ref2) 

599 self.assertEqual(ref2b, registry.findDataset(datasetType, dataId2, collections=run2)) 

600 # Look in the chain for a permaflat that is in run1; should get the 

601 # same ref as if we'd searched run1 directly. 

602 dataId3 = {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"} 

603 self.assertEqual(registry.findDataset("permaflat", dataId3, collections=chain1), 

604 registry.findDataset("permaflat", dataId3, collections=run1),) 

605 # Define a new chain so we can test recursive chains. 

606 chain2 = "chain2" 

607 registry.registerCollection(chain2, type=CollectionType.CHAINED) 

608 registry.setCollectionChain(chain2, [(run2, "permabias"), chain1]) 

609 # Search for permabias with dataId1 should find it via tag1 in chain2, 

610 # recursing, because is not in run1. 

611 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=run2)) 

612 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain2), ref1) 

613 # Search for permabias with dataId2 should find it in run2 (ref2b). 

614 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain2), ref2b) 

615 # Search for a permaflat that is in run2. That should not be found 

616 # at the front of chain2, because of the restriction to permabias 

617 # on run2 there, but it should be found in at the end of chain1. 

618 dataId4 = {"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-R2"} 

619 ref4 = registry.findDataset("permaflat", dataId4, collections=run2) 

620 self.assertIsNotNone(ref4) 

621 self.assertEqual(ref4, registry.findDataset("permaflat", dataId4, collections=chain2)) 

622 # Deleting a collection that's part of a CHAINED collection is not 

623 # allowed, and is exception-safe. 

624 with self.assertRaises(Exception): 

625 registry.removeCollection(run2) 

626 self.assertEqual(registry.getCollectionType(run2), CollectionType.RUN) 

627 with self.assertRaises(Exception): 

628 registry.removeCollection(chain1) 

629 self.assertEqual(registry.getCollectionType(chain1), CollectionType.CHAINED) 

630 # Actually remove chain2, test that it's gone by asking for its type. 

631 registry.removeCollection(chain2) 

632 with self.assertRaises(MissingCollectionError): 

633 registry.getCollectionType(chain2) 

634 # Actually remove run2 and chain1, which should work now. 

635 registry.removeCollection(chain1) 

636 registry.removeCollection(run2) 

637 with self.assertRaises(MissingCollectionError): 

638 registry.getCollectionType(run2) 

639 with self.assertRaises(MissingCollectionError): 

640 registry.getCollectionType(chain1) 

641 # Remove tag1 as well, just to test that we can remove TAGGED 

642 # collections. 

643 registry.removeCollection(tag1) 

644 with self.assertRaises(MissingCollectionError): 

645 registry.getCollectionType(tag1) 

646 

647 def testBasicTransaction(self): 

648 """Test that all operations within a single transaction block are 

649 rolled back if an exception propagates out of the block. 

650 """ 

651 registry = self.makeRegistry() 

652 storageClass = StorageClass("testDatasetType") 

653 registry.storageClasses.registerStorageClass(storageClass) 

654 with registry.transaction(): 

655 registry.insertDimensionData("instrument", {"name": "Cam1", "class_name": "A"}) 

656 with self.assertRaises(ValueError): 

657 with registry.transaction(): 

658 registry.insertDimensionData("instrument", {"name": "Cam2"}) 

659 raise ValueError("Oops, something went wrong") 

660 # Cam1 should exist 

661 self.assertEqual(registry.expandDataId(instrument="Cam1").records["instrument"].class_name, "A") 

662 # But Cam2 and Cam3 should both not exist 

663 with self.assertRaises(LookupError): 

664 registry.expandDataId(instrument="Cam2") 

665 with self.assertRaises(LookupError): 

666 registry.expandDataId(instrument="Cam3") 

667 

668 def testNestedTransaction(self): 

669 """Test that operations within a transaction block are not rolled back 

670 if an exception propagates out of an inner transaction block and is 

671 then caught. 

672 """ 

673 registry = self.makeRegistry() 

674 dimension = registry.dimensions["instrument"] 

675 dataId1 = {"instrument": "DummyCam"} 

676 dataId2 = {"instrument": "DummyCam2"} 

677 checkpointReached = False 

678 with registry.transaction(): 

679 # This should be added and (ultimately) committed. 

680 registry.insertDimensionData(dimension, dataId1) 

681 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

682 with registry.transaction(): 

683 # This does not conflict, and should succeed (but not 

684 # be committed). 

685 registry.insertDimensionData(dimension, dataId2) 

686 checkpointReached = True 

687 # This should conflict and raise, triggerring a rollback 

688 # of the previous insertion within the same transaction 

689 # context, but not the original insertion in the outer 

690 # block. 

691 registry.insertDimensionData(dimension, dataId1) 

692 self.assertTrue(checkpointReached) 

693 self.assertIsNotNone(registry.expandDataId(dataId1, graph=dimension.graph)) 

694 with self.assertRaises(LookupError): 

695 registry.expandDataId(dataId2, graph=dimension.graph) 

696 

697 def testInstrumentDimensions(self): 

698 """Test queries involving only instrument dimensions, with no joins to 

699 skymap.""" 

700 registry = self.makeRegistry() 

701 

702 # need a bunch of dimensions and datasets for test 

703 registry.insertDimensionData( 

704 "instrument", 

705 dict(name="DummyCam", visit_max=25, exposure_max=300, detector_max=6) 

706 ) 

707 registry.insertDimensionData( 

708 "physical_filter", 

709 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

710 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

711 ) 

712 registry.insertDimensionData( 

713 "detector", 

714 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in range(1, 6)] 

715 ) 

716 registry.insertDimensionData( 

717 "visit_system", 

718 dict(instrument="DummyCam", id=1, name="default"), 

719 ) 

720 registry.insertDimensionData( 

721 "visit", 

722 dict(instrument="DummyCam", id=10, name="ten", physical_filter="dummy_i", visit_system=1), 

723 dict(instrument="DummyCam", id=11, name="eleven", physical_filter="dummy_r", visit_system=1), 

724 dict(instrument="DummyCam", id=20, name="twelve", physical_filter="dummy_r", visit_system=1), 

725 ) 

726 registry.insertDimensionData( 

727 "exposure", 

728 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i"), 

729 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i"), 

730 dict(instrument="DummyCam", id=110, name="110", physical_filter="dummy_r"), 

731 dict(instrument="DummyCam", id=111, name="111", physical_filter="dummy_r"), 

732 dict(instrument="DummyCam", id=200, name="200", physical_filter="dummy_r"), 

733 dict(instrument="DummyCam", id=201, name="201", physical_filter="dummy_r"), 

734 ) 

735 registry.insertDimensionData( 

736 "visit_definition", 

737 dict(instrument="DummyCam", exposure=100, visit_system=1, visit=10), 

738 dict(instrument="DummyCam", exposure=101, visit_system=1, visit=10), 

739 dict(instrument="DummyCam", exposure=110, visit_system=1, visit=11), 

740 dict(instrument="DummyCam", exposure=111, visit_system=1, visit=11), 

741 dict(instrument="DummyCam", exposure=200, visit_system=1, visit=20), 

742 dict(instrument="DummyCam", exposure=201, visit_system=1, visit=20), 

743 ) 

744 # dataset types 

745 run1 = "test1_r" 

746 run2 = "test2_r" 

747 tagged2 = "test2_t" 

748 registry.registerRun(run1) 

749 registry.registerRun(run2) 

750 registry.registerCollection(tagged2) 

751 storageClass = StorageClass("testDataset") 

752 registry.storageClasses.registerStorageClass(storageClass) 

753 rawType = DatasetType(name="RAW", 

754 dimensions=registry.dimensions.extract(("instrument", "exposure", "detector")), 

755 storageClass=storageClass) 

756 registry.registerDatasetType(rawType) 

757 calexpType = DatasetType(name="CALEXP", 

758 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

759 storageClass=storageClass) 

760 registry.registerDatasetType(calexpType) 

761 

762 # add pre-existing datasets 

763 for exposure in (100, 101, 110, 111): 

764 for detector in (1, 2, 3): 

765 # note that only 3 of 5 detectors have datasets 

766 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

767 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run1) 

768 # exposures 100 and 101 appear in both run1 and tagged2. 

769 # 100 has different datasets in the different collections 

770 # 101 has the same dataset in both collections. 

771 if exposure == 100: 

772 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

773 if exposure in (100, 101): 

774 registry.associate(tagged2, [ref]) 

775 # Add pre-existing datasets to tagged2. 

776 for exposure in (200, 201): 

777 for detector in (3, 4, 5): 

778 # note that only 3 of 5 detectors have datasets 

779 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

780 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

781 registry.associate(tagged2, [ref]) 

782 

783 dimensions = DimensionGraph( 

784 registry.dimensions, 

785 dimensions=(rawType.dimensions.required | calexpType.dimensions.required) 

786 ) 

787 # Test that single dim string works as well as list of str 

788 rows = list(registry.queryDimensions("visit", datasets=rawType, collections=run1, expand=True)) 

789 rowsI = list(registry.queryDimensions(["visit"], datasets=rawType, collections=run1, expand=True)) 

790 self.assertEqual(rows, rowsI) 

791 # with empty expression 

792 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, expand=True)) 

793 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

794 for dataId in rows: 

795 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

796 packer1 = registry.dimensions.makePacker("visit_detector", dataId) 

797 packer2 = registry.dimensions.makePacker("exposure_detector", dataId) 

798 self.assertEqual(packer1.unpack(packer1.pack(dataId)), 

799 DataCoordinate.standardize(dataId, graph=packer1.dimensions)) 

800 self.assertEqual(packer2.unpack(packer2.pack(dataId)), 

801 DataCoordinate.standardize(dataId, graph=packer2.dimensions)) 

802 self.assertNotEqual(packer1.pack(dataId), packer2.pack(dataId)) 

803 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

804 (100, 101, 110, 111)) 

805 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11)) 

806 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

807 

808 # second collection 

809 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=tagged2)) 

810 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

811 for dataId in rows: 

812 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

813 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

814 (100, 101, 200, 201)) 

815 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 20)) 

816 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

817 

818 # with two input datasets 

819 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=[run1, tagged2])) 

820 self.assertEqual(len(set(rows)), 6*3) # 6 exposures times 3 detectors; set needed to de-dupe 

821 for dataId in rows: 

822 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

823 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

824 (100, 101, 110, 111, 200, 201)) 

825 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11, 20)) 

826 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

827 

828 # limit to single visit 

829 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

830 where="visit = 10")) 

831 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

832 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

833 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

834 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

835 

836 # more limiting expression, using link names instead of Table.column 

837 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

838 where="visit = 10 and detector > 1")) 

839 self.assertEqual(len(rows), 2*2) # 2 exposures times 2 detectors 

840 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

841 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

842 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (2, 3)) 

843 

844 # expression excludes everything 

845 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

846 where="visit > 1000")) 

847 self.assertEqual(len(rows), 0) 

848 

849 # Selecting by physical_filter, this is not in the dimensions, but it 

850 # is a part of the full expression so it should work too. 

851 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

852 where="physical_filter = 'dummy_r'")) 

853 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

854 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (110, 111)) 

855 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (11,)) 

856 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

857 

858 def testSkyMapDimensions(self): 

859 """Tests involving only skymap dimensions, no joins to instrument.""" 

860 registry = self.makeRegistry() 

861 

862 # need a bunch of dimensions and datasets for test, we want 

863 # "abstract_filter" in the test so also have to add physical_filter 

864 # dimensions 

865 registry.insertDimensionData( 

866 "instrument", 

867 dict(instrument="DummyCam") 

868 ) 

869 registry.insertDimensionData( 

870 "physical_filter", 

871 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

872 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

873 ) 

874 registry.insertDimensionData( 

875 "skymap", 

876 dict(name="DummyMap", hash="sha!".encode("utf8")) 

877 ) 

878 for tract in range(10): 

879 registry.insertDimensionData("tract", dict(skymap="DummyMap", id=tract)) 

880 registry.insertDimensionData( 

881 "patch", 

882 *[dict(skymap="DummyMap", tract=tract, id=patch, cell_x=0, cell_y=0) 

883 for patch in range(10)] 

884 ) 

885 

886 # dataset types 

887 run = "test" 

888 registry.registerRun(run) 

889 storageClass = StorageClass("testDataset") 

890 registry.storageClasses.registerStorageClass(storageClass) 

891 calexpType = DatasetType(name="deepCoadd_calexp", 

892 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

893 "abstract_filter")), 

894 storageClass=storageClass) 

895 registry.registerDatasetType(calexpType) 

896 mergeType = DatasetType(name="deepCoadd_mergeDet", 

897 dimensions=registry.dimensions.extract(("skymap", "tract", "patch")), 

898 storageClass=storageClass) 

899 registry.registerDatasetType(mergeType) 

900 measType = DatasetType(name="deepCoadd_meas", 

901 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

902 "abstract_filter")), 

903 storageClass=storageClass) 

904 registry.registerDatasetType(measType) 

905 

906 dimensions = DimensionGraph( 

907 registry.dimensions, 

908 dimensions=(calexpType.dimensions.required | mergeType.dimensions.required 

909 | measType.dimensions.required) 

910 ) 

911 

912 # add pre-existing datasets 

913 for tract in (1, 3, 5): 

914 for patch in (2, 4, 6, 7): 

915 dataId = dict(skymap="DummyMap", tract=tract, patch=patch) 

916 registry.insertDatasets(mergeType, dataIds=[dataId], run=run) 

917 for aFilter in ("i", "r"): 

918 dataId = dict(skymap="DummyMap", tract=tract, patch=patch, abstract_filter=aFilter) 

919 registry.insertDatasets(calexpType, dataIds=[dataId], run=run) 

920 

921 # with empty expression 

922 rows = list(registry.queryDimensions(dimensions, 

923 datasets=[calexpType, mergeType], collections=run)) 

924 self.assertEqual(len(rows), 3*4*2) # 4 tracts x 4 patches x 2 filters 

925 for dataId in rows: 

926 self.assertCountEqual(dataId.keys(), ("skymap", "tract", "patch", "abstract_filter")) 

927 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

928 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

929 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

930 

931 # limit to 2 tracts and 2 patches 

932 rows = list(registry.queryDimensions(dimensions, 

933 datasets=[calexpType, mergeType], collections=run, 

934 where="tract IN (1, 5) AND patch IN (2, 7)")) 

935 self.assertEqual(len(rows), 2*2*2) # 2 tracts x 2 patches x 2 filters 

936 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 5)) 

937 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 7)) 

938 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

939 

940 # limit to single filter 

941 rows = list(registry.queryDimensions(dimensions, 

942 datasets=[calexpType, mergeType], collections=run, 

943 where="abstract_filter = 'i'")) 

944 self.assertEqual(len(rows), 3*4*1) # 4 tracts x 4 patches x 2 filters 

945 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

946 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

947 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i",)) 

948 

949 # expression excludes everything, specifying non-existing skymap is 

950 # not a fatal error, it's operator error 

951 rows = list(registry.queryDimensions(dimensions, 

952 datasets=[calexpType, mergeType], collections=run, 

953 where="skymap = 'Mars'")) 

954 self.assertEqual(len(rows), 0) 

955 

956 def testSpatialMatch(self): 

957 """Test involving spatial match using join tables. 

958 

959 Note that realistic test needs a reasonably-defined skypix and regions 

960 in registry tables which is hard to implement in this simple test. 

961 So we do not actually fill registry with any data and all queries will 

962 return empty result, but this is still useful for coverage of the code 

963 that generates query. 

964 """ 

965 registry = self.makeRegistry() 

966 

967 # dataset types 

968 collection = "test" 

969 registry.registerRun(name=collection) 

970 storageClass = StorageClass("testDataset") 

971 registry.storageClasses.registerStorageClass(storageClass) 

972 

973 calexpType = DatasetType(name="CALEXP", 

974 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

975 storageClass=storageClass) 

976 registry.registerDatasetType(calexpType) 

977 

978 coaddType = DatasetType(name="deepCoadd_calexp", 

979 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

980 "abstract_filter")), 

981 storageClass=storageClass) 

982 registry.registerDatasetType(coaddType) 

983 

984 dimensions = DimensionGraph( 

985 registry.dimensions, 

986 dimensions=(calexpType.dimensions.required | coaddType.dimensions.required) 

987 ) 

988 

989 # without data this should run OK but return empty set 

990 rows = list(registry.queryDimensions(dimensions, datasets=calexpType, collections=collection)) 

991 self.assertEqual(len(rows), 0) 

992 

993 def testCalibrationLabelIndirection(self): 

994 """Test that we can look up datasets with calibration_label dimensions 

995 from a data ID with exposure dimensions. 

996 """ 

997 

998 def _dt(iso_string): 

999 return astropy.time.Time(iso_string, format="iso", scale="utc") 

1000 

1001 registry = self.makeRegistry() 

1002 

1003 flat = DatasetType( 

1004 "flat", 

1005 registry.dimensions.extract( 

1006 ["instrument", "detector", "physical_filter", "calibration_label"] 

1007 ), 

1008 "ImageU" 

1009 ) 

1010 registry.registerDatasetType(flat) 

1011 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

1012 registry.insertDimensionData( 

1013 "physical_filter", 

1014 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

1015 ) 

1016 registry.insertDimensionData( 

1017 "detector", 

1018 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in (1, 2, 3, 4, 5)] 

1019 ) 

1020 registry.insertDimensionData( 

1021 "exposure", 

1022 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i", 

1023 datetime_begin=_dt("2005-12-15 02:00:00"), datetime_end=_dt("2005-12-15 03:00:00")), 

1024 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i", 

1025 datetime_begin=_dt("2005-12-16 02:00:00"), datetime_end=_dt("2005-12-16 03:00:00")), 

1026 ) 

1027 registry.insertDimensionData( 

1028 "calibration_label", 

1029 dict(instrument="DummyCam", name="first_night", 

1030 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-15 04:00:00")), 

1031 dict(instrument="DummyCam", name="second_night", 

1032 datetime_begin=_dt("2005-12-16 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

1033 dict(instrument="DummyCam", name="both_nights", 

1034 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

1035 ) 

1036 # Different flats for different nights for detectors 1-3 in first 

1037 # collection. 

1038 run1 = "calibs1" 

1039 registry.registerRun(run1) 

1040 for detector in (1, 2, 3): 

1041 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="first_night", 

1042 physical_filter="dummy_i", detector=detector)], 

1043 run=run1) 

1044 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="second_night", 

1045 physical_filter="dummy_i", detector=detector)], 

1046 run=run1) 

1047 # The same flat for both nights for detectors 3-5 (so detector 3 has 

1048 # multiple valid flats) in second collection. 

1049 run2 = "calib2" 

1050 registry.registerRun(run2) 

1051 for detector in (3, 4, 5): 

1052 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="both_nights", 

1053 physical_filter="dummy_i", detector=detector)], 

1054 run=run2) 

1055 # Perform queries for individual exposure+detector combinations, which 

1056 # should always return exactly one flat. 

1057 for exposure in (100, 101): 

1058 for detector in (1, 2, 3): 

1059 with self.subTest(exposure=exposure, detector=detector): 

1060 rows = list(registry.queryDatasets("flat", collections=[run1], 

1061 instrument="DummyCam", 

1062 exposure=exposure, 

1063 detector=detector)) 

1064 self.assertEqual(len(rows), 1) 

1065 for detector in (3, 4, 5): 

1066 with self.subTest(exposure=exposure, detector=detector): 

1067 rows = registry.queryDatasets("flat", collections=[run2], 

1068 instrument="DummyCam", 

1069 exposure=exposure, 

1070 detector=detector) 

1071 self.assertEqual(len(list(rows)), 1) 

1072 for detector in (1, 2, 4, 5): 

1073 with self.subTest(exposure=exposure, detector=detector): 

1074 rows = registry.queryDatasets("flat", collections=[run1, run2], 

1075 instrument="DummyCam", 

1076 exposure=exposure, 

1077 detector=detector) 

1078 self.assertEqual(len(list(rows)), 1) 

1079 for detector in (3,): 

1080 with self.subTest(exposure=exposure, detector=detector): 

1081 rows = registry.queryDatasets("flat", collections=[run1, run2], 

1082 instrument="DummyCam", 

1083 exposure=exposure, 

1084 detector=detector) 

1085 self.assertEqual(len(list(rows)), 2) 

1086 

1087 def testAbstractFilterQuery(self): 

1088 """Test that we can run a query that just lists the known 

1089 abstract_filters. This is tricky because abstract_filter is 

1090 backed by a query against physical_filter. 

1091 """ 

1092 registry = self.makeRegistry() 

1093 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

1094 registry.insertDimensionData( 

1095 "physical_filter", 

1096 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

1097 dict(instrument="DummyCam", name="dummy_i2", abstract_filter="i"), 

1098 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

1099 ) 

1100 rows = list(registry.queryDimensions(["abstract_filter"])) 

1101 self.assertCountEqual( 

1102 rows, 

1103 [DataCoordinate.standardize(abstract_filter="i", universe=registry.dimensions), 

1104 DataCoordinate.standardize(abstract_filter="r", universe=registry.dimensions)] 

1105 )