Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["RegistryTests"] 

24 

25from abc import ABC, abstractmethod 

26import os 

27 

28import astropy.time 

29import sqlalchemy 

30 

31from ...core import ( 

32 DataCoordinate, 

33 DatasetType, 

34 DimensionGraph, 

35 StorageClass, 

36 ddl, 

37 YamlRepoImportBackend 

38) 

39from .._registry import ( 

40 CollectionType, 

41 ConflictingDefinitionError, 

42 ConsistentDataIds, 

43 OrphanedRecordError, 

44 Registry, 

45) 

46from ..wildcards import DatasetTypeRestriction 

47 

48 

49class RegistryTests(ABC): 

50 """Generic tests for the `Registry` class that can be subclassed to 

51 generate tests for different configurations. 

52 """ 

53 

54 @classmethod 

55 @abstractmethod 

56 def getDataDir(cls) -> str: 

57 """Return the root directory containing test data YAML files. 

58 """ 

59 raise NotImplementedError() 

60 

61 @abstractmethod 

62 def makeRegistry(self) -> Registry: 

63 """Return the Registry instance to be tested. 

64 """ 

65 raise NotImplementedError() 

66 

67 def loadData(self, registry: Registry, filename: str): 

68 """Load registry test data from ``getDataDir/<filename>``, 

69 which should be a YAML import/export file. 

70 """ 

71 with open(os.path.join(self.getDataDir(), filename), 'r') as stream: 

72 backend = YamlRepoImportBackend(stream, registry) 

73 backend.register() 

74 backend.load(datastore=None) 

75 

76 def assertRowCount(self, registry: Registry, table: str, count: int): 

77 """Check the number of rows in table. 

78 """ 

79 # TODO: all tests that rely on this method should be rewritten, as it 

80 # needs to depend on Registry implementation details to have any chance 

81 # of working. 

82 sql = sqlalchemy.sql.select( 

83 [sqlalchemy.sql.func.count()] 

84 ).select_from( 

85 getattr(registry._tables, table) 

86 ) 

87 self.assertEqual(registry._db.query(sql).scalar(), count) 

88 

89 def testOpaque(self): 

90 """Tests for `Registry.registerOpaqueTable`, 

91 `Registry.insertOpaqueData`, `Registry.fetchOpaqueData`, and 

92 `Registry.deleteOpaqueData`. 

93 """ 

94 registry = self.makeRegistry() 

95 table = "opaque_table_for_testing" 

96 registry.registerOpaqueTable( 

97 table, 

98 spec=ddl.TableSpec( 

99 fields=[ 

100 ddl.FieldSpec("id", dtype=sqlalchemy.BigInteger, primaryKey=True), 

101 ddl.FieldSpec("name", dtype=sqlalchemy.String, length=16, nullable=False), 

102 ddl.FieldSpec("count", dtype=sqlalchemy.SmallInteger, nullable=True), 

103 ], 

104 ) 

105 ) 

106 rows = [ 

107 {"id": 1, "name": "one", "count": None}, 

108 {"id": 2, "name": "two", "count": 5}, 

109 {"id": 3, "name": "three", "count": 6}, 

110 ] 

111 registry.insertOpaqueData(table, *rows) 

112 self.assertCountEqual(rows, list(registry.fetchOpaqueData(table))) 

113 self.assertEqual(rows[0:1], list(registry.fetchOpaqueData(table, id=1))) 

114 self.assertEqual(rows[1:2], list(registry.fetchOpaqueData(table, name="two"))) 

115 self.assertEqual([], list(registry.fetchOpaqueData(table, id=1, name="two"))) 

116 registry.deleteOpaqueData(table, id=3) 

117 self.assertCountEqual(rows[:2], list(registry.fetchOpaqueData(table))) 

118 registry.deleteOpaqueData(table) 

119 self.assertEqual([], list(registry.fetchOpaqueData(table))) 

120 

121 def testDatasetType(self): 

122 """Tests for `Registry.registerDatasetType` and 

123 `Registry.getDatasetType`. 

124 """ 

125 registry = self.makeRegistry() 

126 # Check valid insert 

127 datasetTypeName = "test" 

128 storageClass = StorageClass("testDatasetType") 

129 registry.storageClasses.registerStorageClass(storageClass) 

130 dimensions = registry.dimensions.extract(("instrument", "visit")) 

131 differentDimensions = registry.dimensions.extract(("instrument", "patch")) 

132 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

133 # Inserting for the first time should return True 

134 self.assertTrue(registry.registerDatasetType(inDatasetType)) 

135 outDatasetType1 = registry.getDatasetType(datasetTypeName) 

136 self.assertEqual(outDatasetType1, inDatasetType) 

137 

138 # Re-inserting should work 

139 self.assertFalse(registry.registerDatasetType(inDatasetType)) 

140 # Except when they are not identical 

141 with self.assertRaises(ConflictingDefinitionError): 

142 nonIdenticalDatasetType = DatasetType(datasetTypeName, differentDimensions, storageClass) 

143 registry.registerDatasetType(nonIdenticalDatasetType) 

144 

145 # Template can be None 

146 datasetTypeName = "testNoneTemplate" 

147 storageClass = StorageClass("testDatasetType2") 

148 registry.storageClasses.registerStorageClass(storageClass) 

149 dimensions = registry.dimensions.extract(("instrument", "visit")) 

150 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

151 registry.registerDatasetType(inDatasetType) 

152 outDatasetType2 = registry.getDatasetType(datasetTypeName) 

153 self.assertEqual(outDatasetType2, inDatasetType) 

154 

155 allTypes = set(registry.queryDatasetTypes()) 

156 self.assertEqual(allTypes, {outDatasetType1, outDatasetType2}) 

157 

158 def testDimensions(self): 

159 """Tests for `Registry.insertDimensionData`, 

160 `Registry.syncDimensionData`, and `Registry.expandDataId`. 

161 """ 

162 registry = self.makeRegistry() 

163 dimensionName = "instrument" 

164 dimension = registry.dimensions[dimensionName] 

165 dimensionValue = {"name": "DummyCam", "visit_max": 10, "exposure_max": 10, "detector_max": 2, 

166 "class_name": "lsst.obs.base.Instrument"} 

167 registry.insertDimensionData(dimensionName, dimensionValue) 

168 # Inserting the same value twice should fail 

169 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

170 registry.insertDimensionData(dimensionName, dimensionValue) 

171 # expandDataId should retrieve the record we just inserted 

172 self.assertEqual( 

173 registry.expandDataId( 

174 instrument="DummyCam", 

175 graph=dimension.graph 

176 ).records[dimensionName].toDict(), 

177 dimensionValue 

178 ) 

179 # expandDataId should raise if there is no record with the given ID. 

180 with self.assertRaises(LookupError): 

181 registry.expandDataId({"instrument": "Unknown"}, graph=dimension.graph) 

182 # abstract_filter doesn't have a table; insert should fail. 

183 with self.assertRaises(TypeError): 

184 registry.insertDimensionData("abstract_filter", {"abstract_filter": "i"}) 

185 dimensionName2 = "physical_filter" 

186 dimension2 = registry.dimensions[dimensionName2] 

187 dimensionValue2 = {"name": "DummyCam_i", "abstract_filter": "i"} 

188 # Missing required dependency ("instrument") should fail 

189 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

190 registry.insertDimensionData(dimensionName2, dimensionValue2) 

191 # Adding required dependency should fix the failure 

192 dimensionValue2["instrument"] = "DummyCam" 

193 registry.insertDimensionData(dimensionName2, dimensionValue2) 

194 # expandDataId should retrieve the record we just inserted. 

195 self.assertEqual( 

196 registry.expandDataId( 

197 instrument="DummyCam", physical_filter="DummyCam_i", 

198 graph=dimension2.graph 

199 ).records[dimensionName2].toDict(), 

200 dimensionValue2 

201 ) 

202 # Use syncDimensionData to insert a new record successfully. 

203 dimensionName3 = "detector" 

204 dimensionValue3 = {"instrument": "DummyCam", "id": 1, "full_name": "one", 

205 "name_in_raft": "zero", "purpose": "SCIENCE"} 

206 self.assertTrue(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

207 # Sync that again. Note that one field ("raft") is NULL, and that 

208 # should be okay. 

209 self.assertFalse(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

210 # Now try that sync with the same primary key but a different value. 

211 # This should fail. 

212 with self.assertRaises(ConflictingDefinitionError): 

213 registry.syncDimensionData( 

214 dimensionName3, 

215 {"instrument": "DummyCam", "id": 1, "full_name": "one", 

216 "name_in_raft": "four", "purpose": "SCIENCE"} 

217 ) 

218 

219 def testDataIdRelationships(self): 

220 """Test `Registry.relateDataId`. 

221 """ 

222 registry = self.makeRegistry() 

223 self.loadData(registry, "base.yaml") 

224 # Simple cases where the dimension key-value pairs tell us everything. 

225 self.assertEqual( 

226 registry.relateDataIds( 

227 {"instrument": "Cam1"}, 

228 {"instrument": "Cam1"}, 

229 ), 

230 ConsistentDataIds(contains=True, within=True, overlaps=True) 

231 ) 

232 self.assertEqual( 

233 registry.relateDataIds({}, {}), 

234 ConsistentDataIds(contains=True, within=True, overlaps=False) 

235 ) 

236 self.assertEqual( 

237 registry.relateDataIds({"instrument": "Cam1"}, {}), 

238 ConsistentDataIds(contains=True, within=False, overlaps=False) 

239 ) 

240 self.assertEqual( 

241 registry.relateDataIds({}, {"instrument": "Cam1"}), 

242 ConsistentDataIds(contains=False, within=True, overlaps=False) 

243 ) 

244 self.assertEqual( 

245 registry.relateDataIds( 

246 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

247 {"instrument": "Cam1"}, 

248 ), 

249 ConsistentDataIds(contains=True, within=False, overlaps=True) 

250 ) 

251 self.assertEqual( 

252 registry.relateDataIds( 

253 {"instrument": "Cam1"}, 

254 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

255 ), 

256 ConsistentDataIds(contains=False, within=True, overlaps=True) 

257 ) 

258 self.assertIsNone( 

259 registry.relateDataIds( 

260 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

261 {"instrument": "Cam1", "physical_filter": "Cam1-R1"}, 

262 ) 

263 ) 

264 # Trickier cases where we need to expand data IDs, but it's still just 

265 # required and implied dimension relationships. 

266 self.assertEqual( 

267 registry.relateDataIds( 

268 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

269 {"instrument": "Cam1", "abstract_filter": "g"}, 

270 ), 

271 ConsistentDataIds(contains=True, within=False, overlaps=True) 

272 ) 

273 self.assertEqual( 

274 registry.relateDataIds( 

275 {"instrument": "Cam1", "abstract_filter": "g"}, 

276 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

277 ), 

278 ConsistentDataIds(contains=False, within=True, overlaps=True) 

279 ) 

280 self.assertEqual( 

281 registry.relateDataIds( 

282 {"instrument": "Cam1"}, 

283 {"htm7": 131073}, 

284 ), 

285 ConsistentDataIds(contains=False, within=False, overlaps=False) 

286 ) 

287 # Trickiest cases involve spatial or temporal overlaps or non-dimension 

288 # elements that relate things (of which visit_definition is our only 

289 # current example). 

290 # 

291 # These two HTM IDs at different levels have a "contains" relationship 

292 # spatially, but there is no overlap of dimension keys. The exact 

293 # result of relateDataIds is unspecified for this case, but it's 

294 # guaranteed to be truthy (see relateDataIds docs.). 

295 self.assertTrue( 

296 registry.relateDataIds({"htm7": 131073}, {"htm9": 2097169}) 

297 ) 

298 # These two HTM IDs at different levels are disjoint spatially, which 

299 # means the data IDs are inconsistent. 

300 self.assertIsNone( 

301 registry.relateDataIds({"htm7": 131073}, {"htm9": 2097391}) 

302 ) 

303 # Insert a few more dimension records for the next test. 

304 registry.insertDimensionData( 

305 "exposure", 

306 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G"}, 

307 ) 

308 registry.insertDimensionData( 

309 "exposure", 

310 {"instrument": "Cam1", "id": 2, "name": "two", "physical_filter": "Cam1-G"}, 

311 ) 

312 registry.insertDimensionData( 

313 "visit_system", 

314 {"instrument": "Cam1", "id": 0, "name": "one-to-one"}, 

315 ) 

316 registry.insertDimensionData( 

317 "visit", 

318 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G", "visit_system": 0}, 

319 ) 

320 registry.insertDimensionData( 

321 "visit_definition", 

322 {"instrument": "Cam1", "visit": 1, "exposure": 1, "visit_system": 0}, 

323 ) 

324 self.assertEqual( 

325 registry.relateDataIds( 

326 {"instrument": "Cam1", "visit": 1}, 

327 {"instrument": "Cam1", "exposure": 1}, 

328 ), 

329 ConsistentDataIds(contains=False, within=False, overlaps=True) 

330 ) 

331 self.assertIsNone( 

332 registry.relateDataIds( 

333 {"instrument": "Cam1", "visit": 1}, 

334 {"instrument": "Cam1", "exposure": 2}, 

335 ) 

336 ) 

337 

338 def testDataset(self): 

339 """Basic tests for `Registry.insertDatasets`, `Registry.getDataset`, 

340 and `Registry.removeDataset`. 

341 """ 

342 registry = self.makeRegistry() 

343 self.loadData(registry, "base.yaml") 

344 run = "test" 

345 registry.registerRun(run) 

346 datasetType = registry.getDatasetType("permabias") 

347 dataId = {"instrument": "Cam1", "detector": 2} 

348 ref, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

349 outRef = registry.getDataset(ref.id) 

350 self.assertIsNotNone(ref.id) 

351 self.assertEqual(ref, outRef) 

352 with self.assertRaises(ConflictingDefinitionError): 

353 registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

354 registry.removeDataset(ref) 

355 self.assertIsNone(registry.findDataset(datasetType, dataId, collections=[run])) 

356 

357 def testComponents(self): 

358 """Tests for `Registry.attachComponent` and other dataset operations 

359 on composite datasets. 

360 """ 

361 registry = self.makeRegistry() 

362 self.loadData(registry, "base.yaml") 

363 run = "test" 

364 registry.registerRun(run) 

365 parentDatasetType = registry.getDatasetType("permabias") 

366 childDatasetType1 = registry.getDatasetType("permabias.image") 

367 childDatasetType2 = registry.getDatasetType("permabias.mask") 

368 dataId = {"instrument": "Cam1", "detector": 2} 

369 parent, = registry.insertDatasets(parentDatasetType, dataIds=[dataId], run=run) 

370 children = {"image": registry.insertDatasets(childDatasetType1, dataIds=[dataId], run=run)[0], 

371 "mask": registry.insertDatasets(childDatasetType2, dataIds=[dataId], run=run)[0]} 

372 for name, child in children.items(): 

373 registry.attachComponent(name, parent, child) 

374 self.assertEqual(parent.components, children) 

375 outParent = registry.getDataset(parent.id) 

376 self.assertEqual(outParent.components, children) 

377 # Remove the parent; this should remove all children. 

378 registry.removeDataset(parent) 

379 self.assertIsNone(registry.findDataset(parentDatasetType, dataId, collections=[run])) 

380 self.assertIsNone(registry.findDataset(childDatasetType1, dataId, collections=[run])) 

381 self.assertIsNone(registry.findDataset(childDatasetType2, dataId, collections=[run])) 

382 

383 def testFindDataset(self): 

384 """Tests for `Registry.findDataset`. 

385 """ 

386 registry = self.makeRegistry() 

387 self.loadData(registry, "base.yaml") 

388 run = "test" 

389 datasetType = registry.getDatasetType("permabias") 

390 dataId = {"instrument": "Cam1", "detector": 4} 

391 registry.registerRun(run) 

392 inputRef, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

393 outputRef = registry.findDataset(datasetType, dataId, collections=[run]) 

394 self.assertEqual(outputRef, inputRef) 

395 # Check that retrieval with invalid dataId raises 

396 with self.assertRaises(LookupError): 

397 dataId = {"instrument": "Cam1"} # no detector 

398 registry.findDataset(datasetType, dataId, collections=run) 

399 # Check that different dataIds match to different datasets 

400 dataId1 = {"instrument": "Cam1", "detector": 1} 

401 inputRef1, = registry.insertDatasets(datasetType, dataIds=[dataId1], run=run) 

402 dataId2 = {"instrument": "Cam1", "detector": 2} 

403 inputRef2, = registry.insertDatasets(datasetType, dataIds=[dataId2], run=run) 

404 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef1) 

405 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef2) 

406 self.assertNotEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef2) 

407 self.assertNotEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef1) 

408 # Check that requesting a non-existing dataId returns None 

409 nonExistingDataId = {"instrument": "Cam1", "detector": 3} 

410 self.assertIsNone(registry.findDataset(datasetType, nonExistingDataId, collections=run)) 

411 

412 def testCollections(self): 

413 """Tests for registry methods that manage collections. 

414 """ 

415 registry = self.makeRegistry() 

416 self.loadData(registry, "base.yaml") 

417 self.loadData(registry, "datasets.yaml") 

418 run1 = "imported_g" 

419 run2 = "imported_r" 

420 datasetType = "permabias" 

421 # Find some datasets via their run's collection. 

422 dataId1 = {"instrument": "Cam1", "detector": 1} 

423 ref1 = registry.findDataset(datasetType, dataId1, collections=run1) 

424 self.assertIsNotNone(ref1) 

425 dataId2 = {"instrument": "Cam1", "detector": 2} 

426 ref2 = registry.findDataset(datasetType, dataId2, collections=run1) 

427 self.assertIsNotNone(ref2) 

428 # Associate those into a new collection,then look for them there. 

429 tag1 = "tag1" 

430 registry.registerCollection(tag1, type=CollectionType.TAGGED) 

431 registry.associate(tag1, [ref1, ref2]) 

432 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

433 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

434 # Disassociate one and verify that we can't it there anymore... 

435 registry.disassociate(tag1, [ref1]) 

436 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=tag1)) 

437 # ...but we can still find ref2 in tag1, and ref1 in the run. 

438 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run1), ref1) 

439 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

440 collections = set(registry.queryCollections()) 

441 self.assertEqual(collections, {run1, run2, tag1}) 

442 # Associate both refs into tag1 again; ref2 is already there, but that 

443 # should be a harmless no-op. 

444 registry.associate(tag1, [ref1, ref2]) 

445 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

446 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

447 # Get a different dataset (from a different run) that has the same 

448 # dataset type and data ID as ref2. 

449 ref2b = registry.findDataset(datasetType, dataId2, collections=run2) 

450 self.assertNotEqual(ref2, ref2b) 

451 # Attempting to associate that into tag1 should be an error. 

452 with self.assertRaises(ConflictingDefinitionError): 

453 registry.associate(tag1, [ref2b]) 

454 # That error shouldn't have messed up what we had before. 

455 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

456 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

457 # Attempt to associate the conflicting dataset again, this time with 

458 # a dataset that isn't in the collection and won't cause a conflict. 

459 # Should also fail without modifying anything. 

460 dataId3 = {"instrument": "Cam1", "detector": 3} 

461 ref3 = registry.findDataset(datasetType, dataId3, collections=run1) 

462 with self.assertRaises(ConflictingDefinitionError): 

463 registry.associate(tag1, [ref3, ref2b]) 

464 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

465 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

466 self.assertIsNone(registry.findDataset(datasetType, dataId3, collections=tag1)) 

467 # Register a chained collection that searches: 

468 # 1. 'tag1' 

469 # 2. 'run1', but only for the permaflat dataset 

470 # 3. 'run2' 

471 chain1 = "chain1" 

472 registry.registerCollection(chain1, type=CollectionType.CHAINED) 

473 self.assertIs(registry.getCollectionType(chain1), CollectionType.CHAINED) 

474 # Chained collection exists, but has no collections in it. 

475 self.assertFalse(registry.getCollectionChain(chain1)) 

476 # If we query for all collections, we should get the chained collection 

477 # only if we don't ask to flatten it (i.e. yield only its children). 

478 self.assertEqual(set(registry.queryCollections(flattenChains=False)), {tag1, run1, run2, chain1}) 

479 self.assertEqual(set(registry.queryCollections(flattenChains=True)), {tag1, run1, run2}) 

480 # Attempt to set its child collections to something circular; that 

481 # should fail. 

482 with self.assertRaises(ValueError): 

483 registry.setCollectionChain(chain1, [tag1, chain1]) 

484 # Add the child collections. 

485 registry.setCollectionChain(chain1, [tag1, (run1, "permaflat"), run2]) 

486 self.assertEqual( 

487 list(registry.getCollectionChain(chain1)), 

488 [(tag1, DatasetTypeRestriction.any), 

489 (run1, DatasetTypeRestriction.fromExpression("permaflat")), 

490 (run2, DatasetTypeRestriction.any)] 

491 ) 

492 # Searching for dataId1 or dataId2 in the chain should return ref1 and 

493 # ref2, because both are in tag1. 

494 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain1), ref1) 

495 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain1), ref2) 

496 # Now disassociate ref2 from tag1. The search (for permabias) with 

497 # dataId2 in chain1 should then: 

498 # 1. not find it in tag1 

499 # 2. not look in tag2, because it's restricted to permaflat here 

500 # 3. find a different dataset in run2 

501 registry.disassociate(tag1, [ref2]) 

502 ref2b = registry.findDataset(datasetType, dataId2, collections=chain1) 

503 self.assertNotEqual(ref2b, ref2) 

504 self.assertEqual(ref2b, registry.findDataset(datasetType, dataId2, collections=run2)) 

505 # Look in the chain for a permaflat that is in run1; should get the 

506 # same ref as if we'd searched run1 directly. 

507 dataId3 = {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"} 

508 self.assertEqual(registry.findDataset("permaflat", dataId3, collections=chain1), 

509 registry.findDataset("permaflat", dataId3, collections=run1),) 

510 # Define a new chain so we can test recursive chains. 

511 chain2 = "chain2" 

512 registry.registerCollection(chain2, type=CollectionType.CHAINED) 

513 registry.setCollectionChain(chain2, [(run2, "permabias"), chain1]) 

514 # Search for permabias with dataId1 should find it via tag1 in chain2, 

515 # recursing, because is not in run1. 

516 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=run2)) 

517 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain2), ref1) 

518 # Search for permabias with dataId2 should find it in run2 (ref2b). 

519 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain2), ref2b) 

520 # Search for a permaflat that is in run2. That should not be found 

521 # at the front of chain2, because of the restriction to permabias 

522 # on run2 there, but it should be found in at the end of chain1. 

523 dataId4 = {"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-R2"} 

524 ref4 = registry.findDataset("permaflat", dataId4, collections=run2) 

525 self.assertIsNotNone(ref4) 

526 self.assertEqual(ref4, registry.findDataset("permaflat", dataId4, collections=chain2)) 

527 

528 def testDatasetLocations(self): 

529 """Tests for `Registry.insertDatasetLocations`, 

530 `Registry.getDatasetLocations`, and `Registry.removeDatasetLocation`. 

531 """ 

532 registry = self.makeRegistry() 

533 self.loadData(registry, "base.yaml") 

534 self.loadData(registry, "datasets.yaml") 

535 run = "imported_g" 

536 ref = registry.findDataset("permabias", dataId={"instrument": "Cam1", "detector": 1}, collections=run) 

537 ref2 = registry.findDataset("permaflat", 

538 dataId={"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-G"}, 

539 collections=run) 

540 datastoreName = "dummystore" 

541 datastoreName2 = "dummystore2" 

542 # Test adding information about a new dataset 

543 registry.insertDatasetLocations(datastoreName, [ref]) 

544 addresses = registry.getDatasetLocations(ref) 

545 self.assertIn(datastoreName, addresses) 

546 self.assertEqual(len(addresses), 1) 

547 registry.insertDatasetLocations(datastoreName2, [ref, ref2]) 

548 addresses = registry.getDatasetLocations(ref) 

549 self.assertEqual(len(addresses), 2) 

550 self.assertIn(datastoreName, addresses) 

551 self.assertIn(datastoreName2, addresses) 

552 registry.removeDatasetLocation(datastoreName, [ref]) 

553 addresses = registry.getDatasetLocations(ref) 

554 self.assertEqual(len(addresses), 1) 

555 self.assertNotIn(datastoreName, addresses) 

556 self.assertIn(datastoreName2, addresses) 

557 with self.assertRaises(OrphanedRecordError): 

558 registry.removeDataset(ref) 

559 registry.removeDatasetLocation(datastoreName2, [ref]) 

560 addresses = registry.getDatasetLocations(ref) 

561 self.assertEqual(len(addresses), 0) 

562 self.assertNotIn(datastoreName2, addresses) 

563 registry.removeDataset(ref) # should not raise 

564 addresses = registry.getDatasetLocations(ref2) 

565 self.assertEqual(len(addresses), 1) 

566 self.assertIn(datastoreName2, addresses) 

567 

568 def testBasicTransaction(self): 

569 """Test that all operations within a single transaction block are 

570 rolled back if an exception propagates out of the block. 

571 """ 

572 registry = self.makeRegistry() 

573 storageClass = StorageClass("testDatasetType") 

574 registry.storageClasses.registerStorageClass(storageClass) 

575 dimensions = registry.dimensions.extract(("instrument",)) 

576 dataId = {"instrument": "DummyCam"} 

577 datasetTypeA = DatasetType(name="A", 

578 dimensions=dimensions, 

579 storageClass=storageClass) 

580 datasetTypeB = DatasetType(name="B", 

581 dimensions=dimensions, 

582 storageClass=storageClass) 

583 datasetTypeC = DatasetType(name="C", 

584 dimensions=dimensions, 

585 storageClass=storageClass) 

586 run = "test" 

587 registry.registerRun(run) 

588 refId = None 

589 with registry.transaction(): 

590 registry.registerDatasetType(datasetTypeA) 

591 with self.assertRaises(ValueError): 

592 with registry.transaction(): 

593 registry.registerDatasetType(datasetTypeB) 

594 registry.registerDatasetType(datasetTypeC) 

595 registry.insertDimensionData("instrument", {"instrument": "DummyCam"}) 

596 ref, = registry.insertDatasets(datasetTypeA, dataIds=[dataId], run=run) 

597 refId = ref.id 

598 raise ValueError("Oops, something went wrong") 

599 # A should exist 

600 self.assertEqual(registry.getDatasetType("A"), datasetTypeA) 

601 # But B and C should both not exist 

602 with self.assertRaises(KeyError): 

603 registry.getDatasetType("B") 

604 with self.assertRaises(KeyError): 

605 registry.getDatasetType("C") 

606 # And neither should the dataset 

607 self.assertIsNotNone(refId) 

608 self.assertIsNone(registry.getDataset(refId)) 

609 # Or the Dimension entries 

610 with self.assertRaises(LookupError): 

611 registry.expandDataId({"instrument": "DummyCam"}) 

612 

613 def testNestedTransaction(self): 

614 """Test that operations within a transaction block are not rolled back 

615 if an exception propagates out of an inner transaction block and is 

616 then caught. 

617 """ 

618 registry = self.makeRegistry() 

619 dimension = registry.dimensions["instrument"] 

620 dataId1 = {"instrument": "DummyCam"} 

621 dataId2 = {"instrument": "DummyCam2"} 

622 checkpointReached = False 

623 with registry.transaction(): 

624 # This should be added and (ultimately) committed. 

625 registry.insertDimensionData(dimension, dataId1) 

626 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

627 with registry.transaction(): 

628 # This does not conflict, and should succeed (but not 

629 # be committed). 

630 registry.insertDimensionData(dimension, dataId2) 

631 checkpointReached = True 

632 # This should conflict and raise, triggerring a rollback 

633 # of the previous insertion within the same transaction 

634 # context, but not the original insertion in the outer 

635 # block. 

636 registry.insertDimensionData(dimension, dataId1) 

637 self.assertTrue(checkpointReached) 

638 self.assertIsNotNone(registry.expandDataId(dataId1, graph=dimension.graph)) 

639 with self.assertRaises(LookupError): 

640 registry.expandDataId(dataId2, graph=dimension.graph) 

641 

642 def testInstrumentDimensions(self): 

643 """Test queries involving only instrument dimensions, with no joins to 

644 skymap.""" 

645 registry = self.makeRegistry() 

646 

647 # need a bunch of dimensions and datasets for test 

648 registry.insertDimensionData( 

649 "instrument", 

650 dict(name="DummyCam", visit_max=25, exposure_max=300, detector_max=6) 

651 ) 

652 registry.insertDimensionData( 

653 "physical_filter", 

654 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

655 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

656 ) 

657 registry.insertDimensionData( 

658 "detector", 

659 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in range(1, 6)] 

660 ) 

661 registry.insertDimensionData( 

662 "visit_system", 

663 dict(instrument="DummyCam", id=1, name="default"), 

664 ) 

665 registry.insertDimensionData( 

666 "visit", 

667 dict(instrument="DummyCam", id=10, name="ten", physical_filter="dummy_i", visit_system=1), 

668 dict(instrument="DummyCam", id=11, name="eleven", physical_filter="dummy_r", visit_system=1), 

669 dict(instrument="DummyCam", id=20, name="twelve", physical_filter="dummy_r", visit_system=1), 

670 ) 

671 registry.insertDimensionData( 

672 "exposure", 

673 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i"), 

674 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i"), 

675 dict(instrument="DummyCam", id=110, name="110", physical_filter="dummy_r"), 

676 dict(instrument="DummyCam", id=111, name="111", physical_filter="dummy_r"), 

677 dict(instrument="DummyCam", id=200, name="200", physical_filter="dummy_r"), 

678 dict(instrument="DummyCam", id=201, name="201", physical_filter="dummy_r"), 

679 ) 

680 registry.insertDimensionData( 

681 "visit_definition", 

682 dict(instrument="DummyCam", exposure=100, visit_system=1, visit=10), 

683 dict(instrument="DummyCam", exposure=101, visit_system=1, visit=10), 

684 dict(instrument="DummyCam", exposure=110, visit_system=1, visit=11), 

685 dict(instrument="DummyCam", exposure=111, visit_system=1, visit=11), 

686 dict(instrument="DummyCam", exposure=200, visit_system=1, visit=20), 

687 dict(instrument="DummyCam", exposure=201, visit_system=1, visit=20), 

688 ) 

689 # dataset types 

690 run1 = "test1_r" 

691 run2 = "test2_r" 

692 tagged2 = "test2_t" 

693 registry.registerRun(run1) 

694 registry.registerRun(run2) 

695 registry.registerCollection(tagged2) 

696 storageClass = StorageClass("testDataset") 

697 registry.storageClasses.registerStorageClass(storageClass) 

698 rawType = DatasetType(name="RAW", 

699 dimensions=registry.dimensions.extract(("instrument", "exposure", "detector")), 

700 storageClass=storageClass) 

701 registry.registerDatasetType(rawType) 

702 calexpType = DatasetType(name="CALEXP", 

703 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

704 storageClass=storageClass) 

705 registry.registerDatasetType(calexpType) 

706 

707 # add pre-existing datasets 

708 for exposure in (100, 101, 110, 111): 

709 for detector in (1, 2, 3): 

710 # note that only 3 of 5 detectors have datasets 

711 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

712 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run1) 

713 # exposures 100 and 101 appear in both run1 and tagged2. 

714 # 100 has different datasets in the different collections 

715 # 101 has the same dataset in both collections. 

716 if exposure == 100: 

717 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

718 if exposure in (100, 101): 

719 registry.associate(tagged2, [ref]) 

720 # Add pre-existing datasets to tagged2. 

721 for exposure in (200, 201): 

722 for detector in (3, 4, 5): 

723 # note that only 3 of 5 detectors have datasets 

724 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

725 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

726 registry.associate(tagged2, [ref]) 

727 

728 dimensions = DimensionGraph( 

729 registry.dimensions, 

730 dimensions=(rawType.dimensions.required | calexpType.dimensions.required) 

731 ) 

732 # Test that single dim string works as well as list of str 

733 rows = list(registry.queryDimensions("visit", datasets=rawType, collections=run1, expand=True)) 

734 rowsI = list(registry.queryDimensions(["visit"], datasets=rawType, collections=run1, expand=True)) 

735 self.assertEqual(rows, rowsI) 

736 # with empty expression 

737 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, expand=True)) 

738 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

739 for dataId in rows: 

740 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

741 packer1 = registry.dimensions.makePacker("visit_detector", dataId) 

742 packer2 = registry.dimensions.makePacker("exposure_detector", dataId) 

743 self.assertEqual(packer1.unpack(packer1.pack(dataId)), 

744 DataCoordinate.standardize(dataId, graph=packer1.dimensions)) 

745 self.assertEqual(packer2.unpack(packer2.pack(dataId)), 

746 DataCoordinate.standardize(dataId, graph=packer2.dimensions)) 

747 self.assertNotEqual(packer1.pack(dataId), packer2.pack(dataId)) 

748 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

749 (100, 101, 110, 111)) 

750 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11)) 

751 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

752 

753 # second collection 

754 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=tagged2)) 

755 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

756 for dataId in rows: 

757 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

758 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

759 (100, 101, 200, 201)) 

760 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 20)) 

761 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

762 

763 # with two input datasets 

764 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=[run1, tagged2])) 

765 self.assertEqual(len(set(rows)), 6*3) # 6 exposures times 3 detectors; set needed to de-dupe 

766 for dataId in rows: 

767 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

768 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

769 (100, 101, 110, 111, 200, 201)) 

770 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11, 20)) 

771 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

772 

773 # limit to single visit 

774 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

775 where="visit = 10")) 

776 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

777 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

778 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

779 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

780 

781 # more limiting expression, using link names instead of Table.column 

782 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

783 where="visit = 10 and detector > 1")) 

784 self.assertEqual(len(rows), 2*2) # 2 exposures times 2 detectors 

785 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

786 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

787 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (2, 3)) 

788 

789 # expression excludes everything 

790 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

791 where="visit > 1000")) 

792 self.assertEqual(len(rows), 0) 

793 

794 # Selecting by physical_filter, this is not in the dimensions, but it 

795 # is a part of the full expression so it should work too. 

796 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

797 where="physical_filter = 'dummy_r'")) 

798 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

799 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (110, 111)) 

800 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (11,)) 

801 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

802 

803 def testSkyMapDimensions(self): 

804 """Tests involving only skymap dimensions, no joins to instrument.""" 

805 registry = self.makeRegistry() 

806 

807 # need a bunch of dimensions and datasets for test, we want 

808 # "abstract_filter" in the test so also have to add physical_filter 

809 # dimensions 

810 registry.insertDimensionData( 

811 "instrument", 

812 dict(instrument="DummyCam") 

813 ) 

814 registry.insertDimensionData( 

815 "physical_filter", 

816 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

817 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

818 ) 

819 registry.insertDimensionData( 

820 "skymap", 

821 dict(name="DummyMap", hash="sha!".encode("utf8")) 

822 ) 

823 for tract in range(10): 

824 registry.insertDimensionData("tract", dict(skymap="DummyMap", id=tract)) 

825 registry.insertDimensionData( 

826 "patch", 

827 *[dict(skymap="DummyMap", tract=tract, id=patch, cell_x=0, cell_y=0) 

828 for patch in range(10)] 

829 ) 

830 

831 # dataset types 

832 run = "test" 

833 registry.registerRun(run) 

834 storageClass = StorageClass("testDataset") 

835 registry.storageClasses.registerStorageClass(storageClass) 

836 calexpType = DatasetType(name="deepCoadd_calexp", 

837 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

838 "abstract_filter")), 

839 storageClass=storageClass) 

840 registry.registerDatasetType(calexpType) 

841 mergeType = DatasetType(name="deepCoadd_mergeDet", 

842 dimensions=registry.dimensions.extract(("skymap", "tract", "patch")), 

843 storageClass=storageClass) 

844 registry.registerDatasetType(mergeType) 

845 measType = DatasetType(name="deepCoadd_meas", 

846 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

847 "abstract_filter")), 

848 storageClass=storageClass) 

849 registry.registerDatasetType(measType) 

850 

851 dimensions = DimensionGraph( 

852 registry.dimensions, 

853 dimensions=(calexpType.dimensions.required | mergeType.dimensions.required 

854 | measType.dimensions.required) 

855 ) 

856 

857 # add pre-existing datasets 

858 for tract in (1, 3, 5): 

859 for patch in (2, 4, 6, 7): 

860 dataId = dict(skymap="DummyMap", tract=tract, patch=patch) 

861 registry.insertDatasets(mergeType, dataIds=[dataId], run=run) 

862 for aFilter in ("i", "r"): 

863 dataId = dict(skymap="DummyMap", tract=tract, patch=patch, abstract_filter=aFilter) 

864 registry.insertDatasets(calexpType, dataIds=[dataId], run=run) 

865 

866 # with empty expression 

867 rows = list(registry.queryDimensions(dimensions, 

868 datasets=[calexpType, mergeType], collections=run)) 

869 self.assertEqual(len(rows), 3*4*2) # 4 tracts x 4 patches x 2 filters 

870 for dataId in rows: 

871 self.assertCountEqual(dataId.keys(), ("skymap", "tract", "patch", "abstract_filter")) 

872 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

873 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

874 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

875 

876 # limit to 2 tracts and 2 patches 

877 rows = list(registry.queryDimensions(dimensions, 

878 datasets=[calexpType, mergeType], collections=run, 

879 where="tract IN (1, 5) AND patch IN (2, 7)")) 

880 self.assertEqual(len(rows), 2*2*2) # 2 tracts x 2 patches x 2 filters 

881 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 5)) 

882 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 7)) 

883 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

884 

885 # limit to single filter 

886 rows = list(registry.queryDimensions(dimensions, 

887 datasets=[calexpType, mergeType], collections=run, 

888 where="abstract_filter = 'i'")) 

889 self.assertEqual(len(rows), 3*4*1) # 4 tracts x 4 patches x 2 filters 

890 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

891 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

892 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i",)) 

893 

894 # expression excludes everything, specifying non-existing skymap is 

895 # not a fatal error, it's operator error 

896 rows = list(registry.queryDimensions(dimensions, 

897 datasets=[calexpType, mergeType], collections=run, 

898 where="skymap = 'Mars'")) 

899 self.assertEqual(len(rows), 0) 

900 

901 def testSpatialMatch(self): 

902 """Test involving spatial match using join tables. 

903 

904 Note that realistic test needs a reasonably-defined skypix and regions 

905 in registry tables which is hard to implement in this simple test. 

906 So we do not actually fill registry with any data and all queries will 

907 return empty result, but this is still useful for coverage of the code 

908 that generates query. 

909 """ 

910 registry = self.makeRegistry() 

911 

912 # dataset types 

913 collection = "test" 

914 registry.registerRun(name=collection) 

915 storageClass = StorageClass("testDataset") 

916 registry.storageClasses.registerStorageClass(storageClass) 

917 

918 calexpType = DatasetType(name="CALEXP", 

919 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

920 storageClass=storageClass) 

921 registry.registerDatasetType(calexpType) 

922 

923 coaddType = DatasetType(name="deepCoadd_calexp", 

924 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

925 "abstract_filter")), 

926 storageClass=storageClass) 

927 registry.registerDatasetType(coaddType) 

928 

929 dimensions = DimensionGraph( 

930 registry.dimensions, 

931 dimensions=(calexpType.dimensions.required | coaddType.dimensions.required) 

932 ) 

933 

934 # without data this should run OK but return empty set 

935 rows = list(registry.queryDimensions(dimensions, datasets=calexpType, collections=collection)) 

936 self.assertEqual(len(rows), 0) 

937 

938 def testCalibrationLabelIndirection(self): 

939 """Test that we can look up datasets with calibration_label dimensions 

940 from a data ID with exposure dimensions. 

941 """ 

942 

943 def _dt(iso_string): 

944 return astropy.time.Time(iso_string, format="iso", scale="utc") 

945 

946 registry = self.makeRegistry() 

947 

948 flat = DatasetType( 

949 "flat", 

950 registry.dimensions.extract( 

951 ["instrument", "detector", "physical_filter", "calibration_label"] 

952 ), 

953 "ImageU" 

954 ) 

955 registry.registerDatasetType(flat) 

956 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

957 registry.insertDimensionData( 

958 "physical_filter", 

959 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

960 ) 

961 registry.insertDimensionData( 

962 "detector", 

963 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in (1, 2, 3, 4, 5)] 

964 ) 

965 registry.insertDimensionData( 

966 "exposure", 

967 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i", 

968 datetime_begin=_dt("2005-12-15 02:00:00"), datetime_end=_dt("2005-12-15 03:00:00")), 

969 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i", 

970 datetime_begin=_dt("2005-12-16 02:00:00"), datetime_end=_dt("2005-12-16 03:00:00")), 

971 ) 

972 registry.insertDimensionData( 

973 "calibration_label", 

974 dict(instrument="DummyCam", name="first_night", 

975 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-15 04:00:00")), 

976 dict(instrument="DummyCam", name="second_night", 

977 datetime_begin=_dt("2005-12-16 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

978 dict(instrument="DummyCam", name="both_nights", 

979 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

980 ) 

981 # Different flats for different nights for detectors 1-3 in first 

982 # collection. 

983 run1 = "calibs1" 

984 registry.registerRun(run1) 

985 for detector in (1, 2, 3): 

986 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="first_night", 

987 physical_filter="dummy_i", detector=detector)], 

988 run=run1) 

989 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="second_night", 

990 physical_filter="dummy_i", detector=detector)], 

991 run=run1) 

992 # The same flat for both nights for detectors 3-5 (so detector 3 has 

993 # multiple valid flats) in second collection. 

994 run2 = "calib2" 

995 registry.registerRun(run2) 

996 for detector in (3, 4, 5): 

997 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="both_nights", 

998 physical_filter="dummy_i", detector=detector)], 

999 run=run2) 

1000 # Perform queries for individual exposure+detector combinations, which 

1001 # should always return exactly one flat. 

1002 for exposure in (100, 101): 

1003 for detector in (1, 2, 3): 

1004 with self.subTest(exposure=exposure, detector=detector): 

1005 rows = list(registry.queryDatasets("flat", collections=[run1], 

1006 instrument="DummyCam", 

1007 exposure=exposure, 

1008 detector=detector)) 

1009 self.assertEqual(len(rows), 1) 

1010 for detector in (3, 4, 5): 

1011 with self.subTest(exposure=exposure, detector=detector): 

1012 rows = registry.queryDatasets("flat", collections=[run2], 

1013 instrument="DummyCam", 

1014 exposure=exposure, 

1015 detector=detector) 

1016 self.assertEqual(len(list(rows)), 1) 

1017 for detector in (1, 2, 4, 5): 

1018 with self.subTest(exposure=exposure, detector=detector): 

1019 rows = registry.queryDatasets("flat", collections=[run1, run2], 

1020 instrument="DummyCam", 

1021 exposure=exposure, 

1022 detector=detector) 

1023 self.assertEqual(len(list(rows)), 1) 

1024 for detector in (3,): 

1025 with self.subTest(exposure=exposure, detector=detector): 

1026 rows = registry.queryDatasets("flat", collections=[run1, run2], 

1027 instrument="DummyCam", 

1028 exposure=exposure, 

1029 detector=detector) 

1030 self.assertEqual(len(list(rows)), 2) 

1031 

1032 def testAbstractFilterQuery(self): 

1033 """Test that we can run a query that just lists the known 

1034 abstract_filters. This is tricky because abstract_filter is 

1035 backed by a query against physical_filter. 

1036 """ 

1037 registry = self.makeRegistry() 

1038 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

1039 registry.insertDimensionData( 

1040 "physical_filter", 

1041 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

1042 dict(instrument="DummyCam", name="dummy_i2", abstract_filter="i"), 

1043 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

1044 ) 

1045 rows = list(registry.queryDimensions(["abstract_filter"])) 

1046 self.assertCountEqual( 

1047 rows, 

1048 [DataCoordinate.standardize(abstract_filter="i", universe=registry.dimensions), 

1049 DataCoordinate.standardize(abstract_filter="r", universe=registry.dimensions)] 

1050 )