Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["RegistryTests"] 

24 

25from abc import ABC, abstractmethod 

26import os 

27 

28import astropy.time 

29import sqlalchemy 

30 

31from ...core import ( 

32 DataCoordinate, 

33 DatasetType, 

34 DimensionGraph, 

35 StorageClass, 

36 ddl, 

37 YamlRepoImportBackend 

38) 

39from .._registry import ( 

40 CollectionType, 

41 ConflictingDefinitionError, 

42 ConsistentDataIds, 

43 OrphanedRecordError, 

44 Registry, 

45) 

46from ..wildcards import DatasetTypeRestriction 

47from ..interfaces import MissingCollectionError 

48 

49 

50class RegistryTests(ABC): 

51 """Generic tests for the `Registry` class that can be subclassed to 

52 generate tests for different configurations. 

53 """ 

54 

55 @classmethod 

56 @abstractmethod 

57 def getDataDir(cls) -> str: 

58 """Return the root directory containing test data YAML files. 

59 """ 

60 raise NotImplementedError() 

61 

62 @abstractmethod 

63 def makeRegistry(self) -> Registry: 

64 """Return the Registry instance to be tested. 

65 """ 

66 raise NotImplementedError() 

67 

68 def loadData(self, registry: Registry, filename: str): 

69 """Load registry test data from ``getDataDir/<filename>``, 

70 which should be a YAML import/export file. 

71 """ 

72 with open(os.path.join(self.getDataDir(), filename), 'r') as stream: 

73 backend = YamlRepoImportBackend(stream, registry) 

74 backend.register() 

75 backend.load(datastore=None) 

76 

77 def assertRowCount(self, registry: Registry, table: str, count: int): 

78 """Check the number of rows in table. 

79 """ 

80 # TODO: all tests that rely on this method should be rewritten, as it 

81 # needs to depend on Registry implementation details to have any chance 

82 # of working. 

83 sql = sqlalchemy.sql.select( 

84 [sqlalchemy.sql.func.count()] 

85 ).select_from( 

86 getattr(registry._tables, table) 

87 ) 

88 self.assertEqual(registry._db.query(sql).scalar(), count) 

89 

90 def testOpaque(self): 

91 """Tests for `Registry.registerOpaqueTable`, 

92 `Registry.insertOpaqueData`, `Registry.fetchOpaqueData`, and 

93 `Registry.deleteOpaqueData`. 

94 """ 

95 registry = self.makeRegistry() 

96 table = "opaque_table_for_testing" 

97 registry.registerOpaqueTable( 

98 table, 

99 spec=ddl.TableSpec( 

100 fields=[ 

101 ddl.FieldSpec("id", dtype=sqlalchemy.BigInteger, primaryKey=True), 

102 ddl.FieldSpec("name", dtype=sqlalchemy.String, length=16, nullable=False), 

103 ddl.FieldSpec("count", dtype=sqlalchemy.SmallInteger, nullable=True), 

104 ], 

105 ) 

106 ) 

107 rows = [ 

108 {"id": 1, "name": "one", "count": None}, 

109 {"id": 2, "name": "two", "count": 5}, 

110 {"id": 3, "name": "three", "count": 6}, 

111 ] 

112 registry.insertOpaqueData(table, *rows) 

113 self.assertCountEqual(rows, list(registry.fetchOpaqueData(table))) 

114 self.assertEqual(rows[0:1], list(registry.fetchOpaqueData(table, id=1))) 

115 self.assertEqual(rows[1:2], list(registry.fetchOpaqueData(table, name="two"))) 

116 self.assertEqual([], list(registry.fetchOpaqueData(table, id=1, name="two"))) 

117 registry.deleteOpaqueData(table, id=3) 

118 self.assertCountEqual(rows[:2], list(registry.fetchOpaqueData(table))) 

119 registry.deleteOpaqueData(table) 

120 self.assertEqual([], list(registry.fetchOpaqueData(table))) 

121 

122 def testDatasetType(self): 

123 """Tests for `Registry.registerDatasetType` and 

124 `Registry.getDatasetType`. 

125 """ 

126 registry = self.makeRegistry() 

127 # Check valid insert 

128 datasetTypeName = "test" 

129 storageClass = StorageClass("testDatasetType") 

130 registry.storageClasses.registerStorageClass(storageClass) 

131 dimensions = registry.dimensions.extract(("instrument", "visit")) 

132 differentDimensions = registry.dimensions.extract(("instrument", "patch")) 

133 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

134 # Inserting for the first time should return True 

135 self.assertTrue(registry.registerDatasetType(inDatasetType)) 

136 outDatasetType1 = registry.getDatasetType(datasetTypeName) 

137 self.assertEqual(outDatasetType1, inDatasetType) 

138 

139 # Re-inserting should work 

140 self.assertFalse(registry.registerDatasetType(inDatasetType)) 

141 # Except when they are not identical 

142 with self.assertRaises(ConflictingDefinitionError): 

143 nonIdenticalDatasetType = DatasetType(datasetTypeName, differentDimensions, storageClass) 

144 registry.registerDatasetType(nonIdenticalDatasetType) 

145 

146 # Template can be None 

147 datasetTypeName = "testNoneTemplate" 

148 storageClass = StorageClass("testDatasetType2") 

149 registry.storageClasses.registerStorageClass(storageClass) 

150 dimensions = registry.dimensions.extract(("instrument", "visit")) 

151 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

152 registry.registerDatasetType(inDatasetType) 

153 outDatasetType2 = registry.getDatasetType(datasetTypeName) 

154 self.assertEqual(outDatasetType2, inDatasetType) 

155 

156 allTypes = set(registry.queryDatasetTypes()) 

157 self.assertEqual(allTypes, {outDatasetType1, outDatasetType2}) 

158 

159 def testDimensions(self): 

160 """Tests for `Registry.insertDimensionData`, 

161 `Registry.syncDimensionData`, and `Registry.expandDataId`. 

162 """ 

163 registry = self.makeRegistry() 

164 dimensionName = "instrument" 

165 dimension = registry.dimensions[dimensionName] 

166 dimensionValue = {"name": "DummyCam", "visit_max": 10, "exposure_max": 10, "detector_max": 2, 

167 "class_name": "lsst.obs.base.Instrument"} 

168 registry.insertDimensionData(dimensionName, dimensionValue) 

169 # Inserting the same value twice should fail 

170 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

171 registry.insertDimensionData(dimensionName, dimensionValue) 

172 # expandDataId should retrieve the record we just inserted 

173 self.assertEqual( 

174 registry.expandDataId( 

175 instrument="DummyCam", 

176 graph=dimension.graph 

177 ).records[dimensionName].toDict(), 

178 dimensionValue 

179 ) 

180 # expandDataId should raise if there is no record with the given ID. 

181 with self.assertRaises(LookupError): 

182 registry.expandDataId({"instrument": "Unknown"}, graph=dimension.graph) 

183 # abstract_filter doesn't have a table; insert should fail. 

184 with self.assertRaises(TypeError): 

185 registry.insertDimensionData("abstract_filter", {"abstract_filter": "i"}) 

186 dimensionName2 = "physical_filter" 

187 dimension2 = registry.dimensions[dimensionName2] 

188 dimensionValue2 = {"name": "DummyCam_i", "abstract_filter": "i"} 

189 # Missing required dependency ("instrument") should fail 

190 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

191 registry.insertDimensionData(dimensionName2, dimensionValue2) 

192 # Adding required dependency should fix the failure 

193 dimensionValue2["instrument"] = "DummyCam" 

194 registry.insertDimensionData(dimensionName2, dimensionValue2) 

195 # expandDataId should retrieve the record we just inserted. 

196 self.assertEqual( 

197 registry.expandDataId( 

198 instrument="DummyCam", physical_filter="DummyCam_i", 

199 graph=dimension2.graph 

200 ).records[dimensionName2].toDict(), 

201 dimensionValue2 

202 ) 

203 # Use syncDimensionData to insert a new record successfully. 

204 dimensionName3 = "detector" 

205 dimensionValue3 = {"instrument": "DummyCam", "id": 1, "full_name": "one", 

206 "name_in_raft": "zero", "purpose": "SCIENCE"} 

207 self.assertTrue(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

208 # Sync that again. Note that one field ("raft") is NULL, and that 

209 # should be okay. 

210 self.assertFalse(registry.syncDimensionData(dimensionName3, dimensionValue3)) 

211 # Now try that sync with the same primary key but a different value. 

212 # This should fail. 

213 with self.assertRaises(ConflictingDefinitionError): 

214 registry.syncDimensionData( 

215 dimensionName3, 

216 {"instrument": "DummyCam", "id": 1, "full_name": "one", 

217 "name_in_raft": "four", "purpose": "SCIENCE"} 

218 ) 

219 

220 def testDataIdRelationships(self): 

221 """Test `Registry.relateDataId`. 

222 """ 

223 registry = self.makeRegistry() 

224 self.loadData(registry, "base.yaml") 

225 # Simple cases where the dimension key-value pairs tell us everything. 

226 self.assertEqual( 

227 registry.relateDataIds( 

228 {"instrument": "Cam1"}, 

229 {"instrument": "Cam1"}, 

230 ), 

231 ConsistentDataIds(contains=True, within=True, overlaps=True) 

232 ) 

233 self.assertEqual( 

234 registry.relateDataIds({}, {}), 

235 ConsistentDataIds(contains=True, within=True, overlaps=False) 

236 ) 

237 self.assertEqual( 

238 registry.relateDataIds({"instrument": "Cam1"}, {}), 

239 ConsistentDataIds(contains=True, within=False, overlaps=False) 

240 ) 

241 self.assertEqual( 

242 registry.relateDataIds({}, {"instrument": "Cam1"}), 

243 ConsistentDataIds(contains=False, within=True, overlaps=False) 

244 ) 

245 self.assertEqual( 

246 registry.relateDataIds( 

247 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

248 {"instrument": "Cam1"}, 

249 ), 

250 ConsistentDataIds(contains=True, within=False, overlaps=True) 

251 ) 

252 self.assertEqual( 

253 registry.relateDataIds( 

254 {"instrument": "Cam1"}, 

255 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

256 ), 

257 ConsistentDataIds(contains=False, within=True, overlaps=True) 

258 ) 

259 self.assertIsNone( 

260 registry.relateDataIds( 

261 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

262 {"instrument": "Cam1", "physical_filter": "Cam1-R1"}, 

263 ) 

264 ) 

265 # Trickier cases where we need to expand data IDs, but it's still just 

266 # required and implied dimension relationships. 

267 self.assertEqual( 

268 registry.relateDataIds( 

269 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

270 {"instrument": "Cam1", "abstract_filter": "g"}, 

271 ), 

272 ConsistentDataIds(contains=True, within=False, overlaps=True) 

273 ) 

274 self.assertEqual( 

275 registry.relateDataIds( 

276 {"instrument": "Cam1", "abstract_filter": "g"}, 

277 {"instrument": "Cam1", "physical_filter": "Cam1-G"}, 

278 ), 

279 ConsistentDataIds(contains=False, within=True, overlaps=True) 

280 ) 

281 self.assertEqual( 

282 registry.relateDataIds( 

283 {"instrument": "Cam1"}, 

284 {"htm7": 131073}, 

285 ), 

286 ConsistentDataIds(contains=False, within=False, overlaps=False) 

287 ) 

288 # Trickiest cases involve spatial or temporal overlaps or non-dimension 

289 # elements that relate things (of which visit_definition is our only 

290 # current example). 

291 # 

292 # These two HTM IDs at different levels have a "contains" relationship 

293 # spatially, but there is no overlap of dimension keys. The exact 

294 # result of relateDataIds is unspecified for this case, but it's 

295 # guaranteed to be truthy (see relateDataIds docs.). 

296 self.assertTrue( 

297 registry.relateDataIds({"htm7": 131073}, {"htm9": 2097169}) 

298 ) 

299 # These two HTM IDs at different levels are disjoint spatially, which 

300 # means the data IDs are inconsistent. 

301 self.assertIsNone( 

302 registry.relateDataIds({"htm7": 131073}, {"htm9": 2097391}) 

303 ) 

304 # Insert a few more dimension records for the next test. 

305 registry.insertDimensionData( 

306 "exposure", 

307 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G"}, 

308 ) 

309 registry.insertDimensionData( 

310 "exposure", 

311 {"instrument": "Cam1", "id": 2, "name": "two", "physical_filter": "Cam1-G"}, 

312 ) 

313 registry.insertDimensionData( 

314 "visit_system", 

315 {"instrument": "Cam1", "id": 0, "name": "one-to-one"}, 

316 ) 

317 registry.insertDimensionData( 

318 "visit", 

319 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G", "visit_system": 0}, 

320 ) 

321 registry.insertDimensionData( 

322 "visit_definition", 

323 {"instrument": "Cam1", "visit": 1, "exposure": 1, "visit_system": 0}, 

324 ) 

325 self.assertEqual( 

326 registry.relateDataIds( 

327 {"instrument": "Cam1", "visit": 1}, 

328 {"instrument": "Cam1", "exposure": 1}, 

329 ), 

330 ConsistentDataIds(contains=False, within=False, overlaps=True) 

331 ) 

332 self.assertIsNone( 

333 registry.relateDataIds( 

334 {"instrument": "Cam1", "visit": 1}, 

335 {"instrument": "Cam1", "exposure": 2}, 

336 ) 

337 ) 

338 

339 def testDataset(self): 

340 """Basic tests for `Registry.insertDatasets`, `Registry.getDataset`, 

341 and `Registry.removeDatasets`. 

342 """ 

343 registry = self.makeRegistry() 

344 self.loadData(registry, "base.yaml") 

345 run = "test" 

346 registry.registerRun(run) 

347 datasetType = registry.getDatasetType("permabias") 

348 dataId = {"instrument": "Cam1", "detector": 2} 

349 ref, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

350 outRef = registry.getDataset(ref.id) 

351 self.assertIsNotNone(ref.id) 

352 self.assertEqual(ref, outRef) 

353 with self.assertRaises(ConflictingDefinitionError): 

354 registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

355 registry.removeDatasets([ref]) 

356 self.assertIsNone(registry.findDataset(datasetType, dataId, collections=[run])) 

357 

358 def testComponents(self): 

359 """Tests for `Registry.attachComponents` and other dataset operations 

360 on composite datasets. 

361 """ 

362 registry = self.makeRegistry() 

363 self.loadData(registry, "base.yaml") 

364 run = "test" 

365 registry.registerRun(run) 

366 parentDatasetType = registry.getDatasetType("permabias") 

367 childDatasetType1 = registry.getDatasetType("permabias.image") 

368 childDatasetType2 = registry.getDatasetType("permabias.mask") 

369 dataId = {"instrument": "Cam1", "detector": 2} 

370 parent, = registry.insertDatasets(parentDatasetType, dataIds=[dataId], run=run) 

371 children = {"image": registry.insertDatasets(childDatasetType1, dataIds=[dataId], run=run)[0], 

372 "mask": registry.insertDatasets(childDatasetType2, dataIds=[dataId], run=run)[0]} 

373 parent = registry.attachComponents(parent, children) 

374 self.assertEqual(parent.components, children) 

375 outParent = registry.getDataset(parent.id) 

376 self.assertEqual(outParent.components, children) 

377 # Remove the parent; this should remove all children. 

378 registry.removeDatasets([parent]) 

379 self.assertIsNone(registry.findDataset(parentDatasetType, dataId, collections=[run])) 

380 self.assertIsNone(registry.findDataset(childDatasetType1, dataId, collections=[run])) 

381 self.assertIsNone(registry.findDataset(childDatasetType2, dataId, collections=[run])) 

382 

383 def testFindDataset(self): 

384 """Tests for `Registry.findDataset`. 

385 """ 

386 registry = self.makeRegistry() 

387 self.loadData(registry, "base.yaml") 

388 run = "test" 

389 datasetType = registry.getDatasetType("permabias") 

390 dataId = {"instrument": "Cam1", "detector": 4} 

391 registry.registerRun(run) 

392 inputRef, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run) 

393 outputRef = registry.findDataset(datasetType, dataId, collections=[run]) 

394 self.assertEqual(outputRef, inputRef) 

395 # Check that retrieval with invalid dataId raises 

396 with self.assertRaises(LookupError): 

397 dataId = {"instrument": "Cam1"} # no detector 

398 registry.findDataset(datasetType, dataId, collections=run) 

399 # Check that different dataIds match to different datasets 

400 dataId1 = {"instrument": "Cam1", "detector": 1} 

401 inputRef1, = registry.insertDatasets(datasetType, dataIds=[dataId1], run=run) 

402 dataId2 = {"instrument": "Cam1", "detector": 2} 

403 inputRef2, = registry.insertDatasets(datasetType, dataIds=[dataId2], run=run) 

404 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef1) 

405 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef2) 

406 self.assertNotEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef2) 

407 self.assertNotEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef1) 

408 # Check that requesting a non-existing dataId returns None 

409 nonExistingDataId = {"instrument": "Cam1", "detector": 3} 

410 self.assertIsNone(registry.findDataset(datasetType, nonExistingDataId, collections=run)) 

411 

412 def testCollections(self): 

413 """Tests for registry methods that manage collections. 

414 """ 

415 registry = self.makeRegistry() 

416 self.loadData(registry, "base.yaml") 

417 self.loadData(registry, "datasets.yaml") 

418 run1 = "imported_g" 

419 run2 = "imported_r" 

420 datasetType = "permabias" 

421 # Find some datasets via their run's collection. 

422 dataId1 = {"instrument": "Cam1", "detector": 1} 

423 ref1 = registry.findDataset(datasetType, dataId1, collections=run1) 

424 self.assertIsNotNone(ref1) 

425 dataId2 = {"instrument": "Cam1", "detector": 2} 

426 ref2 = registry.findDataset(datasetType, dataId2, collections=run1) 

427 self.assertIsNotNone(ref2) 

428 # Associate those into a new collection,then look for them there. 

429 tag1 = "tag1" 

430 registry.registerCollection(tag1, type=CollectionType.TAGGED) 

431 registry.associate(tag1, [ref1, ref2]) 

432 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

433 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

434 # Disassociate one and verify that we can't it there anymore... 

435 registry.disassociate(tag1, [ref1]) 

436 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=tag1)) 

437 # ...but we can still find ref2 in tag1, and ref1 in the run. 

438 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run1), ref1) 

439 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

440 collections = set(registry.queryCollections()) 

441 self.assertEqual(collections, {run1, run2, tag1}) 

442 # Associate both refs into tag1 again; ref2 is already there, but that 

443 # should be a harmless no-op. 

444 registry.associate(tag1, [ref1, ref2]) 

445 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

446 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

447 # Get a different dataset (from a different run) that has the same 

448 # dataset type and data ID as ref2. 

449 ref2b = registry.findDataset(datasetType, dataId2, collections=run2) 

450 self.assertNotEqual(ref2, ref2b) 

451 # Attempting to associate that into tag1 should be an error. 

452 with self.assertRaises(ConflictingDefinitionError): 

453 registry.associate(tag1, [ref2b]) 

454 # That error shouldn't have messed up what we had before. 

455 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

456 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

457 # Attempt to associate the conflicting dataset again, this time with 

458 # a dataset that isn't in the collection and won't cause a conflict. 

459 # Should also fail without modifying anything. 

460 dataId3 = {"instrument": "Cam1", "detector": 3} 

461 ref3 = registry.findDataset(datasetType, dataId3, collections=run1) 

462 with self.assertRaises(ConflictingDefinitionError): 

463 registry.associate(tag1, [ref3, ref2b]) 

464 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1) 

465 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2) 

466 self.assertIsNone(registry.findDataset(datasetType, dataId3, collections=tag1)) 

467 # Register a chained collection that searches: 

468 # 1. 'tag1' 

469 # 2. 'run1', but only for the permaflat dataset 

470 # 3. 'run2' 

471 chain1 = "chain1" 

472 registry.registerCollection(chain1, type=CollectionType.CHAINED) 

473 self.assertIs(registry.getCollectionType(chain1), CollectionType.CHAINED) 

474 # Chained collection exists, but has no collections in it. 

475 self.assertFalse(registry.getCollectionChain(chain1)) 

476 # If we query for all collections, we should get the chained collection 

477 # only if we don't ask to flatten it (i.e. yield only its children). 

478 self.assertEqual(set(registry.queryCollections(flattenChains=False)), {tag1, run1, run2, chain1}) 

479 self.assertEqual(set(registry.queryCollections(flattenChains=True)), {tag1, run1, run2}) 

480 # Attempt to set its child collections to something circular; that 

481 # should fail. 

482 with self.assertRaises(ValueError): 

483 registry.setCollectionChain(chain1, [tag1, chain1]) 

484 # Add the child collections. 

485 registry.setCollectionChain(chain1, [tag1, (run1, "permaflat"), run2]) 

486 self.assertEqual( 

487 list(registry.getCollectionChain(chain1)), 

488 [(tag1, DatasetTypeRestriction.any), 

489 (run1, DatasetTypeRestriction.fromExpression("permaflat")), 

490 (run2, DatasetTypeRestriction.any)] 

491 ) 

492 # Searching for dataId1 or dataId2 in the chain should return ref1 and 

493 # ref2, because both are in tag1. 

494 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain1), ref1) 

495 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain1), ref2) 

496 # Now disassociate ref2 from tag1. The search (for permabias) with 

497 # dataId2 in chain1 should then: 

498 # 1. not find it in tag1 

499 # 2. not look in tag2, because it's restricted to permaflat here 

500 # 3. find a different dataset in run2 

501 registry.disassociate(tag1, [ref2]) 

502 ref2b = registry.findDataset(datasetType, dataId2, collections=chain1) 

503 self.assertNotEqual(ref2b, ref2) 

504 self.assertEqual(ref2b, registry.findDataset(datasetType, dataId2, collections=run2)) 

505 # Look in the chain for a permaflat that is in run1; should get the 

506 # same ref as if we'd searched run1 directly. 

507 dataId3 = {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"} 

508 self.assertEqual(registry.findDataset("permaflat", dataId3, collections=chain1), 

509 registry.findDataset("permaflat", dataId3, collections=run1),) 

510 # Define a new chain so we can test recursive chains. 

511 chain2 = "chain2" 

512 registry.registerCollection(chain2, type=CollectionType.CHAINED) 

513 registry.setCollectionChain(chain2, [(run2, "permabias"), chain1]) 

514 # Search for permabias with dataId1 should find it via tag1 in chain2, 

515 # recursing, because is not in run1. 

516 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=run2)) 

517 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain2), ref1) 

518 # Search for permabias with dataId2 should find it in run2 (ref2b). 

519 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain2), ref2b) 

520 # Search for a permaflat that is in run2. That should not be found 

521 # at the front of chain2, because of the restriction to permabias 

522 # on run2 there, but it should be found in at the end of chain1. 

523 dataId4 = {"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-R2"} 

524 ref4 = registry.findDataset("permaflat", dataId4, collections=run2) 

525 self.assertIsNotNone(ref4) 

526 self.assertEqual(ref4, registry.findDataset("permaflat", dataId4, collections=chain2)) 

527 # Deleting a collection that's part of a CHAINED collection is not 

528 # allowed, and is exception-safe. 

529 with self.assertRaises(Exception): 

530 registry.removeCollection(run2) 

531 self.assertEqual(registry.getCollectionType(run2), CollectionType.RUN) 

532 with self.assertRaises(Exception): 

533 registry.removeCollection(chain1) 

534 self.assertEqual(registry.getCollectionType(chain1), CollectionType.CHAINED) 

535 # Actually remove chain2, test that it's gone by asking for its type. 

536 registry.removeCollection(chain2) 

537 with self.assertRaises(MissingCollectionError): 

538 registry.getCollectionType(chain2) 

539 # Actually remove run2 and chain1, which should work now. 

540 registry.removeCollection(chain1) 

541 registry.removeCollection(run2) 

542 with self.assertRaises(MissingCollectionError): 

543 registry.getCollectionType(run2) 

544 with self.assertRaises(MissingCollectionError): 

545 registry.getCollectionType(chain1) 

546 # Remove tag1 as well, just to test that we can remove TAGGED 

547 # collections. 

548 registry.removeCollection(tag1) 

549 with self.assertRaises(MissingCollectionError): 

550 registry.getCollectionType(tag1) 

551 

552 def testDatasetLocations(self): 

553 """Tests for `Registry.insertDatasetLocations`, 

554 `Registry.getDatasetLocations`, and `Registry.removeDatasetLocation`. 

555 """ 

556 registry = self.makeRegistry() 

557 self.loadData(registry, "base.yaml") 

558 self.loadData(registry, "datasets.yaml") 

559 run = "imported_g" 

560 ref = registry.findDataset("permabias", dataId={"instrument": "Cam1", "detector": 1}, collections=run) 

561 ref2 = registry.findDataset("permaflat", 

562 dataId={"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-G"}, 

563 collections=run) 

564 datastoreName = "dummystore" 

565 datastoreName2 = "dummystore2" 

566 # Test adding information about a new dataset 

567 registry.insertDatasetLocations(datastoreName, [ref]) 

568 addresses = registry.getDatasetLocations(ref) 

569 self.assertIn(datastoreName, addresses) 

570 self.assertEqual(len(addresses), 1) 

571 registry.insertDatasetLocations(datastoreName2, [ref, ref2]) 

572 addresses = registry.getDatasetLocations(ref) 

573 self.assertEqual(len(addresses), 2) 

574 self.assertIn(datastoreName, addresses) 

575 self.assertIn(datastoreName2, addresses) 

576 registry.removeDatasetLocation(datastoreName, [ref]) 

577 addresses = registry.getDatasetLocations(ref) 

578 self.assertEqual(len(addresses), 1) 

579 self.assertNotIn(datastoreName, addresses) 

580 self.assertIn(datastoreName2, addresses) 

581 with self.assertRaises(OrphanedRecordError): 

582 registry.removeDatasets([ref]) 

583 registry.removeDatasetLocation(datastoreName2, [ref]) 

584 addresses = registry.getDatasetLocations(ref) 

585 self.assertEqual(len(addresses), 0) 

586 self.assertNotIn(datastoreName2, addresses) 

587 registry.removeDatasets([ref]) # should not raise 

588 addresses = registry.getDatasetLocations(ref2) 

589 self.assertEqual(len(addresses), 1) 

590 self.assertIn(datastoreName2, addresses) 

591 

592 def testBasicTransaction(self): 

593 """Test that all operations within a single transaction block are 

594 rolled back if an exception propagates out of the block. 

595 """ 

596 registry = self.makeRegistry() 

597 storageClass = StorageClass("testDatasetType") 

598 registry.storageClasses.registerStorageClass(storageClass) 

599 with registry.transaction(): 

600 registry.insertDimensionData("instrument", {"name": "Cam1", "class_name": "A"}) 

601 with self.assertRaises(ValueError): 

602 with registry.transaction(): 

603 registry.insertDimensionData("instrument", {"name": "Cam2"}) 

604 raise ValueError("Oops, something went wrong") 

605 # Cam1 should exist 

606 self.assertEqual(registry.expandDataId(instrument="Cam1").records["instrument"].class_name, "A") 

607 # But Cam2 and Cam3 should both not exist 

608 with self.assertRaises(LookupError): 

609 registry.expandDataId(instrument="Cam2") 

610 with self.assertRaises(LookupError): 

611 registry.expandDataId(instrument="Cam3") 

612 

613 def testNestedTransaction(self): 

614 """Test that operations within a transaction block are not rolled back 

615 if an exception propagates out of an inner transaction block and is 

616 then caught. 

617 """ 

618 registry = self.makeRegistry() 

619 dimension = registry.dimensions["instrument"] 

620 dataId1 = {"instrument": "DummyCam"} 

621 dataId2 = {"instrument": "DummyCam2"} 

622 checkpointReached = False 

623 with registry.transaction(): 

624 # This should be added and (ultimately) committed. 

625 registry.insertDimensionData(dimension, dataId1) 

626 with self.assertRaises(sqlalchemy.exc.IntegrityError): 

627 with registry.transaction(): 

628 # This does not conflict, and should succeed (but not 

629 # be committed). 

630 registry.insertDimensionData(dimension, dataId2) 

631 checkpointReached = True 

632 # This should conflict and raise, triggerring a rollback 

633 # of the previous insertion within the same transaction 

634 # context, but not the original insertion in the outer 

635 # block. 

636 registry.insertDimensionData(dimension, dataId1) 

637 self.assertTrue(checkpointReached) 

638 self.assertIsNotNone(registry.expandDataId(dataId1, graph=dimension.graph)) 

639 with self.assertRaises(LookupError): 

640 registry.expandDataId(dataId2, graph=dimension.graph) 

641 

642 def testInstrumentDimensions(self): 

643 """Test queries involving only instrument dimensions, with no joins to 

644 skymap.""" 

645 registry = self.makeRegistry() 

646 

647 # need a bunch of dimensions and datasets for test 

648 registry.insertDimensionData( 

649 "instrument", 

650 dict(name="DummyCam", visit_max=25, exposure_max=300, detector_max=6) 

651 ) 

652 registry.insertDimensionData( 

653 "physical_filter", 

654 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

655 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

656 ) 

657 registry.insertDimensionData( 

658 "detector", 

659 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in range(1, 6)] 

660 ) 

661 registry.insertDimensionData( 

662 "visit_system", 

663 dict(instrument="DummyCam", id=1, name="default"), 

664 ) 

665 registry.insertDimensionData( 

666 "visit", 

667 dict(instrument="DummyCam", id=10, name="ten", physical_filter="dummy_i", visit_system=1), 

668 dict(instrument="DummyCam", id=11, name="eleven", physical_filter="dummy_r", visit_system=1), 

669 dict(instrument="DummyCam", id=20, name="twelve", physical_filter="dummy_r", visit_system=1), 

670 ) 

671 registry.insertDimensionData( 

672 "exposure", 

673 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i"), 

674 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i"), 

675 dict(instrument="DummyCam", id=110, name="110", physical_filter="dummy_r"), 

676 dict(instrument="DummyCam", id=111, name="111", physical_filter="dummy_r"), 

677 dict(instrument="DummyCam", id=200, name="200", physical_filter="dummy_r"), 

678 dict(instrument="DummyCam", id=201, name="201", physical_filter="dummy_r"), 

679 ) 

680 registry.insertDimensionData( 

681 "visit_definition", 

682 dict(instrument="DummyCam", exposure=100, visit_system=1, visit=10), 

683 dict(instrument="DummyCam", exposure=101, visit_system=1, visit=10), 

684 dict(instrument="DummyCam", exposure=110, visit_system=1, visit=11), 

685 dict(instrument="DummyCam", exposure=111, visit_system=1, visit=11), 

686 dict(instrument="DummyCam", exposure=200, visit_system=1, visit=20), 

687 dict(instrument="DummyCam", exposure=201, visit_system=1, visit=20), 

688 ) 

689 # dataset types 

690 run1 = "test1_r" 

691 run2 = "test2_r" 

692 tagged2 = "test2_t" 

693 registry.registerRun(run1) 

694 registry.registerRun(run2) 

695 registry.registerCollection(tagged2) 

696 storageClass = StorageClass("testDataset") 

697 registry.storageClasses.registerStorageClass(storageClass) 

698 rawType = DatasetType(name="RAW", 

699 dimensions=registry.dimensions.extract(("instrument", "exposure", "detector")), 

700 storageClass=storageClass) 

701 registry.registerDatasetType(rawType) 

702 calexpType = DatasetType(name="CALEXP", 

703 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

704 storageClass=storageClass) 

705 registry.registerDatasetType(calexpType) 

706 

707 # add pre-existing datasets 

708 for exposure in (100, 101, 110, 111): 

709 for detector in (1, 2, 3): 

710 # note that only 3 of 5 detectors have datasets 

711 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

712 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run1) 

713 # exposures 100 and 101 appear in both run1 and tagged2. 

714 # 100 has different datasets in the different collections 

715 # 101 has the same dataset in both collections. 

716 if exposure == 100: 

717 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

718 if exposure in (100, 101): 

719 registry.associate(tagged2, [ref]) 

720 # Add pre-existing datasets to tagged2. 

721 for exposure in (200, 201): 

722 for detector in (3, 4, 5): 

723 # note that only 3 of 5 detectors have datasets 

724 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector) 

725 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2) 

726 registry.associate(tagged2, [ref]) 

727 

728 dimensions = DimensionGraph( 

729 registry.dimensions, 

730 dimensions=(rawType.dimensions.required | calexpType.dimensions.required) 

731 ) 

732 # Test that single dim string works as well as list of str 

733 rows = list(registry.queryDimensions("visit", datasets=rawType, collections=run1, expand=True)) 

734 rowsI = list(registry.queryDimensions(["visit"], datasets=rawType, collections=run1, expand=True)) 

735 self.assertEqual(rows, rowsI) 

736 # with empty expression 

737 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, expand=True)) 

738 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

739 for dataId in rows: 

740 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

741 packer1 = registry.dimensions.makePacker("visit_detector", dataId) 

742 packer2 = registry.dimensions.makePacker("exposure_detector", dataId) 

743 self.assertEqual(packer1.unpack(packer1.pack(dataId)), 

744 DataCoordinate.standardize(dataId, graph=packer1.dimensions)) 

745 self.assertEqual(packer2.unpack(packer2.pack(dataId)), 

746 DataCoordinate.standardize(dataId, graph=packer2.dimensions)) 

747 self.assertNotEqual(packer1.pack(dataId), packer2.pack(dataId)) 

748 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

749 (100, 101, 110, 111)) 

750 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11)) 

751 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

752 

753 # second collection 

754 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=tagged2)) 

755 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors 

756 for dataId in rows: 

757 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

758 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

759 (100, 101, 200, 201)) 

760 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 20)) 

761 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

762 

763 # with two input datasets 

764 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=[run1, tagged2])) 

765 self.assertEqual(len(set(rows)), 6*3) # 6 exposures times 3 detectors; set needed to de-dupe 

766 for dataId in rows: 

767 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit")) 

768 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), 

769 (100, 101, 110, 111, 200, 201)) 

770 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11, 20)) 

771 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5)) 

772 

773 # limit to single visit 

774 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

775 where="visit = 10")) 

776 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

777 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

778 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

779 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

780 

781 # more limiting expression, using link names instead of Table.column 

782 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

783 where="visit = 10 and detector > 1")) 

784 self.assertEqual(len(rows), 2*2) # 2 exposures times 2 detectors 

785 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101)) 

786 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,)) 

787 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (2, 3)) 

788 

789 # expression excludes everything 

790 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

791 where="visit > 1000")) 

792 self.assertEqual(len(rows), 0) 

793 

794 # Selecting by physical_filter, this is not in the dimensions, but it 

795 # is a part of the full expression so it should work too. 

796 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, 

797 where="physical_filter = 'dummy_r'")) 

798 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors 

799 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (110, 111)) 

800 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (11,)) 

801 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3)) 

802 

803 def testSkyMapDimensions(self): 

804 """Tests involving only skymap dimensions, no joins to instrument.""" 

805 registry = self.makeRegistry() 

806 

807 # need a bunch of dimensions and datasets for test, we want 

808 # "abstract_filter" in the test so also have to add physical_filter 

809 # dimensions 

810 registry.insertDimensionData( 

811 "instrument", 

812 dict(instrument="DummyCam") 

813 ) 

814 registry.insertDimensionData( 

815 "physical_filter", 

816 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

817 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

818 ) 

819 registry.insertDimensionData( 

820 "skymap", 

821 dict(name="DummyMap", hash="sha!".encode("utf8")) 

822 ) 

823 for tract in range(10): 

824 registry.insertDimensionData("tract", dict(skymap="DummyMap", id=tract)) 

825 registry.insertDimensionData( 

826 "patch", 

827 *[dict(skymap="DummyMap", tract=tract, id=patch, cell_x=0, cell_y=0) 

828 for patch in range(10)] 

829 ) 

830 

831 # dataset types 

832 run = "test" 

833 registry.registerRun(run) 

834 storageClass = StorageClass("testDataset") 

835 registry.storageClasses.registerStorageClass(storageClass) 

836 calexpType = DatasetType(name="deepCoadd_calexp", 

837 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

838 "abstract_filter")), 

839 storageClass=storageClass) 

840 registry.registerDatasetType(calexpType) 

841 mergeType = DatasetType(name="deepCoadd_mergeDet", 

842 dimensions=registry.dimensions.extract(("skymap", "tract", "patch")), 

843 storageClass=storageClass) 

844 registry.registerDatasetType(mergeType) 

845 measType = DatasetType(name="deepCoadd_meas", 

846 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

847 "abstract_filter")), 

848 storageClass=storageClass) 

849 registry.registerDatasetType(measType) 

850 

851 dimensions = DimensionGraph( 

852 registry.dimensions, 

853 dimensions=(calexpType.dimensions.required | mergeType.dimensions.required 

854 | measType.dimensions.required) 

855 ) 

856 

857 # add pre-existing datasets 

858 for tract in (1, 3, 5): 

859 for patch in (2, 4, 6, 7): 

860 dataId = dict(skymap="DummyMap", tract=tract, patch=patch) 

861 registry.insertDatasets(mergeType, dataIds=[dataId], run=run) 

862 for aFilter in ("i", "r"): 

863 dataId = dict(skymap="DummyMap", tract=tract, patch=patch, abstract_filter=aFilter) 

864 registry.insertDatasets(calexpType, dataIds=[dataId], run=run) 

865 

866 # with empty expression 

867 rows = list(registry.queryDimensions(dimensions, 

868 datasets=[calexpType, mergeType], collections=run)) 

869 self.assertEqual(len(rows), 3*4*2) # 4 tracts x 4 patches x 2 filters 

870 for dataId in rows: 

871 self.assertCountEqual(dataId.keys(), ("skymap", "tract", "patch", "abstract_filter")) 

872 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

873 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

874 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

875 

876 # limit to 2 tracts and 2 patches 

877 rows = list(registry.queryDimensions(dimensions, 

878 datasets=[calexpType, mergeType], collections=run, 

879 where="tract IN (1, 5) AND patch IN (2, 7)")) 

880 self.assertEqual(len(rows), 2*2*2) # 2 tracts x 2 patches x 2 filters 

881 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 5)) 

882 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 7)) 

883 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) 

884 

885 # limit to single filter 

886 rows = list(registry.queryDimensions(dimensions, 

887 datasets=[calexpType, mergeType], collections=run, 

888 where="abstract_filter = 'i'")) 

889 self.assertEqual(len(rows), 3*4*1) # 4 tracts x 4 patches x 2 filters 

890 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) 

891 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) 

892 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i",)) 

893 

894 # expression excludes everything, specifying non-existing skymap is 

895 # not a fatal error, it's operator error 

896 rows = list(registry.queryDimensions(dimensions, 

897 datasets=[calexpType, mergeType], collections=run, 

898 where="skymap = 'Mars'")) 

899 self.assertEqual(len(rows), 0) 

900 

901 def testSpatialMatch(self): 

902 """Test involving spatial match using join tables. 

903 

904 Note that realistic test needs a reasonably-defined skypix and regions 

905 in registry tables which is hard to implement in this simple test. 

906 So we do not actually fill registry with any data and all queries will 

907 return empty result, but this is still useful for coverage of the code 

908 that generates query. 

909 """ 

910 registry = self.makeRegistry() 

911 

912 # dataset types 

913 collection = "test" 

914 registry.registerRun(name=collection) 

915 storageClass = StorageClass("testDataset") 

916 registry.storageClasses.registerStorageClass(storageClass) 

917 

918 calexpType = DatasetType(name="CALEXP", 

919 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")), 

920 storageClass=storageClass) 

921 registry.registerDatasetType(calexpType) 

922 

923 coaddType = DatasetType(name="deepCoadd_calexp", 

924 dimensions=registry.dimensions.extract(("skymap", "tract", "patch", 

925 "abstract_filter")), 

926 storageClass=storageClass) 

927 registry.registerDatasetType(coaddType) 

928 

929 dimensions = DimensionGraph( 

930 registry.dimensions, 

931 dimensions=(calexpType.dimensions.required | coaddType.dimensions.required) 

932 ) 

933 

934 # without data this should run OK but return empty set 

935 rows = list(registry.queryDimensions(dimensions, datasets=calexpType, collections=collection)) 

936 self.assertEqual(len(rows), 0) 

937 

938 def testCalibrationLabelIndirection(self): 

939 """Test that we can look up datasets with calibration_label dimensions 

940 from a data ID with exposure dimensions. 

941 """ 

942 

943 def _dt(iso_string): 

944 return astropy.time.Time(iso_string, format="iso", scale="utc") 

945 

946 registry = self.makeRegistry() 

947 

948 flat = DatasetType( 

949 "flat", 

950 registry.dimensions.extract( 

951 ["instrument", "detector", "physical_filter", "calibration_label"] 

952 ), 

953 "ImageU" 

954 ) 

955 registry.registerDatasetType(flat) 

956 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

957 registry.insertDimensionData( 

958 "physical_filter", 

959 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

960 ) 

961 registry.insertDimensionData( 

962 "detector", 

963 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in (1, 2, 3, 4, 5)] 

964 ) 

965 registry.insertDimensionData( 

966 "exposure", 

967 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i", 

968 datetime_begin=_dt("2005-12-15 02:00:00"), datetime_end=_dt("2005-12-15 03:00:00")), 

969 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i", 

970 datetime_begin=_dt("2005-12-16 02:00:00"), datetime_end=_dt("2005-12-16 03:00:00")), 

971 ) 

972 registry.insertDimensionData( 

973 "calibration_label", 

974 dict(instrument="DummyCam", name="first_night", 

975 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-15 04:00:00")), 

976 dict(instrument="DummyCam", name="second_night", 

977 datetime_begin=_dt("2005-12-16 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

978 dict(instrument="DummyCam", name="both_nights", 

979 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")), 

980 ) 

981 # Different flats for different nights for detectors 1-3 in first 

982 # collection. 

983 run1 = "calibs1" 

984 registry.registerRun(run1) 

985 for detector in (1, 2, 3): 

986 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="first_night", 

987 physical_filter="dummy_i", detector=detector)], 

988 run=run1) 

989 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="second_night", 

990 physical_filter="dummy_i", detector=detector)], 

991 run=run1) 

992 # The same flat for both nights for detectors 3-5 (so detector 3 has 

993 # multiple valid flats) in second collection. 

994 run2 = "calib2" 

995 registry.registerRun(run2) 

996 for detector in (3, 4, 5): 

997 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="both_nights", 

998 physical_filter="dummy_i", detector=detector)], 

999 run=run2) 

1000 # Perform queries for individual exposure+detector combinations, which 

1001 # should always return exactly one flat. 

1002 for exposure in (100, 101): 

1003 for detector in (1, 2, 3): 

1004 with self.subTest(exposure=exposure, detector=detector): 

1005 rows = list(registry.queryDatasets("flat", collections=[run1], 

1006 instrument="DummyCam", 

1007 exposure=exposure, 

1008 detector=detector)) 

1009 self.assertEqual(len(rows), 1) 

1010 for detector in (3, 4, 5): 

1011 with self.subTest(exposure=exposure, detector=detector): 

1012 rows = registry.queryDatasets("flat", collections=[run2], 

1013 instrument="DummyCam", 

1014 exposure=exposure, 

1015 detector=detector) 

1016 self.assertEqual(len(list(rows)), 1) 

1017 for detector in (1, 2, 4, 5): 

1018 with self.subTest(exposure=exposure, detector=detector): 

1019 rows = registry.queryDatasets("flat", collections=[run1, run2], 

1020 instrument="DummyCam", 

1021 exposure=exposure, 

1022 detector=detector) 

1023 self.assertEqual(len(list(rows)), 1) 

1024 for detector in (3,): 

1025 with self.subTest(exposure=exposure, detector=detector): 

1026 rows = registry.queryDatasets("flat", collections=[run1, run2], 

1027 instrument="DummyCam", 

1028 exposure=exposure, 

1029 detector=detector) 

1030 self.assertEqual(len(list(rows)), 2) 

1031 

1032 def testAbstractFilterQuery(self): 

1033 """Test that we can run a query that just lists the known 

1034 abstract_filters. This is tricky because abstract_filter is 

1035 backed by a query against physical_filter. 

1036 """ 

1037 registry = self.makeRegistry() 

1038 registry.insertDimensionData("instrument", dict(name="DummyCam")) 

1039 registry.insertDimensionData( 

1040 "physical_filter", 

1041 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"), 

1042 dict(instrument="DummyCam", name="dummy_i2", abstract_filter="i"), 

1043 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"), 

1044 ) 

1045 rows = list(registry.queryDimensions(["abstract_filter"])) 

1046 self.assertCountEqual( 

1047 rows, 

1048 [DataCoordinate.standardize(abstract_filter="i", universe=registry.dimensions), 

1049 DataCoordinate.standardize(abstract_filter="r", universe=registry.dimensions)] 

1050 )