Coverage for python/lsst/daf/butler/registry/tests/_registry.py : 5%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = ["RegistryTests"]
25from abc import ABC, abstractmethod
26import os
28import astropy.time
29import sqlalchemy
31from ...core import (
32 DataCoordinate,
33 DatasetType,
34 DimensionGraph,
35 StorageClass,
36 ddl,
37 YamlRepoImportBackend
38)
39from .._registry import (
40 CollectionType,
41 ConflictingDefinitionError,
42 ConsistentDataIds,
43 OrphanedRecordError,
44 Registry,
45)
46from ..wildcards import DatasetTypeRestriction
49class RegistryTests(ABC):
50 """Generic tests for the `Registry` class that can be subclassed to
51 generate tests for different configurations.
52 """
54 @classmethod
55 @abstractmethod
56 def getDataDir(cls) -> str:
57 """Return the root directory containing test data YAML files.
58 """
59 raise NotImplementedError()
61 @abstractmethod
62 def makeRegistry(self) -> Registry:
63 """Return the Registry instance to be tested.
64 """
65 raise NotImplementedError()
67 def loadData(self, registry: Registry, filename: str):
68 """Load registry test data from ``getDataDir/<filename>``,
69 which should be a YAML import/export file.
70 """
71 with open(os.path.join(self.getDataDir(), filename), 'r') as stream:
72 backend = YamlRepoImportBackend(stream, registry)
73 backend.register()
74 backend.load(datastore=None)
76 def assertRowCount(self, registry: Registry, table: str, count: int):
77 """Check the number of rows in table.
78 """
79 # TODO: all tests that rely on this method should be rewritten, as it
80 # needs to depend on Registry implementation details to have any chance
81 # of working.
82 sql = sqlalchemy.sql.select(
83 [sqlalchemy.sql.func.count()]
84 ).select_from(
85 getattr(registry._tables, table)
86 )
87 self.assertEqual(registry._db.query(sql).scalar(), count)
89 def testOpaque(self):
90 """Tests for `Registry.registerOpaqueTable`,
91 `Registry.insertOpaqueData`, `Registry.fetchOpaqueData`, and
92 `Registry.deleteOpaqueData`.
93 """
94 registry = self.makeRegistry()
95 table = "opaque_table_for_testing"
96 registry.registerOpaqueTable(
97 table,
98 spec=ddl.TableSpec(
99 fields=[
100 ddl.FieldSpec("id", dtype=sqlalchemy.BigInteger, primaryKey=True),
101 ddl.FieldSpec("name", dtype=sqlalchemy.String, length=16, nullable=False),
102 ddl.FieldSpec("count", dtype=sqlalchemy.SmallInteger, nullable=True),
103 ],
104 )
105 )
106 rows = [
107 {"id": 1, "name": "one", "count": None},
108 {"id": 2, "name": "two", "count": 5},
109 {"id": 3, "name": "three", "count": 6},
110 ]
111 registry.insertOpaqueData(table, *rows)
112 self.assertCountEqual(rows, list(registry.fetchOpaqueData(table)))
113 self.assertEqual(rows[0:1], list(registry.fetchOpaqueData(table, id=1)))
114 self.assertEqual(rows[1:2], list(registry.fetchOpaqueData(table, name="two")))
115 self.assertEqual([], list(registry.fetchOpaqueData(table, id=1, name="two")))
116 registry.deleteOpaqueData(table, id=3)
117 self.assertCountEqual(rows[:2], list(registry.fetchOpaqueData(table)))
118 registry.deleteOpaqueData(table)
119 self.assertEqual([], list(registry.fetchOpaqueData(table)))
121 def testDatasetType(self):
122 """Tests for `Registry.registerDatasetType` and
123 `Registry.getDatasetType`.
124 """
125 registry = self.makeRegistry()
126 # Check valid insert
127 datasetTypeName = "test"
128 storageClass = StorageClass("testDatasetType")
129 registry.storageClasses.registerStorageClass(storageClass)
130 dimensions = registry.dimensions.extract(("instrument", "visit"))
131 differentDimensions = registry.dimensions.extract(("instrument", "patch"))
132 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass)
133 # Inserting for the first time should return True
134 self.assertTrue(registry.registerDatasetType(inDatasetType))
135 outDatasetType1 = registry.getDatasetType(datasetTypeName)
136 self.assertEqual(outDatasetType1, inDatasetType)
138 # Re-inserting should work
139 self.assertFalse(registry.registerDatasetType(inDatasetType))
140 # Except when they are not identical
141 with self.assertRaises(ConflictingDefinitionError):
142 nonIdenticalDatasetType = DatasetType(datasetTypeName, differentDimensions, storageClass)
143 registry.registerDatasetType(nonIdenticalDatasetType)
145 # Template can be None
146 datasetTypeName = "testNoneTemplate"
147 storageClass = StorageClass("testDatasetType2")
148 registry.storageClasses.registerStorageClass(storageClass)
149 dimensions = registry.dimensions.extract(("instrument", "visit"))
150 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass)
151 registry.registerDatasetType(inDatasetType)
152 outDatasetType2 = registry.getDatasetType(datasetTypeName)
153 self.assertEqual(outDatasetType2, inDatasetType)
155 allTypes = set(registry.queryDatasetTypes())
156 self.assertEqual(allTypes, {outDatasetType1, outDatasetType2})
158 def testDimensions(self):
159 """Tests for `Registry.insertDimensionData`,
160 `Registry.syncDimensionData`, and `Registry.expandDataId`.
161 """
162 registry = self.makeRegistry()
163 dimensionName = "instrument"
164 dimension = registry.dimensions[dimensionName]
165 dimensionValue = {"name": "DummyCam", "visit_max": 10, "exposure_max": 10, "detector_max": 2,
166 "class_name": "lsst.obs.base.Instrument"}
167 registry.insertDimensionData(dimensionName, dimensionValue)
168 # Inserting the same value twice should fail
169 with self.assertRaises(sqlalchemy.exc.IntegrityError):
170 registry.insertDimensionData(dimensionName, dimensionValue)
171 # expandDataId should retrieve the record we just inserted
172 self.assertEqual(
173 registry.expandDataId(
174 instrument="DummyCam",
175 graph=dimension.graph
176 ).records[dimensionName].toDict(),
177 dimensionValue
178 )
179 # expandDataId should raise if there is no record with the given ID.
180 with self.assertRaises(LookupError):
181 registry.expandDataId({"instrument": "Unknown"}, graph=dimension.graph)
182 # abstract_filter doesn't have a table; insert should fail.
183 with self.assertRaises(TypeError):
184 registry.insertDimensionData("abstract_filter", {"abstract_filter": "i"})
185 dimensionName2 = "physical_filter"
186 dimension2 = registry.dimensions[dimensionName2]
187 dimensionValue2 = {"name": "DummyCam_i", "abstract_filter": "i"}
188 # Missing required dependency ("instrument") should fail
189 with self.assertRaises(sqlalchemy.exc.IntegrityError):
190 registry.insertDimensionData(dimensionName2, dimensionValue2)
191 # Adding required dependency should fix the failure
192 dimensionValue2["instrument"] = "DummyCam"
193 registry.insertDimensionData(dimensionName2, dimensionValue2)
194 # expandDataId should retrieve the record we just inserted.
195 self.assertEqual(
196 registry.expandDataId(
197 instrument="DummyCam", physical_filter="DummyCam_i",
198 graph=dimension2.graph
199 ).records[dimensionName2].toDict(),
200 dimensionValue2
201 )
202 # Use syncDimensionData to insert a new record successfully.
203 dimensionName3 = "detector"
204 dimensionValue3 = {"instrument": "DummyCam", "id": 1, "full_name": "one",
205 "name_in_raft": "zero", "purpose": "SCIENCE"}
206 self.assertTrue(registry.syncDimensionData(dimensionName3, dimensionValue3))
207 # Sync that again. Note that one field ("raft") is NULL, and that
208 # should be okay.
209 self.assertFalse(registry.syncDimensionData(dimensionName3, dimensionValue3))
210 # Now try that sync with the same primary key but a different value.
211 # This should fail.
212 with self.assertRaises(ConflictingDefinitionError):
213 registry.syncDimensionData(
214 dimensionName3,
215 {"instrument": "DummyCam", "id": 1, "full_name": "one",
216 "name_in_raft": "four", "purpose": "SCIENCE"}
217 )
219 def testDataIdRelationships(self):
220 """Test `Registry.relateDataId`.
221 """
222 registry = self.makeRegistry()
223 self.loadData(registry, "base.yaml")
224 # Simple cases where the dimension key-value pairs tell us everything.
225 self.assertEqual(
226 registry.relateDataIds(
227 {"instrument": "Cam1"},
228 {"instrument": "Cam1"},
229 ),
230 ConsistentDataIds(contains=True, within=True, overlaps=True)
231 )
232 self.assertEqual(
233 registry.relateDataIds({}, {}),
234 ConsistentDataIds(contains=True, within=True, overlaps=False)
235 )
236 self.assertEqual(
237 registry.relateDataIds({"instrument": "Cam1"}, {}),
238 ConsistentDataIds(contains=True, within=False, overlaps=False)
239 )
240 self.assertEqual(
241 registry.relateDataIds({}, {"instrument": "Cam1"}),
242 ConsistentDataIds(contains=False, within=True, overlaps=False)
243 )
244 self.assertEqual(
245 registry.relateDataIds(
246 {"instrument": "Cam1", "physical_filter": "Cam1-G"},
247 {"instrument": "Cam1"},
248 ),
249 ConsistentDataIds(contains=True, within=False, overlaps=True)
250 )
251 self.assertEqual(
252 registry.relateDataIds(
253 {"instrument": "Cam1"},
254 {"instrument": "Cam1", "physical_filter": "Cam1-G"},
255 ),
256 ConsistentDataIds(contains=False, within=True, overlaps=True)
257 )
258 self.assertIsNone(
259 registry.relateDataIds(
260 {"instrument": "Cam1", "physical_filter": "Cam1-G"},
261 {"instrument": "Cam1", "physical_filter": "Cam1-R1"},
262 )
263 )
264 # Trickier cases where we need to expand data IDs, but it's still just
265 # required and implied dimension relationships.
266 self.assertEqual(
267 registry.relateDataIds(
268 {"instrument": "Cam1", "physical_filter": "Cam1-G"},
269 {"instrument": "Cam1", "abstract_filter": "g"},
270 ),
271 ConsistentDataIds(contains=True, within=False, overlaps=True)
272 )
273 self.assertEqual(
274 registry.relateDataIds(
275 {"instrument": "Cam1", "abstract_filter": "g"},
276 {"instrument": "Cam1", "physical_filter": "Cam1-G"},
277 ),
278 ConsistentDataIds(contains=False, within=True, overlaps=True)
279 )
280 self.assertEqual(
281 registry.relateDataIds(
282 {"instrument": "Cam1"},
283 {"htm7": 131073},
284 ),
285 ConsistentDataIds(contains=False, within=False, overlaps=False)
286 )
287 # Trickiest cases involve spatial or temporal overlaps or non-dimension
288 # elements that relate things (of which visit_definition is our only
289 # current example).
290 #
291 # These two HTM IDs at different levels have a "contains" relationship
292 # spatially, but there is no overlap of dimension keys. The exact
293 # result of relateDataIds is unspecified for this case, but it's
294 # guaranteed to be truthy (see relateDataIds docs.).
295 self.assertTrue(
296 registry.relateDataIds({"htm7": 131073}, {"htm9": 2097169})
297 )
298 # These two HTM IDs at different levels are disjoint spatially, which
299 # means the data IDs are inconsistent.
300 self.assertIsNone(
301 registry.relateDataIds({"htm7": 131073}, {"htm9": 2097391})
302 )
303 # Insert a few more dimension records for the next test.
304 registry.insertDimensionData(
305 "exposure",
306 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G"},
307 )
308 registry.insertDimensionData(
309 "exposure",
310 {"instrument": "Cam1", "id": 2, "name": "two", "physical_filter": "Cam1-G"},
311 )
312 registry.insertDimensionData(
313 "visit_system",
314 {"instrument": "Cam1", "id": 0, "name": "one-to-one"},
315 )
316 registry.insertDimensionData(
317 "visit",
318 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G", "visit_system": 0},
319 )
320 registry.insertDimensionData(
321 "visit_definition",
322 {"instrument": "Cam1", "visit": 1, "exposure": 1, "visit_system": 0},
323 )
324 self.assertEqual(
325 registry.relateDataIds(
326 {"instrument": "Cam1", "visit": 1},
327 {"instrument": "Cam1", "exposure": 1},
328 ),
329 ConsistentDataIds(contains=False, within=False, overlaps=True)
330 )
331 self.assertIsNone(
332 registry.relateDataIds(
333 {"instrument": "Cam1", "visit": 1},
334 {"instrument": "Cam1", "exposure": 2},
335 )
336 )
338 def testDataset(self):
339 """Basic tests for `Registry.insertDatasets`, `Registry.getDataset`,
340 and `Registry.removeDataset`.
341 """
342 registry = self.makeRegistry()
343 self.loadData(registry, "base.yaml")
344 run = "test"
345 registry.registerRun(run)
346 datasetType = registry.getDatasetType("permabias")
347 dataId = {"instrument": "Cam1", "detector": 2}
348 ref, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run)
349 outRef = registry.getDataset(ref.id)
350 self.assertIsNotNone(ref.id)
351 self.assertEqual(ref, outRef)
352 with self.assertRaises(ConflictingDefinitionError):
353 registry.insertDatasets(datasetType, dataIds=[dataId], run=run)
354 registry.removeDataset(ref)
355 self.assertIsNone(registry.findDataset(datasetType, dataId, collections=[run]))
357 def testComponents(self):
358 """Tests for `Registry.attachComponent` and other dataset operations
359 on composite datasets.
360 """
361 registry = self.makeRegistry()
362 self.loadData(registry, "base.yaml")
363 run = "test"
364 registry.registerRun(run)
365 parentDatasetType = registry.getDatasetType("permabias")
366 childDatasetType1 = registry.getDatasetType("permabias.image")
367 childDatasetType2 = registry.getDatasetType("permabias.mask")
368 dataId = {"instrument": "Cam1", "detector": 2}
369 parent, = registry.insertDatasets(parentDatasetType, dataIds=[dataId], run=run)
370 children = {"image": registry.insertDatasets(childDatasetType1, dataIds=[dataId], run=run)[0],
371 "mask": registry.insertDatasets(childDatasetType2, dataIds=[dataId], run=run)[0]}
372 for name, child in children.items():
373 registry.attachComponent(name, parent, child)
374 self.assertEqual(parent.components, children)
375 outParent = registry.getDataset(parent.id)
376 self.assertEqual(outParent.components, children)
377 # Remove the parent; this should remove all children.
378 registry.removeDataset(parent)
379 self.assertIsNone(registry.findDataset(parentDatasetType, dataId, collections=[run]))
380 self.assertIsNone(registry.findDataset(childDatasetType1, dataId, collections=[run]))
381 self.assertIsNone(registry.findDataset(childDatasetType2, dataId, collections=[run]))
383 def testFindDataset(self):
384 """Tests for `Registry.findDataset`.
385 """
386 registry = self.makeRegistry()
387 self.loadData(registry, "base.yaml")
388 run = "test"
389 datasetType = registry.getDatasetType("permabias")
390 dataId = {"instrument": "Cam1", "detector": 4}
391 registry.registerRun(run)
392 inputRef, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run)
393 outputRef = registry.findDataset(datasetType, dataId, collections=[run])
394 self.assertEqual(outputRef, inputRef)
395 # Check that retrieval with invalid dataId raises
396 with self.assertRaises(LookupError):
397 dataId = {"instrument": "Cam1"} # no detector
398 registry.findDataset(datasetType, dataId, collections=run)
399 # Check that different dataIds match to different datasets
400 dataId1 = {"instrument": "Cam1", "detector": 1}
401 inputRef1, = registry.insertDatasets(datasetType, dataIds=[dataId1], run=run)
402 dataId2 = {"instrument": "Cam1", "detector": 2}
403 inputRef2, = registry.insertDatasets(datasetType, dataIds=[dataId2], run=run)
404 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef1)
405 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef2)
406 self.assertNotEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef2)
407 self.assertNotEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef1)
408 # Check that requesting a non-existing dataId returns None
409 nonExistingDataId = {"instrument": "Cam1", "detector": 3}
410 self.assertIsNone(registry.findDataset(datasetType, nonExistingDataId, collections=run))
412 def testCollections(self):
413 """Tests for registry methods that manage collections.
414 """
415 registry = self.makeRegistry()
416 self.loadData(registry, "base.yaml")
417 self.loadData(registry, "datasets.yaml")
418 run1 = "imported_g"
419 run2 = "imported_r"
420 datasetType = "permabias"
421 # Find some datasets via their run's collection.
422 dataId1 = {"instrument": "Cam1", "detector": 1}
423 ref1 = registry.findDataset(datasetType, dataId1, collections=run1)
424 self.assertIsNotNone(ref1)
425 dataId2 = {"instrument": "Cam1", "detector": 2}
426 ref2 = registry.findDataset(datasetType, dataId2, collections=run1)
427 self.assertIsNotNone(ref2)
428 # Associate those into a new collection,then look for them there.
429 tag1 = "tag1"
430 registry.registerCollection(tag1, type=CollectionType.TAGGED)
431 registry.associate(tag1, [ref1, ref2])
432 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1)
433 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
434 # Disassociate one and verify that we can't it there anymore...
435 registry.disassociate(tag1, [ref1])
436 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=tag1))
437 # ...but we can still find ref2 in tag1, and ref1 in the run.
438 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run1), ref1)
439 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
440 collections = set(registry.queryCollections())
441 self.assertEqual(collections, {run1, run2, tag1})
442 # Associate both refs into tag1 again; ref2 is already there, but that
443 # should be a harmless no-op.
444 registry.associate(tag1, [ref1, ref2])
445 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1)
446 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
447 # Get a different dataset (from a different run) that has the same
448 # dataset type and data ID as ref2.
449 ref2b = registry.findDataset(datasetType, dataId2, collections=run2)
450 self.assertNotEqual(ref2, ref2b)
451 # Attempting to associate that into tag1 should be an error.
452 with self.assertRaises(ConflictingDefinitionError):
453 registry.associate(tag1, [ref2b])
454 # That error shouldn't have messed up what we had before.
455 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1)
456 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
457 # Attempt to associate the conflicting dataset again, this time with
458 # a dataset that isn't in the collection and won't cause a conflict.
459 # Should also fail without modifying anything.
460 dataId3 = {"instrument": "Cam1", "detector": 3}
461 ref3 = registry.findDataset(datasetType, dataId3, collections=run1)
462 with self.assertRaises(ConflictingDefinitionError):
463 registry.associate(tag1, [ref3, ref2b])
464 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1)
465 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
466 self.assertIsNone(registry.findDataset(datasetType, dataId3, collections=tag1))
467 # Register a chained collection that searches:
468 # 1. 'tag1'
469 # 2. 'run1', but only for the permaflat dataset
470 # 3. 'run2'
471 chain1 = "chain1"
472 registry.registerCollection(chain1, type=CollectionType.CHAINED)
473 self.assertIs(registry.getCollectionType(chain1), CollectionType.CHAINED)
474 # Chained collection exists, but has no collections in it.
475 self.assertFalse(registry.getCollectionChain(chain1))
476 # If we query for all collections, we should get the chained collection
477 # only if we don't ask to flatten it (i.e. yield only its children).
478 self.assertEqual(set(registry.queryCollections(flattenChains=False)), {tag1, run1, run2, chain1})
479 self.assertEqual(set(registry.queryCollections(flattenChains=True)), {tag1, run1, run2})
480 # Attempt to set its child collections to something circular; that
481 # should fail.
482 with self.assertRaises(ValueError):
483 registry.setCollectionChain(chain1, [tag1, chain1])
484 # Add the child collections.
485 registry.setCollectionChain(chain1, [tag1, (run1, "permaflat"), run2])
486 self.assertEqual(
487 list(registry.getCollectionChain(chain1)),
488 [(tag1, DatasetTypeRestriction.any),
489 (run1, DatasetTypeRestriction.fromExpression("permaflat")),
490 (run2, DatasetTypeRestriction.any)]
491 )
492 # Searching for dataId1 or dataId2 in the chain should return ref1 and
493 # ref2, because both are in tag1.
494 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain1), ref1)
495 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain1), ref2)
496 # Now disassociate ref2 from tag1. The search (for permabias) with
497 # dataId2 in chain1 should then:
498 # 1. not find it in tag1
499 # 2. not look in tag2, because it's restricted to permaflat here
500 # 3. find a different dataset in run2
501 registry.disassociate(tag1, [ref2])
502 ref2b = registry.findDataset(datasetType, dataId2, collections=chain1)
503 self.assertNotEqual(ref2b, ref2)
504 self.assertEqual(ref2b, registry.findDataset(datasetType, dataId2, collections=run2))
505 # Look in the chain for a permaflat that is in run1; should get the
506 # same ref as if we'd searched run1 directly.
507 dataId3 = {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}
508 self.assertEqual(registry.findDataset("permaflat", dataId3, collections=chain1),
509 registry.findDataset("permaflat", dataId3, collections=run1),)
510 # Define a new chain so we can test recursive chains.
511 chain2 = "chain2"
512 registry.registerCollection(chain2, type=CollectionType.CHAINED)
513 registry.setCollectionChain(chain2, [(run2, "permabias"), chain1])
514 # Search for permabias with dataId1 should find it via tag1 in chain2,
515 # recursing, because is not in run1.
516 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=run2))
517 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain2), ref1)
518 # Search for permabias with dataId2 should find it in run2 (ref2b).
519 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain2), ref2b)
520 # Search for a permaflat that is in run2. That should not be found
521 # at the front of chain2, because of the restriction to permabias
522 # on run2 there, but it should be found in at the end of chain1.
523 dataId4 = {"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-R2"}
524 ref4 = registry.findDataset("permaflat", dataId4, collections=run2)
525 self.assertIsNotNone(ref4)
526 self.assertEqual(ref4, registry.findDataset("permaflat", dataId4, collections=chain2))
528 def testDatasetLocations(self):
529 """Tests for `Registry.insertDatasetLocations`,
530 `Registry.getDatasetLocations`, and `Registry.removeDatasetLocation`.
531 """
532 registry = self.makeRegistry()
533 self.loadData(registry, "base.yaml")
534 self.loadData(registry, "datasets.yaml")
535 run = "imported_g"
536 ref = registry.findDataset("permabias", dataId={"instrument": "Cam1", "detector": 1}, collections=run)
537 ref2 = registry.findDataset("permaflat",
538 dataId={"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-G"},
539 collections=run)
540 datastoreName = "dummystore"
541 datastoreName2 = "dummystore2"
542 # Test adding information about a new dataset
543 registry.insertDatasetLocations(datastoreName, [ref])
544 addresses = registry.getDatasetLocations(ref)
545 self.assertIn(datastoreName, addresses)
546 self.assertEqual(len(addresses), 1)
547 registry.insertDatasetLocations(datastoreName2, [ref, ref2])
548 addresses = registry.getDatasetLocations(ref)
549 self.assertEqual(len(addresses), 2)
550 self.assertIn(datastoreName, addresses)
551 self.assertIn(datastoreName2, addresses)
552 registry.removeDatasetLocation(datastoreName, [ref])
553 addresses = registry.getDatasetLocations(ref)
554 self.assertEqual(len(addresses), 1)
555 self.assertNotIn(datastoreName, addresses)
556 self.assertIn(datastoreName2, addresses)
557 with self.assertRaises(OrphanedRecordError):
558 registry.removeDataset(ref)
559 registry.removeDatasetLocation(datastoreName2, [ref])
560 addresses = registry.getDatasetLocations(ref)
561 self.assertEqual(len(addresses), 0)
562 self.assertNotIn(datastoreName2, addresses)
563 registry.removeDataset(ref) # should not raise
564 addresses = registry.getDatasetLocations(ref2)
565 self.assertEqual(len(addresses), 1)
566 self.assertIn(datastoreName2, addresses)
568 def testBasicTransaction(self):
569 """Test that all operations within a single transaction block are
570 rolled back if an exception propagates out of the block.
571 """
572 registry = self.makeRegistry()
573 storageClass = StorageClass("testDatasetType")
574 registry.storageClasses.registerStorageClass(storageClass)
575 dimensions = registry.dimensions.extract(("instrument",))
576 dataId = {"instrument": "DummyCam"}
577 datasetTypeA = DatasetType(name="A",
578 dimensions=dimensions,
579 storageClass=storageClass)
580 datasetTypeB = DatasetType(name="B",
581 dimensions=dimensions,
582 storageClass=storageClass)
583 datasetTypeC = DatasetType(name="C",
584 dimensions=dimensions,
585 storageClass=storageClass)
586 run = "test"
587 registry.registerRun(run)
588 refId = None
589 with registry.transaction():
590 registry.registerDatasetType(datasetTypeA)
591 with self.assertRaises(ValueError):
592 with registry.transaction():
593 registry.registerDatasetType(datasetTypeB)
594 registry.registerDatasetType(datasetTypeC)
595 registry.insertDimensionData("instrument", {"instrument": "DummyCam"})
596 ref, = registry.insertDatasets(datasetTypeA, dataIds=[dataId], run=run)
597 refId = ref.id
598 raise ValueError("Oops, something went wrong")
599 # A should exist
600 self.assertEqual(registry.getDatasetType("A"), datasetTypeA)
601 # But B and C should both not exist
602 with self.assertRaises(KeyError):
603 registry.getDatasetType("B")
604 with self.assertRaises(KeyError):
605 registry.getDatasetType("C")
606 # And neither should the dataset
607 self.assertIsNotNone(refId)
608 self.assertIsNone(registry.getDataset(refId))
609 # Or the Dimension entries
610 with self.assertRaises(LookupError):
611 registry.expandDataId({"instrument": "DummyCam"})
613 def testNestedTransaction(self):
614 """Test that operations within a transaction block are not rolled back
615 if an exception propagates out of an inner transaction block and is
616 then caught.
617 """
618 registry = self.makeRegistry()
619 dimension = registry.dimensions["instrument"]
620 dataId1 = {"instrument": "DummyCam"}
621 dataId2 = {"instrument": "DummyCam2"}
622 checkpointReached = False
623 with registry.transaction():
624 # This should be added and (ultimately) committed.
625 registry.insertDimensionData(dimension, dataId1)
626 with self.assertRaises(sqlalchemy.exc.IntegrityError):
627 with registry.transaction():
628 # This does not conflict, and should succeed (but not
629 # be committed).
630 registry.insertDimensionData(dimension, dataId2)
631 checkpointReached = True
632 # This should conflict and raise, triggerring a rollback
633 # of the previous insertion within the same transaction
634 # context, but not the original insertion in the outer
635 # block.
636 registry.insertDimensionData(dimension, dataId1)
637 self.assertTrue(checkpointReached)
638 self.assertIsNotNone(registry.expandDataId(dataId1, graph=dimension.graph))
639 with self.assertRaises(LookupError):
640 registry.expandDataId(dataId2, graph=dimension.graph)
642 def testInstrumentDimensions(self):
643 """Test queries involving only instrument dimensions, with no joins to
644 skymap."""
645 registry = self.makeRegistry()
647 # need a bunch of dimensions and datasets for test
648 registry.insertDimensionData(
649 "instrument",
650 dict(name="DummyCam", visit_max=25, exposure_max=300, detector_max=6)
651 )
652 registry.insertDimensionData(
653 "physical_filter",
654 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"),
655 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"),
656 )
657 registry.insertDimensionData(
658 "detector",
659 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in range(1, 6)]
660 )
661 registry.insertDimensionData(
662 "visit_system",
663 dict(instrument="DummyCam", id=1, name="default"),
664 )
665 registry.insertDimensionData(
666 "visit",
667 dict(instrument="DummyCam", id=10, name="ten", physical_filter="dummy_i", visit_system=1),
668 dict(instrument="DummyCam", id=11, name="eleven", physical_filter="dummy_r", visit_system=1),
669 dict(instrument="DummyCam", id=20, name="twelve", physical_filter="dummy_r", visit_system=1),
670 )
671 registry.insertDimensionData(
672 "exposure",
673 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i"),
674 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i"),
675 dict(instrument="DummyCam", id=110, name="110", physical_filter="dummy_r"),
676 dict(instrument="DummyCam", id=111, name="111", physical_filter="dummy_r"),
677 dict(instrument="DummyCam", id=200, name="200", physical_filter="dummy_r"),
678 dict(instrument="DummyCam", id=201, name="201", physical_filter="dummy_r"),
679 )
680 registry.insertDimensionData(
681 "visit_definition",
682 dict(instrument="DummyCam", exposure=100, visit_system=1, visit=10),
683 dict(instrument="DummyCam", exposure=101, visit_system=1, visit=10),
684 dict(instrument="DummyCam", exposure=110, visit_system=1, visit=11),
685 dict(instrument="DummyCam", exposure=111, visit_system=1, visit=11),
686 dict(instrument="DummyCam", exposure=200, visit_system=1, visit=20),
687 dict(instrument="DummyCam", exposure=201, visit_system=1, visit=20),
688 )
689 # dataset types
690 run1 = "test1_r"
691 run2 = "test2_r"
692 tagged2 = "test2_t"
693 registry.registerRun(run1)
694 registry.registerRun(run2)
695 registry.registerCollection(tagged2)
696 storageClass = StorageClass("testDataset")
697 registry.storageClasses.registerStorageClass(storageClass)
698 rawType = DatasetType(name="RAW",
699 dimensions=registry.dimensions.extract(("instrument", "exposure", "detector")),
700 storageClass=storageClass)
701 registry.registerDatasetType(rawType)
702 calexpType = DatasetType(name="CALEXP",
703 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")),
704 storageClass=storageClass)
705 registry.registerDatasetType(calexpType)
707 # add pre-existing datasets
708 for exposure in (100, 101, 110, 111):
709 for detector in (1, 2, 3):
710 # note that only 3 of 5 detectors have datasets
711 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector)
712 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run1)
713 # exposures 100 and 101 appear in both run1 and tagged2.
714 # 100 has different datasets in the different collections
715 # 101 has the same dataset in both collections.
716 if exposure == 100:
717 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2)
718 if exposure in (100, 101):
719 registry.associate(tagged2, [ref])
720 # Add pre-existing datasets to tagged2.
721 for exposure in (200, 201):
722 for detector in (3, 4, 5):
723 # note that only 3 of 5 detectors have datasets
724 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector)
725 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2)
726 registry.associate(tagged2, [ref])
728 dimensions = DimensionGraph(
729 registry.dimensions,
730 dimensions=(rawType.dimensions.required | calexpType.dimensions.required)
731 )
732 # Test that single dim string works as well as list of str
733 rows = list(registry.queryDimensions("visit", datasets=rawType, collections=run1, expand=True))
734 rowsI = list(registry.queryDimensions(["visit"], datasets=rawType, collections=run1, expand=True))
735 self.assertEqual(rows, rowsI)
736 # with empty expression
737 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, expand=True))
738 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors
739 for dataId in rows:
740 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit"))
741 packer1 = registry.dimensions.makePacker("visit_detector", dataId)
742 packer2 = registry.dimensions.makePacker("exposure_detector", dataId)
743 self.assertEqual(packer1.unpack(packer1.pack(dataId)),
744 DataCoordinate.standardize(dataId, graph=packer1.dimensions))
745 self.assertEqual(packer2.unpack(packer2.pack(dataId)),
746 DataCoordinate.standardize(dataId, graph=packer2.dimensions))
747 self.assertNotEqual(packer1.pack(dataId), packer2.pack(dataId))
748 self.assertCountEqual(set(dataId["exposure"] for dataId in rows),
749 (100, 101, 110, 111))
750 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11))
751 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3))
753 # second collection
754 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=tagged2))
755 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors
756 for dataId in rows:
757 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit"))
758 self.assertCountEqual(set(dataId["exposure"] for dataId in rows),
759 (100, 101, 200, 201))
760 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 20))
761 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5))
763 # with two input datasets
764 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=[run1, tagged2]))
765 self.assertEqual(len(set(rows)), 6*3) # 6 exposures times 3 detectors; set needed to de-dupe
766 for dataId in rows:
767 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit"))
768 self.assertCountEqual(set(dataId["exposure"] for dataId in rows),
769 (100, 101, 110, 111, 200, 201))
770 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11, 20))
771 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5))
773 # limit to single visit
774 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1,
775 where="visit = 10"))
776 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors
777 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101))
778 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,))
779 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3))
781 # more limiting expression, using link names instead of Table.column
782 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1,
783 where="visit = 10 and detector > 1"))
784 self.assertEqual(len(rows), 2*2) # 2 exposures times 2 detectors
785 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101))
786 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,))
787 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (2, 3))
789 # expression excludes everything
790 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1,
791 where="visit > 1000"))
792 self.assertEqual(len(rows), 0)
794 # Selecting by physical_filter, this is not in the dimensions, but it
795 # is a part of the full expression so it should work too.
796 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1,
797 where="physical_filter = 'dummy_r'"))
798 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors
799 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (110, 111))
800 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (11,))
801 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3))
803 def testSkyMapDimensions(self):
804 """Tests involving only skymap dimensions, no joins to instrument."""
805 registry = self.makeRegistry()
807 # need a bunch of dimensions and datasets for test, we want
808 # "abstract_filter" in the test so also have to add physical_filter
809 # dimensions
810 registry.insertDimensionData(
811 "instrument",
812 dict(instrument="DummyCam")
813 )
814 registry.insertDimensionData(
815 "physical_filter",
816 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"),
817 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"),
818 )
819 registry.insertDimensionData(
820 "skymap",
821 dict(name="DummyMap", hash="sha!".encode("utf8"))
822 )
823 for tract in range(10):
824 registry.insertDimensionData("tract", dict(skymap="DummyMap", id=tract))
825 registry.insertDimensionData(
826 "patch",
827 *[dict(skymap="DummyMap", tract=tract, id=patch, cell_x=0, cell_y=0)
828 for patch in range(10)]
829 )
831 # dataset types
832 run = "test"
833 registry.registerRun(run)
834 storageClass = StorageClass("testDataset")
835 registry.storageClasses.registerStorageClass(storageClass)
836 calexpType = DatasetType(name="deepCoadd_calexp",
837 dimensions=registry.dimensions.extract(("skymap", "tract", "patch",
838 "abstract_filter")),
839 storageClass=storageClass)
840 registry.registerDatasetType(calexpType)
841 mergeType = DatasetType(name="deepCoadd_mergeDet",
842 dimensions=registry.dimensions.extract(("skymap", "tract", "patch")),
843 storageClass=storageClass)
844 registry.registerDatasetType(mergeType)
845 measType = DatasetType(name="deepCoadd_meas",
846 dimensions=registry.dimensions.extract(("skymap", "tract", "patch",
847 "abstract_filter")),
848 storageClass=storageClass)
849 registry.registerDatasetType(measType)
851 dimensions = DimensionGraph(
852 registry.dimensions,
853 dimensions=(calexpType.dimensions.required | mergeType.dimensions.required
854 | measType.dimensions.required)
855 )
857 # add pre-existing datasets
858 for tract in (1, 3, 5):
859 for patch in (2, 4, 6, 7):
860 dataId = dict(skymap="DummyMap", tract=tract, patch=patch)
861 registry.insertDatasets(mergeType, dataIds=[dataId], run=run)
862 for aFilter in ("i", "r"):
863 dataId = dict(skymap="DummyMap", tract=tract, patch=patch, abstract_filter=aFilter)
864 registry.insertDatasets(calexpType, dataIds=[dataId], run=run)
866 # with empty expression
867 rows = list(registry.queryDimensions(dimensions,
868 datasets=[calexpType, mergeType], collections=run))
869 self.assertEqual(len(rows), 3*4*2) # 4 tracts x 4 patches x 2 filters
870 for dataId in rows:
871 self.assertCountEqual(dataId.keys(), ("skymap", "tract", "patch", "abstract_filter"))
872 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5))
873 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7))
874 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r"))
876 # limit to 2 tracts and 2 patches
877 rows = list(registry.queryDimensions(dimensions,
878 datasets=[calexpType, mergeType], collections=run,
879 where="tract IN (1, 5) AND patch IN (2, 7)"))
880 self.assertEqual(len(rows), 2*2*2) # 2 tracts x 2 patches x 2 filters
881 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 5))
882 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 7))
883 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r"))
885 # limit to single filter
886 rows = list(registry.queryDimensions(dimensions,
887 datasets=[calexpType, mergeType], collections=run,
888 where="abstract_filter = 'i'"))
889 self.assertEqual(len(rows), 3*4*1) # 4 tracts x 4 patches x 2 filters
890 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5))
891 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7))
892 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i",))
894 # expression excludes everything, specifying non-existing skymap is
895 # not a fatal error, it's operator error
896 rows = list(registry.queryDimensions(dimensions,
897 datasets=[calexpType, mergeType], collections=run,
898 where="skymap = 'Mars'"))
899 self.assertEqual(len(rows), 0)
901 def testSpatialMatch(self):
902 """Test involving spatial match using join tables.
904 Note that realistic test needs a reasonably-defined skypix and regions
905 in registry tables which is hard to implement in this simple test.
906 So we do not actually fill registry with any data and all queries will
907 return empty result, but this is still useful for coverage of the code
908 that generates query.
909 """
910 registry = self.makeRegistry()
912 # dataset types
913 collection = "test"
914 registry.registerRun(name=collection)
915 storageClass = StorageClass("testDataset")
916 registry.storageClasses.registerStorageClass(storageClass)
918 calexpType = DatasetType(name="CALEXP",
919 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")),
920 storageClass=storageClass)
921 registry.registerDatasetType(calexpType)
923 coaddType = DatasetType(name="deepCoadd_calexp",
924 dimensions=registry.dimensions.extract(("skymap", "tract", "patch",
925 "abstract_filter")),
926 storageClass=storageClass)
927 registry.registerDatasetType(coaddType)
929 dimensions = DimensionGraph(
930 registry.dimensions,
931 dimensions=(calexpType.dimensions.required | coaddType.dimensions.required)
932 )
934 # without data this should run OK but return empty set
935 rows = list(registry.queryDimensions(dimensions, datasets=calexpType, collections=collection))
936 self.assertEqual(len(rows), 0)
938 def testCalibrationLabelIndirection(self):
939 """Test that we can look up datasets with calibration_label dimensions
940 from a data ID with exposure dimensions.
941 """
943 def _dt(iso_string):
944 return astropy.time.Time(iso_string, format="iso", scale="utc")
946 registry = self.makeRegistry()
948 flat = DatasetType(
949 "flat",
950 registry.dimensions.extract(
951 ["instrument", "detector", "physical_filter", "calibration_label"]
952 ),
953 "ImageU"
954 )
955 registry.registerDatasetType(flat)
956 registry.insertDimensionData("instrument", dict(name="DummyCam"))
957 registry.insertDimensionData(
958 "physical_filter",
959 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"),
960 )
961 registry.insertDimensionData(
962 "detector",
963 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in (1, 2, 3, 4, 5)]
964 )
965 registry.insertDimensionData(
966 "exposure",
967 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i",
968 datetime_begin=_dt("2005-12-15 02:00:00"), datetime_end=_dt("2005-12-15 03:00:00")),
969 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i",
970 datetime_begin=_dt("2005-12-16 02:00:00"), datetime_end=_dt("2005-12-16 03:00:00")),
971 )
972 registry.insertDimensionData(
973 "calibration_label",
974 dict(instrument="DummyCam", name="first_night",
975 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-15 04:00:00")),
976 dict(instrument="DummyCam", name="second_night",
977 datetime_begin=_dt("2005-12-16 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")),
978 dict(instrument="DummyCam", name="both_nights",
979 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")),
980 )
981 # Different flats for different nights for detectors 1-3 in first
982 # collection.
983 run1 = "calibs1"
984 registry.registerRun(run1)
985 for detector in (1, 2, 3):
986 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="first_night",
987 physical_filter="dummy_i", detector=detector)],
988 run=run1)
989 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="second_night",
990 physical_filter="dummy_i", detector=detector)],
991 run=run1)
992 # The same flat for both nights for detectors 3-5 (so detector 3 has
993 # multiple valid flats) in second collection.
994 run2 = "calib2"
995 registry.registerRun(run2)
996 for detector in (3, 4, 5):
997 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="both_nights",
998 physical_filter="dummy_i", detector=detector)],
999 run=run2)
1000 # Perform queries for individual exposure+detector combinations, which
1001 # should always return exactly one flat.
1002 for exposure in (100, 101):
1003 for detector in (1, 2, 3):
1004 with self.subTest(exposure=exposure, detector=detector):
1005 rows = list(registry.queryDatasets("flat", collections=[run1],
1006 instrument="DummyCam",
1007 exposure=exposure,
1008 detector=detector))
1009 self.assertEqual(len(rows), 1)
1010 for detector in (3, 4, 5):
1011 with self.subTest(exposure=exposure, detector=detector):
1012 rows = registry.queryDatasets("flat", collections=[run2],
1013 instrument="DummyCam",
1014 exposure=exposure,
1015 detector=detector)
1016 self.assertEqual(len(list(rows)), 1)
1017 for detector in (1, 2, 4, 5):
1018 with self.subTest(exposure=exposure, detector=detector):
1019 rows = registry.queryDatasets("flat", collections=[run1, run2],
1020 instrument="DummyCam",
1021 exposure=exposure,
1022 detector=detector)
1023 self.assertEqual(len(list(rows)), 1)
1024 for detector in (3,):
1025 with self.subTest(exposure=exposure, detector=detector):
1026 rows = registry.queryDatasets("flat", collections=[run1, run2],
1027 instrument="DummyCam",
1028 exposure=exposure,
1029 detector=detector)
1030 self.assertEqual(len(list(rows)), 2)
1032 def testAbstractFilterQuery(self):
1033 """Test that we can run a query that just lists the known
1034 abstract_filters. This is tricky because abstract_filter is
1035 backed by a query against physical_filter.
1036 """
1037 registry = self.makeRegistry()
1038 registry.insertDimensionData("instrument", dict(name="DummyCam"))
1039 registry.insertDimensionData(
1040 "physical_filter",
1041 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"),
1042 dict(instrument="DummyCam", name="dummy_i2", abstract_filter="i"),
1043 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"),
1044 )
1045 rows = list(registry.queryDimensions(["abstract_filter"]))
1046 self.assertCountEqual(
1047 rows,
1048 [DataCoordinate.standardize(abstract_filter="i", universe=registry.dimensions),
1049 DataCoordinate.standardize(abstract_filter="r", universe=registry.dimensions)]
1050 )