Coverage for python/lsst/daf/butler/registry/tests/_registry.py : 6%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = ["RegistryTests"]
25from abc import ABC, abstractmethod
26import os
27import re
28import unittest
30import astropy.time
31import sqlalchemy
32from typing import Optional
34try:
35 import numpy as np
36except ImportError:
37 np = None
39from ...core import (
40 DataCoordinate,
41 DatasetRef,
42 DatasetType,
43 DimensionGraph,
44 NamedValueSet,
45 StorageClass,
46 ddl,
47 YamlRepoImportBackend
48)
49from .._registry import (
50 CollectionType,
51 ConflictingDefinitionError,
52 InconsistentDataIdError,
53 Registry,
54 RegistryConfig,
55)
56from ..wildcards import DatasetTypeRestriction
57from ..interfaces import MissingCollectionError, ButlerAttributeExistsError
60class RegistryTests(ABC):
61 """Generic tests for the `Registry` class that can be subclassed to
62 generate tests for different configurations.
63 """
65 collectionsManager: Optional[str] = None
66 """Name of the collections manager class, if subclass provides value for
67 this member then it overrides name specified in default configuration
68 (`str`).
69 """
71 @classmethod
72 @abstractmethod
73 def getDataDir(cls) -> str:
74 """Return the root directory containing test data YAML files.
75 """
76 raise NotImplementedError()
78 def makeRegistryConfig(self) -> RegistryConfig:
79 """Create RegistryConfig used to create a registry.
81 This method should be called by a subclass from `makeRegistry`.
82 Returned instance will be pre-configured based on the values of class
83 members, and default-configured for all other parametrs. Subclasses
84 that need default configuration should just instantiate
85 `RegistryConfig` directly.
86 """
87 config = RegistryConfig()
88 if self.collectionsManager:
89 config["managers"]["collections"] = self.collectionsManager
90 return config
92 @abstractmethod
93 def makeRegistry(self) -> Registry:
94 """Return the Registry instance to be tested.
95 """
96 raise NotImplementedError()
98 def loadData(self, registry: Registry, filename: str):
99 """Load registry test data from ``getDataDir/<filename>``,
100 which should be a YAML import/export file.
101 """
102 with open(os.path.join(self.getDataDir(), filename), 'r') as stream:
103 backend = YamlRepoImportBackend(stream, registry)
104 backend.register()
105 backend.load(datastore=None)
107 def assertRowCount(self, registry: Registry, table: str, count: int):
108 """Check the number of rows in table.
109 """
110 # TODO: all tests that rely on this method should be rewritten, as it
111 # needs to depend on Registry implementation details to have any chance
112 # of working.
113 sql = sqlalchemy.sql.select(
114 [sqlalchemy.sql.func.count()]
115 ).select_from(
116 getattr(registry._tables, table)
117 )
118 self.assertEqual(registry._db.query(sql).scalar(), count)
120 def testOpaque(self):
121 """Tests for `Registry.registerOpaqueTable`,
122 `Registry.insertOpaqueData`, `Registry.fetchOpaqueData`, and
123 `Registry.deleteOpaqueData`.
124 """
125 registry = self.makeRegistry()
126 table = "opaque_table_for_testing"
127 registry.registerOpaqueTable(
128 table,
129 spec=ddl.TableSpec(
130 fields=[
131 ddl.FieldSpec("id", dtype=sqlalchemy.BigInteger, primaryKey=True),
132 ddl.FieldSpec("name", dtype=sqlalchemy.String, length=16, nullable=False),
133 ddl.FieldSpec("count", dtype=sqlalchemy.SmallInteger, nullable=True),
134 ],
135 )
136 )
137 rows = [
138 {"id": 1, "name": "one", "count": None},
139 {"id": 2, "name": "two", "count": 5},
140 {"id": 3, "name": "three", "count": 6},
141 ]
142 registry.insertOpaqueData(table, *rows)
143 self.assertCountEqual(rows, list(registry.fetchOpaqueData(table)))
144 self.assertEqual(rows[0:1], list(registry.fetchOpaqueData(table, id=1)))
145 self.assertEqual(rows[1:2], list(registry.fetchOpaqueData(table, name="two")))
146 self.assertEqual([], list(registry.fetchOpaqueData(table, id=1, name="two")))
147 registry.deleteOpaqueData(table, id=3)
148 self.assertCountEqual(rows[:2], list(registry.fetchOpaqueData(table)))
149 registry.deleteOpaqueData(table)
150 self.assertEqual([], list(registry.fetchOpaqueData(table)))
152 def testDatasetType(self):
153 """Tests for `Registry.registerDatasetType` and
154 `Registry.getDatasetType`.
155 """
156 registry = self.makeRegistry()
157 # Check valid insert
158 datasetTypeName = "test"
159 storageClass = StorageClass("testDatasetType")
160 registry.storageClasses.registerStorageClass(storageClass)
161 dimensions = registry.dimensions.extract(("instrument", "visit"))
162 differentDimensions = registry.dimensions.extract(("instrument", "patch"))
163 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass)
164 # Inserting for the first time should return True
165 self.assertTrue(registry.registerDatasetType(inDatasetType))
166 outDatasetType1 = registry.getDatasetType(datasetTypeName)
167 self.assertEqual(outDatasetType1, inDatasetType)
169 # Re-inserting should work
170 self.assertFalse(registry.registerDatasetType(inDatasetType))
171 # Except when they are not identical
172 with self.assertRaises(ConflictingDefinitionError):
173 nonIdenticalDatasetType = DatasetType(datasetTypeName, differentDimensions, storageClass)
174 registry.registerDatasetType(nonIdenticalDatasetType)
176 # Template can be None
177 datasetTypeName = "testNoneTemplate"
178 storageClass = StorageClass("testDatasetType2")
179 registry.storageClasses.registerStorageClass(storageClass)
180 dimensions = registry.dimensions.extract(("instrument", "visit"))
181 inDatasetType = DatasetType(datasetTypeName, dimensions, storageClass)
182 registry.registerDatasetType(inDatasetType)
183 outDatasetType2 = registry.getDatasetType(datasetTypeName)
184 self.assertEqual(outDatasetType2, inDatasetType)
186 allTypes = set(registry.queryDatasetTypes())
187 self.assertEqual(allTypes, {outDatasetType1, outDatasetType2})
189 def testDimensions(self):
190 """Tests for `Registry.insertDimensionData`,
191 `Registry.syncDimensionData`, and `Registry.expandDataId`.
192 """
193 registry = self.makeRegistry()
194 dimensionName = "instrument"
195 dimension = registry.dimensions[dimensionName]
196 dimensionValue = {"name": "DummyCam", "visit_max": 10, "exposure_max": 10, "detector_max": 2,
197 "class_name": "lsst.obs.base.Instrument"}
198 registry.insertDimensionData(dimensionName, dimensionValue)
199 # Inserting the same value twice should fail
200 with self.assertRaises(sqlalchemy.exc.IntegrityError):
201 registry.insertDimensionData(dimensionName, dimensionValue)
202 # expandDataId should retrieve the record we just inserted
203 self.assertEqual(
204 registry.expandDataId(
205 instrument="DummyCam",
206 graph=dimension.graph
207 ).records[dimensionName].toDict(),
208 dimensionValue
209 )
210 # expandDataId should raise if there is no record with the given ID.
211 with self.assertRaises(LookupError):
212 registry.expandDataId({"instrument": "Unknown"}, graph=dimension.graph)
213 # abstract_filter doesn't have a table; insert should fail.
214 with self.assertRaises(TypeError):
215 registry.insertDimensionData("abstract_filter", {"abstract_filter": "i"})
216 dimensionName2 = "physical_filter"
217 dimension2 = registry.dimensions[dimensionName2]
218 dimensionValue2 = {"name": "DummyCam_i", "abstract_filter": "i"}
219 # Missing required dependency ("instrument") should fail
220 with self.assertRaises(sqlalchemy.exc.IntegrityError):
221 registry.insertDimensionData(dimensionName2, dimensionValue2)
222 # Adding required dependency should fix the failure
223 dimensionValue2["instrument"] = "DummyCam"
224 registry.insertDimensionData(dimensionName2, dimensionValue2)
225 # expandDataId should retrieve the record we just inserted.
226 self.assertEqual(
227 registry.expandDataId(
228 instrument="DummyCam", physical_filter="DummyCam_i",
229 graph=dimension2.graph
230 ).records[dimensionName2].toDict(),
231 dimensionValue2
232 )
233 # Use syncDimensionData to insert a new record successfully.
234 dimensionName3 = "detector"
235 dimensionValue3 = {"instrument": "DummyCam", "id": 1, "full_name": "one",
236 "name_in_raft": "zero", "purpose": "SCIENCE"}
237 self.assertTrue(registry.syncDimensionData(dimensionName3, dimensionValue3))
238 # Sync that again. Note that one field ("raft") is NULL, and that
239 # should be okay.
240 self.assertFalse(registry.syncDimensionData(dimensionName3, dimensionValue3))
241 # Now try that sync with the same primary key but a different value.
242 # This should fail.
243 with self.assertRaises(ConflictingDefinitionError):
244 registry.syncDimensionData(
245 dimensionName3,
246 {"instrument": "DummyCam", "id": 1, "full_name": "one",
247 "name_in_raft": "four", "purpose": "SCIENCE"}
248 )
250 @unittest.skipIf(np is None, "numpy not available.")
251 def testNumpyDataId(self):
252 """Test that we can use a numpy int in a dataId."""
253 registry = self.makeRegistry()
254 dimensionEntries = [
255 ("instrument", {"instrument": "DummyCam"}),
256 ("physical_filter", {"instrument": "DummyCam", "name": "d-r", "abstract_filter": "R"}),
257 # Using an np.int64 here fails unless Records.fromDict is also
258 # patched to look for numbers.Integral
259 ("visit", {"instrument": "DummyCam", "id": 42, "name": "fortytwo", "physical_filter": "d-r"}),
260 ]
261 for args in dimensionEntries:
262 registry.insertDimensionData(*args)
264 # Try a normal integer and something that looks like an int but
265 # is not.
266 for visit_id in (42, np.int64(42)):
267 with self.subTest(visit_id=visit_id, id_type=type(visit_id).__name__):
268 expanded = registry.expandDataId({"instrument": "DummyCam", "visit": visit_id})
269 self.assertEqual(expanded["visit"], int(visit_id))
270 self.assertIsInstance(expanded["visit"], int)
272 def testDataIdRelationships(self):
273 """Test that `Registry.expandDataId` raises an exception when the given
274 keys are inconsistent.
275 """
276 registry = self.makeRegistry()
277 self.loadData(registry, "base.yaml")
278 # Insert a few more dimension records for the next test.
279 registry.insertDimensionData(
280 "exposure",
281 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G"},
282 )
283 registry.insertDimensionData(
284 "exposure",
285 {"instrument": "Cam1", "id": 2, "name": "two", "physical_filter": "Cam1-G"},
286 )
287 registry.insertDimensionData(
288 "visit_system",
289 {"instrument": "Cam1", "id": 0, "name": "one-to-one"},
290 )
291 registry.insertDimensionData(
292 "visit",
293 {"instrument": "Cam1", "id": 1, "name": "one", "physical_filter": "Cam1-G", "visit_system": 0},
294 )
295 registry.insertDimensionData(
296 "visit_definition",
297 {"instrument": "Cam1", "visit": 1, "exposure": 1, "visit_system": 0},
298 )
299 with self.assertRaises(InconsistentDataIdError):
300 registry.expandDataId(
301 {"instrument": "Cam1", "visit": 1, "exposure": 2},
302 )
304 def testDataset(self):
305 """Basic tests for `Registry.insertDatasets`, `Registry.getDataset`,
306 and `Registry.removeDatasets`.
307 """
308 registry = self.makeRegistry()
309 self.loadData(registry, "base.yaml")
310 run = "test"
311 registry.registerRun(run)
312 datasetType = registry.getDatasetType("permabias")
313 dataId = {"instrument": "Cam1", "detector": 2}
314 ref, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run)
315 outRef = registry.getDataset(ref.id)
316 self.assertIsNotNone(ref.id)
317 self.assertEqual(ref, outRef)
318 with self.assertRaises(ConflictingDefinitionError):
319 registry.insertDatasets(datasetType, dataIds=[dataId], run=run)
320 registry.removeDatasets([ref])
321 self.assertIsNone(registry.findDataset(datasetType, dataId, collections=[run]))
323 def testFindDataset(self):
324 """Tests for `Registry.findDataset`.
325 """
326 registry = self.makeRegistry()
327 self.loadData(registry, "base.yaml")
328 run = "test"
329 datasetType = registry.getDatasetType("permabias")
330 dataId = {"instrument": "Cam1", "detector": 4}
331 registry.registerRun(run)
332 inputRef, = registry.insertDatasets(datasetType, dataIds=[dataId], run=run)
333 outputRef = registry.findDataset(datasetType, dataId, collections=[run])
334 self.assertEqual(outputRef, inputRef)
335 # Check that retrieval with invalid dataId raises
336 with self.assertRaises(LookupError):
337 dataId = {"instrument": "Cam1"} # no detector
338 registry.findDataset(datasetType, dataId, collections=run)
339 # Check that different dataIds match to different datasets
340 dataId1 = {"instrument": "Cam1", "detector": 1}
341 inputRef1, = registry.insertDatasets(datasetType, dataIds=[dataId1], run=run)
342 dataId2 = {"instrument": "Cam1", "detector": 2}
343 inputRef2, = registry.insertDatasets(datasetType, dataIds=[dataId2], run=run)
344 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef1)
345 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef2)
346 self.assertNotEqual(registry.findDataset(datasetType, dataId1, collections=run), inputRef2)
347 self.assertNotEqual(registry.findDataset(datasetType, dataId2, collections=run), inputRef1)
348 # Check that requesting a non-existing dataId returns None
349 nonExistingDataId = {"instrument": "Cam1", "detector": 3}
350 self.assertIsNone(registry.findDataset(datasetType, nonExistingDataId, collections=run))
352 def testDatasetTypeComponentQueries(self):
353 """Test component options when querying for dataset types.
354 """
355 registry = self.makeRegistry()
356 self.loadData(registry, "base.yaml")
357 self.loadData(registry, "datasets.yaml")
358 # Test querying for dataset types with different inputs.
359 # First query for all dataset types; components should only be included
360 # when components=True.
361 self.assertEqual(
362 {"permabias", "permaflat"},
363 NamedValueSet(registry.queryDatasetTypes()).names
364 )
365 self.assertEqual(
366 {"permabias", "permaflat"},
367 NamedValueSet(registry.queryDatasetTypes(components=False)).names
368 )
369 self.assertLess(
370 {"permabias", "permaflat", "permabias.wcs", "permaflat.photoCalib"},
371 NamedValueSet(registry.queryDatasetTypes(components=True)).names
372 )
373 # Use a pattern that can match either parent or components. Again,
374 # components are only returned if components=True.
375 self.assertEqual(
376 {"permabias"},
377 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"))).names
378 )
379 self.assertEqual(
380 {"permabias"},
381 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"), components=False)).names
382 )
383 self.assertLess(
384 {"permabias", "permabias.wcs"},
385 NamedValueSet(registry.queryDatasetTypes(re.compile(".+bias.*"), components=True)).names
386 )
387 # This pattern matches only a component. In this case we also return
388 # that component dataset type if components=None.
389 self.assertEqual(
390 {"permabias.wcs"},
391 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"))).names
392 )
393 self.assertEqual(
394 set(),
395 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"), components=False)).names
396 )
397 self.assertEqual(
398 {"permabias.wcs"},
399 NamedValueSet(registry.queryDatasetTypes(re.compile(r".+bias\.wcs"), components=True)).names
400 )
402 def testComponentLookups(self):
403 """Test searching for component datasets via their parents.
404 """
405 registry = self.makeRegistry()
406 self.loadData(registry, "base.yaml")
407 self.loadData(registry, "datasets.yaml")
408 # Test getting the child dataset type (which does still exist in the
409 # Registry), and check for consistency with
410 # DatasetRef.makeComponentRef.
411 collection = "imported_g"
412 parentType = registry.getDatasetType("permabias")
413 childType = registry.getDatasetType("permabias.wcs")
414 parentRefResolved = registry.findDataset(parentType, collections=collection,
415 instrument="Cam1", detector=1)
416 self.assertIsInstance(parentRefResolved, DatasetRef)
417 self.assertEqual(childType, parentRefResolved.makeComponentRef("wcs").datasetType)
418 # Search for a single dataset with findDataset.
419 childRef1 = registry.findDataset("permabias.wcs", collections=collection,
420 dataId=parentRefResolved.dataId)
421 self.assertEqual(childRef1, parentRefResolved.makeComponentRef("wcs"))
422 # Search for detector data IDs constrained by component dataset
423 # existence with queryDimensions.
424 dataIds = set(registry.queryDimensions(
425 ["detector"],
426 datasets=["permabias.wcs"],
427 collections=collection,
428 expand=False,
429 ))
430 self.assertEqual(
431 dataIds,
432 {
433 DataCoordinate.standardize(instrument="Cam1", detector=d, graph=parentType.dimensions)
434 for d in (1, 2, 3)
435 }
436 )
437 # Search for multiple datasets of a single type with queryDatasets.
438 childRefs2 = set(registry.queryDatasets(
439 "permabias.wcs",
440 collections=collection,
441 expand=False,
442 ))
443 self.assertEqual(
444 {ref.unresolved() for ref in childRefs2},
445 {DatasetRef(childType, dataId) for dataId in dataIds}
446 )
448 def testCollections(self):
449 """Tests for registry methods that manage collections.
450 """
451 registry = self.makeRegistry()
452 self.loadData(registry, "base.yaml")
453 self.loadData(registry, "datasets.yaml")
454 run1 = "imported_g"
455 run2 = "imported_r"
456 datasetType = "permabias"
457 # Find some datasets via their run's collection.
458 dataId1 = {"instrument": "Cam1", "detector": 1}
459 ref1 = registry.findDataset(datasetType, dataId1, collections=run1)
460 self.assertIsNotNone(ref1)
461 dataId2 = {"instrument": "Cam1", "detector": 2}
462 ref2 = registry.findDataset(datasetType, dataId2, collections=run1)
463 self.assertIsNotNone(ref2)
464 # Associate those into a new collection,then look for them there.
465 tag1 = "tag1"
466 registry.registerCollection(tag1, type=CollectionType.TAGGED)
467 registry.associate(tag1, [ref1, ref2])
468 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1)
469 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
470 # Disassociate one and verify that we can't it there anymore...
471 registry.disassociate(tag1, [ref1])
472 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=tag1))
473 # ...but we can still find ref2 in tag1, and ref1 in the run.
474 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=run1), ref1)
475 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
476 collections = set(registry.queryCollections())
477 self.assertEqual(collections, {run1, run2, tag1})
478 # Associate both refs into tag1 again; ref2 is already there, but that
479 # should be a harmless no-op.
480 registry.associate(tag1, [ref1, ref2])
481 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1)
482 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
483 # Get a different dataset (from a different run) that has the same
484 # dataset type and data ID as ref2.
485 ref2b = registry.findDataset(datasetType, dataId2, collections=run2)
486 self.assertNotEqual(ref2, ref2b)
487 # Attempting to associate that into tag1 should be an error.
488 with self.assertRaises(ConflictingDefinitionError):
489 registry.associate(tag1, [ref2b])
490 # That error shouldn't have messed up what we had before.
491 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1)
492 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
493 # Attempt to associate the conflicting dataset again, this time with
494 # a dataset that isn't in the collection and won't cause a conflict.
495 # Should also fail without modifying anything.
496 dataId3 = {"instrument": "Cam1", "detector": 3}
497 ref3 = registry.findDataset(datasetType, dataId3, collections=run1)
498 with self.assertRaises(ConflictingDefinitionError):
499 registry.associate(tag1, [ref3, ref2b])
500 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=tag1), ref1)
501 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=tag1), ref2)
502 self.assertIsNone(registry.findDataset(datasetType, dataId3, collections=tag1))
503 # Register a chained collection that searches:
504 # 1. 'tag1'
505 # 2. 'run1', but only for the permaflat dataset
506 # 3. 'run2'
507 chain1 = "chain1"
508 registry.registerCollection(chain1, type=CollectionType.CHAINED)
509 self.assertIs(registry.getCollectionType(chain1), CollectionType.CHAINED)
510 # Chained collection exists, but has no collections in it.
511 self.assertFalse(registry.getCollectionChain(chain1))
512 # If we query for all collections, we should get the chained collection
513 # only if we don't ask to flatten it (i.e. yield only its children).
514 self.assertEqual(set(registry.queryCollections(flattenChains=False)), {tag1, run1, run2, chain1})
515 self.assertEqual(set(registry.queryCollections(flattenChains=True)), {tag1, run1, run2})
516 # Attempt to set its child collections to something circular; that
517 # should fail.
518 with self.assertRaises(ValueError):
519 registry.setCollectionChain(chain1, [tag1, chain1])
520 # Add the child collections.
521 registry.setCollectionChain(chain1, [tag1, (run1, "permaflat"), run2])
522 self.assertEqual(
523 list(registry.getCollectionChain(chain1)),
524 [(tag1, DatasetTypeRestriction.any),
525 (run1, DatasetTypeRestriction.fromExpression("permaflat")),
526 (run2, DatasetTypeRestriction.any)]
527 )
528 # Searching for dataId1 or dataId2 in the chain should return ref1 and
529 # ref2, because both are in tag1.
530 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain1), ref1)
531 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain1), ref2)
532 # Now disassociate ref2 from tag1. The search (for permabias) with
533 # dataId2 in chain1 should then:
534 # 1. not find it in tag1
535 # 2. not look in tag2, because it's restricted to permaflat here
536 # 3. find a different dataset in run2
537 registry.disassociate(tag1, [ref2])
538 ref2b = registry.findDataset(datasetType, dataId2, collections=chain1)
539 self.assertNotEqual(ref2b, ref2)
540 self.assertEqual(ref2b, registry.findDataset(datasetType, dataId2, collections=run2))
541 # Look in the chain for a permaflat that is in run1; should get the
542 # same ref as if we'd searched run1 directly.
543 dataId3 = {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}
544 self.assertEqual(registry.findDataset("permaflat", dataId3, collections=chain1),
545 registry.findDataset("permaflat", dataId3, collections=run1),)
546 # Define a new chain so we can test recursive chains.
547 chain2 = "chain2"
548 registry.registerCollection(chain2, type=CollectionType.CHAINED)
549 registry.setCollectionChain(chain2, [(run2, "permabias"), chain1])
550 # Query for collections matching a regex.
551 self.assertCountEqual(
552 list(registry.queryCollections(re.compile("imported_."), flattenChains=False)),
553 ["imported_r", "imported_g"]
554 )
555 # Query for collections matching a regex or an explicit str.
556 self.assertCountEqual(
557 list(registry.queryCollections([re.compile("imported_."), "chain1"], flattenChains=False)),
558 ["imported_r", "imported_g", "chain1"]
559 )
560 # Search for permabias with dataId1 should find it via tag1 in chain2,
561 # recursing, because is not in run1.
562 self.assertIsNone(registry.findDataset(datasetType, dataId1, collections=run2))
563 self.assertEqual(registry.findDataset(datasetType, dataId1, collections=chain2), ref1)
564 # Search for permabias with dataId2 should find it in run2 (ref2b).
565 self.assertEqual(registry.findDataset(datasetType, dataId2, collections=chain2), ref2b)
566 # Search for a permaflat that is in run2. That should not be found
567 # at the front of chain2, because of the restriction to permabias
568 # on run2 there, but it should be found in at the end of chain1.
569 dataId4 = {"instrument": "Cam1", "detector": 3, "physical_filter": "Cam1-R2"}
570 ref4 = registry.findDataset("permaflat", dataId4, collections=run2)
571 self.assertIsNotNone(ref4)
572 self.assertEqual(ref4, registry.findDataset("permaflat", dataId4, collections=chain2))
573 # Deleting a collection that's part of a CHAINED collection is not
574 # allowed, and is exception-safe.
575 with self.assertRaises(Exception):
576 registry.removeCollection(run2)
577 self.assertEqual(registry.getCollectionType(run2), CollectionType.RUN)
578 with self.assertRaises(Exception):
579 registry.removeCollection(chain1)
580 self.assertEqual(registry.getCollectionType(chain1), CollectionType.CHAINED)
581 # Actually remove chain2, test that it's gone by asking for its type.
582 registry.removeCollection(chain2)
583 with self.assertRaises(MissingCollectionError):
584 registry.getCollectionType(chain2)
585 # Actually remove run2 and chain1, which should work now.
586 registry.removeCollection(chain1)
587 registry.removeCollection(run2)
588 with self.assertRaises(MissingCollectionError):
589 registry.getCollectionType(run2)
590 with self.assertRaises(MissingCollectionError):
591 registry.getCollectionType(chain1)
592 # Remove tag1 as well, just to test that we can remove TAGGED
593 # collections.
594 registry.removeCollection(tag1)
595 with self.assertRaises(MissingCollectionError):
596 registry.getCollectionType(tag1)
598 def testBasicTransaction(self):
599 """Test that all operations within a single transaction block are
600 rolled back if an exception propagates out of the block.
601 """
602 registry = self.makeRegistry()
603 storageClass = StorageClass("testDatasetType")
604 registry.storageClasses.registerStorageClass(storageClass)
605 with registry.transaction():
606 registry.insertDimensionData("instrument", {"name": "Cam1", "class_name": "A"})
607 with self.assertRaises(ValueError):
608 with registry.transaction():
609 registry.insertDimensionData("instrument", {"name": "Cam2"})
610 raise ValueError("Oops, something went wrong")
611 # Cam1 should exist
612 self.assertEqual(registry.expandDataId(instrument="Cam1").records["instrument"].class_name, "A")
613 # But Cam2 and Cam3 should both not exist
614 with self.assertRaises(LookupError):
615 registry.expandDataId(instrument="Cam2")
616 with self.assertRaises(LookupError):
617 registry.expandDataId(instrument="Cam3")
619 def testNestedTransaction(self):
620 """Test that operations within a transaction block are not rolled back
621 if an exception propagates out of an inner transaction block and is
622 then caught.
623 """
624 registry = self.makeRegistry()
625 dimension = registry.dimensions["instrument"]
626 dataId1 = {"instrument": "DummyCam"}
627 dataId2 = {"instrument": "DummyCam2"}
628 checkpointReached = False
629 with registry.transaction():
630 # This should be added and (ultimately) committed.
631 registry.insertDimensionData(dimension, dataId1)
632 with self.assertRaises(sqlalchemy.exc.IntegrityError):
633 with registry.transaction():
634 # This does not conflict, and should succeed (but not
635 # be committed).
636 registry.insertDimensionData(dimension, dataId2)
637 checkpointReached = True
638 # This should conflict and raise, triggerring a rollback
639 # of the previous insertion within the same transaction
640 # context, but not the original insertion in the outer
641 # block.
642 registry.insertDimensionData(dimension, dataId1)
643 self.assertTrue(checkpointReached)
644 self.assertIsNotNone(registry.expandDataId(dataId1, graph=dimension.graph))
645 with self.assertRaises(LookupError):
646 registry.expandDataId(dataId2, graph=dimension.graph)
648 def testInstrumentDimensions(self):
649 """Test queries involving only instrument dimensions, with no joins to
650 skymap."""
651 registry = self.makeRegistry()
653 # need a bunch of dimensions and datasets for test
654 registry.insertDimensionData(
655 "instrument",
656 dict(name="DummyCam", visit_max=25, exposure_max=300, detector_max=6)
657 )
658 registry.insertDimensionData(
659 "physical_filter",
660 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"),
661 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"),
662 )
663 registry.insertDimensionData(
664 "detector",
665 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in range(1, 6)]
666 )
667 registry.insertDimensionData(
668 "visit_system",
669 dict(instrument="DummyCam", id=1, name="default"),
670 )
671 registry.insertDimensionData(
672 "visit",
673 dict(instrument="DummyCam", id=10, name="ten", physical_filter="dummy_i", visit_system=1),
674 dict(instrument="DummyCam", id=11, name="eleven", physical_filter="dummy_r", visit_system=1),
675 dict(instrument="DummyCam", id=20, name="twelve", physical_filter="dummy_r", visit_system=1),
676 )
677 registry.insertDimensionData(
678 "exposure",
679 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i"),
680 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i"),
681 dict(instrument="DummyCam", id=110, name="110", physical_filter="dummy_r"),
682 dict(instrument="DummyCam", id=111, name="111", physical_filter="dummy_r"),
683 dict(instrument="DummyCam", id=200, name="200", physical_filter="dummy_r"),
684 dict(instrument="DummyCam", id=201, name="201", physical_filter="dummy_r"),
685 )
686 registry.insertDimensionData(
687 "visit_definition",
688 dict(instrument="DummyCam", exposure=100, visit_system=1, visit=10),
689 dict(instrument="DummyCam", exposure=101, visit_system=1, visit=10),
690 dict(instrument="DummyCam", exposure=110, visit_system=1, visit=11),
691 dict(instrument="DummyCam", exposure=111, visit_system=1, visit=11),
692 dict(instrument="DummyCam", exposure=200, visit_system=1, visit=20),
693 dict(instrument="DummyCam", exposure=201, visit_system=1, visit=20),
694 )
695 # dataset types
696 run1 = "test1_r"
697 run2 = "test2_r"
698 tagged2 = "test2_t"
699 registry.registerRun(run1)
700 registry.registerRun(run2)
701 registry.registerCollection(tagged2)
702 storageClass = StorageClass("testDataset")
703 registry.storageClasses.registerStorageClass(storageClass)
704 rawType = DatasetType(name="RAW",
705 dimensions=registry.dimensions.extract(("instrument", "exposure", "detector")),
706 storageClass=storageClass)
707 registry.registerDatasetType(rawType)
708 calexpType = DatasetType(name="CALEXP",
709 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")),
710 storageClass=storageClass)
711 registry.registerDatasetType(calexpType)
713 # add pre-existing datasets
714 for exposure in (100, 101, 110, 111):
715 for detector in (1, 2, 3):
716 # note that only 3 of 5 detectors have datasets
717 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector)
718 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run1)
719 # exposures 100 and 101 appear in both run1 and tagged2.
720 # 100 has different datasets in the different collections
721 # 101 has the same dataset in both collections.
722 if exposure == 100:
723 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2)
724 if exposure in (100, 101):
725 registry.associate(tagged2, [ref])
726 # Add pre-existing datasets to tagged2.
727 for exposure in (200, 201):
728 for detector in (3, 4, 5):
729 # note that only 3 of 5 detectors have datasets
730 dataId = dict(instrument="DummyCam", exposure=exposure, detector=detector)
731 ref, = registry.insertDatasets(rawType, dataIds=[dataId], run=run2)
732 registry.associate(tagged2, [ref])
734 dimensions = DimensionGraph(
735 registry.dimensions,
736 dimensions=(rawType.dimensions.required | calexpType.dimensions.required)
737 )
738 # Test that single dim string works as well as list of str
739 rows = list(registry.queryDimensions("visit", datasets=rawType, collections=run1, expand=True))
740 rowsI = list(registry.queryDimensions(["visit"], datasets=rawType, collections=run1, expand=True))
741 self.assertEqual(rows, rowsI)
742 # with empty expression
743 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1, expand=True))
744 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors
745 for dataId in rows:
746 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit"))
747 packer1 = registry.dimensions.makePacker("visit_detector", dataId)
748 packer2 = registry.dimensions.makePacker("exposure_detector", dataId)
749 self.assertEqual(packer1.unpack(packer1.pack(dataId)),
750 DataCoordinate.standardize(dataId, graph=packer1.dimensions))
751 self.assertEqual(packer2.unpack(packer2.pack(dataId)),
752 DataCoordinate.standardize(dataId, graph=packer2.dimensions))
753 self.assertNotEqual(packer1.pack(dataId), packer2.pack(dataId))
754 self.assertCountEqual(set(dataId["exposure"] for dataId in rows),
755 (100, 101, 110, 111))
756 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11))
757 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3))
759 # second collection
760 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=tagged2))
761 self.assertEqual(len(rows), 4*3) # 4 exposures times 3 detectors
762 for dataId in rows:
763 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit"))
764 self.assertCountEqual(set(dataId["exposure"] for dataId in rows),
765 (100, 101, 200, 201))
766 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 20))
767 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5))
769 # with two input datasets
770 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=[run1, tagged2]))
771 self.assertEqual(len(set(rows)), 6*3) # 6 exposures times 3 detectors; set needed to de-dupe
772 for dataId in rows:
773 self.assertCountEqual(dataId.keys(), ("instrument", "detector", "exposure", "visit"))
774 self.assertCountEqual(set(dataId["exposure"] for dataId in rows),
775 (100, 101, 110, 111, 200, 201))
776 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10, 11, 20))
777 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3, 4, 5))
779 # limit to single visit
780 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1,
781 where="visit = 10"))
782 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors
783 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101))
784 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,))
785 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3))
787 # more limiting expression, using link names instead of Table.column
788 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1,
789 where="visit = 10 and detector > 1"))
790 self.assertEqual(len(rows), 2*2) # 2 exposures times 2 detectors
791 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (100, 101))
792 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (10,))
793 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (2, 3))
795 # expression excludes everything
796 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1,
797 where="visit > 1000"))
798 self.assertEqual(len(rows), 0)
800 # Selecting by physical_filter, this is not in the dimensions, but it
801 # is a part of the full expression so it should work too.
802 rows = list(registry.queryDimensions(dimensions, datasets=rawType, collections=run1,
803 where="physical_filter = 'dummy_r'"))
804 self.assertEqual(len(rows), 2*3) # 2 exposures times 3 detectors
805 self.assertCountEqual(set(dataId["exposure"] for dataId in rows), (110, 111))
806 self.assertCountEqual(set(dataId["visit"] for dataId in rows), (11,))
807 self.assertCountEqual(set(dataId["detector"] for dataId in rows), (1, 2, 3))
809 def testSkyMapDimensions(self):
810 """Tests involving only skymap dimensions, no joins to instrument."""
811 registry = self.makeRegistry()
813 # need a bunch of dimensions and datasets for test, we want
814 # "abstract_filter" in the test so also have to add physical_filter
815 # dimensions
816 registry.insertDimensionData(
817 "instrument",
818 dict(instrument="DummyCam")
819 )
820 registry.insertDimensionData(
821 "physical_filter",
822 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"),
823 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"),
824 )
825 registry.insertDimensionData(
826 "skymap",
827 dict(name="DummyMap", hash="sha!".encode("utf8"))
828 )
829 for tract in range(10):
830 registry.insertDimensionData("tract", dict(skymap="DummyMap", id=tract))
831 registry.insertDimensionData(
832 "patch",
833 *[dict(skymap="DummyMap", tract=tract, id=patch, cell_x=0, cell_y=0)
834 for patch in range(10)]
835 )
837 # dataset types
838 run = "test"
839 registry.registerRun(run)
840 storageClass = StorageClass("testDataset")
841 registry.storageClasses.registerStorageClass(storageClass)
842 calexpType = DatasetType(name="deepCoadd_calexp",
843 dimensions=registry.dimensions.extract(("skymap", "tract", "patch",
844 "abstract_filter")),
845 storageClass=storageClass)
846 registry.registerDatasetType(calexpType)
847 mergeType = DatasetType(name="deepCoadd_mergeDet",
848 dimensions=registry.dimensions.extract(("skymap", "tract", "patch")),
849 storageClass=storageClass)
850 registry.registerDatasetType(mergeType)
851 measType = DatasetType(name="deepCoadd_meas",
852 dimensions=registry.dimensions.extract(("skymap", "tract", "patch",
853 "abstract_filter")),
854 storageClass=storageClass)
855 registry.registerDatasetType(measType)
857 dimensions = DimensionGraph(
858 registry.dimensions,
859 dimensions=(calexpType.dimensions.required | mergeType.dimensions.required
860 | measType.dimensions.required)
861 )
863 # add pre-existing datasets
864 for tract in (1, 3, 5):
865 for patch in (2, 4, 6, 7):
866 dataId = dict(skymap="DummyMap", tract=tract, patch=patch)
867 registry.insertDatasets(mergeType, dataIds=[dataId], run=run)
868 for aFilter in ("i", "r"):
869 dataId = dict(skymap="DummyMap", tract=tract, patch=patch, abstract_filter=aFilter)
870 registry.insertDatasets(calexpType, dataIds=[dataId], run=run)
872 # with empty expression
873 rows = list(registry.queryDimensions(dimensions,
874 datasets=[calexpType, mergeType], collections=run))
875 self.assertEqual(len(rows), 3*4*2) # 4 tracts x 4 patches x 2 filters
876 for dataId in rows:
877 self.assertCountEqual(dataId.keys(), ("skymap", "tract", "patch", "abstract_filter"))
878 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5))
879 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7))
880 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r"))
882 # limit to 2 tracts and 2 patches
883 rows = list(registry.queryDimensions(dimensions,
884 datasets=[calexpType, mergeType], collections=run,
885 where="tract IN (1, 5) AND patch IN (2, 7)"))
886 self.assertEqual(len(rows), 2*2*2) # 2 tracts x 2 patches x 2 filters
887 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 5))
888 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 7))
889 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r"))
891 # limit to single filter
892 rows = list(registry.queryDimensions(dimensions,
893 datasets=[calexpType, mergeType], collections=run,
894 where="abstract_filter = 'i'"))
895 self.assertEqual(len(rows), 3*4*1) # 4 tracts x 4 patches x 2 filters
896 self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5))
897 self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7))
898 self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i",))
900 # expression excludes everything, specifying non-existing skymap is
901 # not a fatal error, it's operator error
902 rows = list(registry.queryDimensions(dimensions,
903 datasets=[calexpType, mergeType], collections=run,
904 where="skymap = 'Mars'"))
905 self.assertEqual(len(rows), 0)
907 def testSpatialMatch(self):
908 """Test involving spatial match using join tables.
910 Note that realistic test needs a reasonably-defined skypix and regions
911 in registry tables which is hard to implement in this simple test.
912 So we do not actually fill registry with any data and all queries will
913 return empty result, but this is still useful for coverage of the code
914 that generates query.
915 """
916 registry = self.makeRegistry()
918 # dataset types
919 collection = "test"
920 registry.registerRun(name=collection)
921 storageClass = StorageClass("testDataset")
922 registry.storageClasses.registerStorageClass(storageClass)
924 calexpType = DatasetType(name="CALEXP",
925 dimensions=registry.dimensions.extract(("instrument", "visit", "detector")),
926 storageClass=storageClass)
927 registry.registerDatasetType(calexpType)
929 coaddType = DatasetType(name="deepCoadd_calexp",
930 dimensions=registry.dimensions.extract(("skymap", "tract", "patch",
931 "abstract_filter")),
932 storageClass=storageClass)
933 registry.registerDatasetType(coaddType)
935 dimensions = DimensionGraph(
936 registry.dimensions,
937 dimensions=(calexpType.dimensions.required | coaddType.dimensions.required)
938 )
940 # without data this should run OK but return empty set
941 rows = list(registry.queryDimensions(dimensions, datasets=calexpType, collections=collection))
942 self.assertEqual(len(rows), 0)
944 def testCalibrationLabelIndirection(self):
945 """Test that we can look up datasets with calibration_label dimensions
946 from a data ID with exposure dimensions.
947 """
949 def _dt(iso_string):
950 return astropy.time.Time(iso_string, format="iso", scale="utc")
952 registry = self.makeRegistry()
954 flat = DatasetType(
955 "flat",
956 registry.dimensions.extract(
957 ["instrument", "detector", "physical_filter", "calibration_label"]
958 ),
959 "ImageU"
960 )
961 registry.registerDatasetType(flat)
962 registry.insertDimensionData("instrument", dict(name="DummyCam"))
963 registry.insertDimensionData(
964 "physical_filter",
965 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"),
966 )
967 registry.insertDimensionData(
968 "detector",
969 *[dict(instrument="DummyCam", id=i, full_name=str(i)) for i in (1, 2, 3, 4, 5)]
970 )
971 registry.insertDimensionData(
972 "exposure",
973 dict(instrument="DummyCam", id=100, name="100", physical_filter="dummy_i",
974 datetime_begin=_dt("2005-12-15 02:00:00"), datetime_end=_dt("2005-12-15 03:00:00")),
975 dict(instrument="DummyCam", id=101, name="101", physical_filter="dummy_i",
976 datetime_begin=_dt("2005-12-16 02:00:00"), datetime_end=_dt("2005-12-16 03:00:00")),
977 )
978 registry.insertDimensionData(
979 "calibration_label",
980 dict(instrument="DummyCam", name="first_night",
981 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-15 04:00:00")),
982 dict(instrument="DummyCam", name="second_night",
983 datetime_begin=_dt("2005-12-16 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")),
984 dict(instrument="DummyCam", name="both_nights",
985 datetime_begin=_dt("2005-12-15 01:00:00"), datetime_end=_dt("2005-12-16 04:00:00")),
986 )
987 # Different flats for different nights for detectors 1-3 in first
988 # collection.
989 run1 = "calibs1"
990 registry.registerRun(run1)
991 for detector in (1, 2, 3):
992 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="first_night",
993 physical_filter="dummy_i", detector=detector)],
994 run=run1)
995 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="second_night",
996 physical_filter="dummy_i", detector=detector)],
997 run=run1)
998 # The same flat for both nights for detectors 3-5 (so detector 3 has
999 # multiple valid flats) in second collection.
1000 run2 = "calib2"
1001 registry.registerRun(run2)
1002 for detector in (3, 4, 5):
1003 registry.insertDatasets(flat, [dict(instrument="DummyCam", calibration_label="both_nights",
1004 physical_filter="dummy_i", detector=detector)],
1005 run=run2)
1006 # Perform queries for individual exposure+detector combinations, which
1007 # should always return exactly one flat.
1008 for exposure in (100, 101):
1009 for detector in (1, 2, 3):
1010 with self.subTest(exposure=exposure, detector=detector):
1011 rows = list(registry.queryDatasets("flat", collections=[run1],
1012 instrument="DummyCam",
1013 exposure=exposure,
1014 detector=detector))
1015 self.assertEqual(len(rows), 1)
1016 for detector in (3, 4, 5):
1017 with self.subTest(exposure=exposure, detector=detector):
1018 rows = registry.queryDatasets("flat", collections=[run2],
1019 instrument="DummyCam",
1020 exposure=exposure,
1021 detector=detector)
1022 self.assertEqual(len(list(rows)), 1)
1023 for detector in (1, 2, 4, 5):
1024 with self.subTest(exposure=exposure, detector=detector):
1025 rows = registry.queryDatasets("flat", collections=[run1, run2],
1026 instrument="DummyCam",
1027 exposure=exposure,
1028 detector=detector)
1029 self.assertEqual(len(list(rows)), 1)
1030 for detector in (3,):
1031 with self.subTest(exposure=exposure, detector=detector):
1032 rows = registry.queryDatasets("flat", collections=[run1, run2],
1033 instrument="DummyCam",
1034 exposure=exposure,
1035 detector=detector)
1036 self.assertEqual(len(list(rows)), 2)
1038 def testAbstractFilterQuery(self):
1039 """Test that we can run a query that just lists the known
1040 abstract_filters. This is tricky because abstract_filter is
1041 backed by a query against physical_filter.
1042 """
1043 registry = self.makeRegistry()
1044 registry.insertDimensionData("instrument", dict(name="DummyCam"))
1045 registry.insertDimensionData(
1046 "physical_filter",
1047 dict(instrument="DummyCam", name="dummy_i", abstract_filter="i"),
1048 dict(instrument="DummyCam", name="dummy_i2", abstract_filter="i"),
1049 dict(instrument="DummyCam", name="dummy_r", abstract_filter="r"),
1050 )
1051 rows = list(registry.queryDimensions(["abstract_filter"]))
1052 self.assertCountEqual(
1053 rows,
1054 [DataCoordinate.standardize(abstract_filter="i", universe=registry.dimensions),
1055 DataCoordinate.standardize(abstract_filter="r", universe=registry.dimensions)]
1056 )
1058 def testAttributeManager(self):
1059 """Test basic functionality of attribute manager.
1060 """
1061 # number of attributes with schema versions in a fresh database,
1062 # 6 managers with 3 records per manager
1063 VERSION_COUNT = 6 * 3
1065 registry = self.makeRegistry()
1066 attributes = registry._attributes
1068 # check what get() returns for non-existing key
1069 self.assertIsNone(attributes.get("attr"))
1070 self.assertEqual(attributes.get("attr", ""), "")
1071 self.assertEqual(attributes.get("attr", "Value"), "Value")
1072 self.assertEqual(len(list(attributes.items())), VERSION_COUNT)
1074 # cannot store empty key or value
1075 with self.assertRaises(ValueError):
1076 attributes.set("", "value")
1077 with self.assertRaises(ValueError):
1078 attributes.set("attr", "")
1080 # set value of non-existing key
1081 attributes.set("attr", "value")
1082 self.assertEqual(len(list(attributes.items())), VERSION_COUNT + 1)
1083 self.assertEqual(attributes.get("attr"), "value")
1085 # update value of existing key
1086 with self.assertRaises(ButlerAttributeExistsError):
1087 attributes.set("attr", "value2")
1089 attributes.set("attr", "value2", force=True)
1090 self.assertEqual(len(list(attributes.items())), VERSION_COUNT + 1)
1091 self.assertEqual(attributes.get("attr"), "value2")
1093 # delete existing key
1094 self.assertTrue(attributes.delete("attr"))
1095 self.assertEqual(len(list(attributes.items())), VERSION_COUNT)
1097 # delete non-existing key
1098 self.assertFalse(attributes.delete("non-attr"))
1100 # store bunch of keys and get the list back
1101 data = [
1102 ("version.core", "1.2.3"),
1103 ("version.dimensions", "3.2.1"),
1104 ("config.managers.opaque", "ByNameOpaqueTableStorageManager"),
1105 ]
1106 for key, value in data:
1107 attributes.set(key, value)
1108 items = dict(attributes.items())
1109 for key, value in data:
1110 self.assertEqual(items[key], value)