Coverage for tests/test_simpleButler.py: 12%
271 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-04 02:04 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-04 02:04 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import json
25import os
26import re
27import tempfile
28import unittest
29import uuid
30from typing import Any
32try:
33 import numpy as np
34except ImportError:
35 np = None
37import astropy.time
38from lsst.daf.butler import Butler, ButlerConfig, CollectionType, DatasetRef, DatasetType, Registry, Timespan
39from lsst.daf.butler.registry import ConflictingDefinitionError, RegistryConfig, RegistryDefaults
40from lsst.daf.butler.tests import DatastoreMock
41from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
43TESTDIR = os.path.abspath(os.path.dirname(__file__))
46class SimpleButlerTestCase(unittest.TestCase):
47 """Tests for butler (including import/export functionality) that should not
48 depend on the Registry Database backend or Datastore implementation, and
49 can instead utilize an in-memory SQLite Registry and a mocked Datastore.
50 """
52 datasetsManager = (
53 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
54 )
55 datasetsImportFile = "datasets-uuid.yaml"
56 datasetsIdType = uuid.UUID
58 def setUp(self):
59 self.root = makeTestTempDir(TESTDIR)
61 def tearDown(self):
62 removeTestTempDir(self.root)
64 def makeButler(self, **kwargs: Any) -> Butler:
65 """Return new Butler instance on each call."""
66 config = ButlerConfig()
68 # make separate temporary directory for registry of this instance
69 tmpdir = tempfile.mkdtemp(dir=self.root)
70 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"
71 config["registry", "managers", "datasets"] = self.datasetsManager
72 config["root"] = self.root
74 # have to make a registry first
75 registryConfig = RegistryConfig(config.get("registry"))
76 if self.datasetsIdType is int:
77 with self.assertWarns(FutureWarning):
78 Registry.createFromConfig(registryConfig)
79 else:
80 Registry.createFromConfig(registryConfig)
82 butler = Butler(config, **kwargs)
83 DatastoreMock.apply(butler)
84 return butler
86 def comparableRef(self, ref: DatasetRef) -> DatasetRef:
87 """Return a DatasetRef that can be compared to a DatasetRef from
88 other repository.
90 For repositories that do not support round-trip of ID values this
91 method returns unresolved DatasetRef, for round-trip-safe repos it
92 returns unchanged ref.
93 """
94 return ref if self.datasetsIdType is uuid.UUID else ref.unresolved()
96 def testReadBackwardsCompatibility(self):
97 """Test that we can read an export file written by a previous version
98 and commit to the daf_butler git repo.
100 Notes
101 -----
102 At present this export file includes only dimension data, not datasets,
103 which greatly limits the usefulness of this test. We should address
104 this at some point, but I think it's best to wait for the changes to
105 the export format required for CALIBRATION collections to land.
106 """
107 butler = self.makeButler(writeable=True)
108 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "hsc-rc2-subset.yaml"))
109 # Spot-check a few things, but the most important test is just that
110 # the above does not raise.
111 self.assertGreaterEqual(
112 set(record.id for record in butler.registry.queryDimensionRecords("detector", instrument="HSC")),
113 set(range(104)), # should have all science CCDs; may have some focus ones.
114 )
115 self.assertGreaterEqual(
116 {
117 (record.id, record.physical_filter)
118 for record in butler.registry.queryDimensionRecords("visit", instrument="HSC")
119 },
120 {
121 (27136, "HSC-Z"),
122 (11694, "HSC-G"),
123 (23910, "HSC-R"),
124 (11720, "HSC-Y"),
125 (23900, "HSC-R"),
126 (22646, "HSC-Y"),
127 (1248, "HSC-I"),
128 (19680, "HSC-I"),
129 (1240, "HSC-I"),
130 (424, "HSC-Y"),
131 (19658, "HSC-I"),
132 (344, "HSC-Y"),
133 (1218, "HSC-R"),
134 (1190, "HSC-Z"),
135 (23718, "HSC-R"),
136 (11700, "HSC-G"),
137 (26036, "HSC-G"),
138 (23872, "HSC-R"),
139 (1170, "HSC-Z"),
140 (1876, "HSC-Y"),
141 },
142 )
144 def testDatasetTransfers(self):
145 """Test exporting all datasets from a repo and then importing them all
146 back in again.
147 """
148 # Import data to play with.
149 butler1 = self.makeButler(writeable=True)
150 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
151 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile))
152 with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml") as file:
153 # Export all datasets.
154 with butler1.export(filename=file.name) as exporter:
155 exporter.saveDatasets(butler1.registry.queryDatasets(..., collections=...))
156 # Import it all again.
157 butler2 = self.makeButler(writeable=True)
158 butler2.import_(filename=file.name)
159 datasets1 = list(butler1.registry.queryDatasets(..., collections=...))
160 datasets2 = list(butler2.registry.queryDatasets(..., collections=...))
161 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets1))
162 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets2))
163 self.assertCountEqual(
164 [self.comparableRef(ref) for ref in datasets1],
165 [self.comparableRef(ref) for ref in datasets2],
166 )
168 def testImportTwice(self):
169 """Test exporting dimension records and datasets from a repo and then
170 importing them all back in again twice.
171 """
172 if self.datasetsIdType is not uuid.UUID:
173 self.skipTest("This test can only work for UUIDs")
174 # Import data to play with.
175 butler1 = self.makeButler(writeable=True)
176 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
177 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile))
178 with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as file:
179 # Export all datasets.
180 with butler1.export(filename=file.name) as exporter:
181 exporter.saveDatasets(butler1.registry.queryDatasets(..., collections=...))
182 butler2 = self.makeButler(writeable=True)
183 # Import it once.
184 butler2.import_(filename=file.name)
185 # Import it again
186 butler2.import_(filename=file.name)
187 datasets1 = list(butler1.registry.queryDatasets(..., collections=...))
188 datasets2 = list(butler2.registry.queryDatasets(..., collections=...))
189 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets1))
190 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets2))
191 self.assertCountEqual(
192 [self.comparableRef(ref) for ref in datasets1],
193 [self.comparableRef(ref) for ref in datasets2],
194 )
196 def testDatasetImportReuseIds(self):
197 """Test for import that should preserve dataset IDs.
199 This test assumes that dataset IDs in datasets YAML are different from
200 what auto-incremental insert would produce.
201 """
202 if self.datasetsIdType is not int:
203 self.skipTest("This test can only work for UUIDs")
204 # Import data to play with.
205 butler = self.makeButler(writeable=True)
206 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
207 filename = os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile)
208 butler.import_(filename=filename, reuseIds=True)
209 datasets = list(butler.registry.queryDatasets(..., collections=...))
210 self.assertTrue(all(isinstance(ref.id, self.datasetsIdType) for ref in datasets))
211 # IDs are copied from YAML, list needs to be updated if file contents
212 # is changed.
213 self.assertCountEqual(
214 [ref.id for ref in datasets],
215 [1001, 1002, 1003, 1010, 1020, 1030, 2001, 2002, 2003, 2010, 2020, 2030, 2040],
216 )
218 # Try once again, it will raise
219 with self.assertRaises(ConflictingDefinitionError):
220 butler.import_(filename=filename, reuseIds=True)
222 def testCollectionTransfers(self):
223 """Test exporting and then importing collections of various types."""
224 # Populate a registry with some datasets.
225 butler1 = self.makeButler(writeable=True)
226 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
227 butler1.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile))
228 registry1 = butler1.registry
229 # Add some more collections.
230 registry1.registerRun("run1")
231 registry1.registerCollection("tag1", CollectionType.TAGGED)
232 registry1.registerCollection("calibration1", CollectionType.CALIBRATION)
233 registry1.registerCollection("chain1", CollectionType.CHAINED)
234 registry1.registerCollection("chain2", CollectionType.CHAINED)
235 registry1.setCollectionChain("chain1", ["tag1", "run1", "chain2"])
236 registry1.setCollectionChain("chain2", ["calibration1", "run1"])
237 # Associate some datasets into the TAGGED and CALIBRATION collections.
238 flats1 = list(registry1.queryDatasets("flat", collections=...))
239 registry1.associate("tag1", flats1)
240 t1 = astropy.time.Time("2020-01-01T01:00:00", format="isot", scale="tai")
241 t2 = astropy.time.Time("2020-01-01T02:00:00", format="isot", scale="tai")
242 t3 = astropy.time.Time("2020-01-01T03:00:00", format="isot", scale="tai")
243 bias1a = registry1.findDataset("bias", instrument="Cam1", detector=1, collections="imported_g")
244 bias2a = registry1.findDataset("bias", instrument="Cam1", detector=2, collections="imported_g")
245 bias3a = registry1.findDataset("bias", instrument="Cam1", detector=3, collections="imported_g")
246 bias2b = registry1.findDataset("bias", instrument="Cam1", detector=2, collections="imported_r")
247 bias3b = registry1.findDataset("bias", instrument="Cam1", detector=3, collections="imported_r")
248 registry1.certify("calibration1", [bias2a, bias3a], Timespan(t1, t2))
249 registry1.certify("calibration1", [bias2b], Timespan(t2, None))
250 registry1.certify("calibration1", [bias3b], Timespan(t2, t3))
251 registry1.certify("calibration1", [bias1a], Timespan.makeEmpty())
253 with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml") as file:
254 # Export all collections, and some datasets.
255 with butler1.export(filename=file.name) as exporter:
256 # Sort results to put chain1 before chain2, which is
257 # intentionally not topological order.
258 for collection in sorted(registry1.queryCollections()):
259 exporter.saveCollection(collection)
260 exporter.saveDatasets(flats1)
261 exporter.saveDatasets([bias1a, bias2a, bias2b, bias3a, bias3b])
262 # Import them into a new registry.
263 butler2 = self.makeButler(writeable=True)
264 butler2.import_(filename=file.name)
265 registry2 = butler2.registry
266 # Check that it all round-tripped, starting with the collections
267 # themselves.
268 self.assertIs(registry2.getCollectionType("run1"), CollectionType.RUN)
269 self.assertIs(registry2.getCollectionType("tag1"), CollectionType.TAGGED)
270 self.assertIs(registry2.getCollectionType("calibration1"), CollectionType.CALIBRATION)
271 self.assertIs(registry2.getCollectionType("chain1"), CollectionType.CHAINED)
272 self.assertIs(registry2.getCollectionType("chain2"), CollectionType.CHAINED)
273 self.assertEqual(
274 list(registry2.getCollectionChain("chain1")),
275 ["tag1", "run1", "chain2"],
276 )
277 self.assertEqual(
278 list(registry2.getCollectionChain("chain2")),
279 ["calibration1", "run1"],
280 )
281 # Check that tag collection contents are the same.
282 self.maxDiff = None
283 self.assertCountEqual(
284 [self.comparableRef(ref) for ref in registry1.queryDatasets(..., collections="tag1")],
285 [self.comparableRef(ref) for ref in registry2.queryDatasets(..., collections="tag1")],
286 )
287 # Check that calibration collection contents are the same.
288 self.assertCountEqual(
289 [
290 (self.comparableRef(assoc.ref), assoc.timespan)
291 for assoc in registry1.queryDatasetAssociations("bias", collections="calibration1")
292 ],
293 [
294 (self.comparableRef(assoc.ref), assoc.timespan)
295 for assoc in registry2.queryDatasetAssociations("bias", collections="calibration1")
296 ],
297 )
299 def testButlerGet(self):
300 """Test that butler.get can work with different variants."""
302 # Import data to play with.
303 butler = self.makeButler(writeable=True)
304 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
305 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile))
307 # Find the DatasetRef for a flat
308 coll = "imported_g"
309 flat2g = butler.registry.findDataset(
310 "flat", instrument="Cam1", detector=2, physical_filter="Cam1-G", collections=coll
311 )
313 # Create a numpy integer to check that works fine
314 detector_np = np.int64(2) if np else 2
315 print(type(detector_np))
317 # Try to get it using different variations of dataId + keyword
318 # arguments
319 # Note that instrument.class_name does not work
320 variants = (
321 (None, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}),
322 (None, {"instrument": "Cam1", "detector": detector_np, "physical_filter": "Cam1-G"}),
323 ({"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}, {}),
324 ({"instrument": "Cam1", "detector": detector_np, "physical_filter": "Cam1-G"}, {}),
325 ({"instrument": "Cam1", "detector": 2}, {"physical_filter": "Cam1-G"}),
326 ({"detector.full_name": "Ab"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}),
327 ({"full_name": "Ab"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}),
328 (None, {"full_name": "Ab", "instrument": "Cam1", "physical_filter": "Cam1-G"}),
329 (None, {"detector": "Ab", "instrument": "Cam1", "physical_filter": "Cam1-G"}),
330 ({"name_in_raft": "b", "raft": "A"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}),
331 ({"name_in_raft": "b"}, {"raft": "A", "instrument": "Cam1", "physical_filter": "Cam1-G"}),
332 (None, {"name_in_raft": "b", "raft": "A", "instrument": "Cam1", "physical_filter": "Cam1-G"}),
333 (
334 {"detector.name_in_raft": "b", "detector.raft": "A"},
335 {"instrument": "Cam1", "physical_filter": "Cam1-G"},
336 ),
337 (
338 {
339 "detector.name_in_raft": "b",
340 "detector.raft": "A",
341 "instrument": "Cam1",
342 "physical_filter": "Cam1-G",
343 },
344 {},
345 ),
346 # Duplicate (but valid) information.
347 (None, {"instrument": "Cam1", "detector": 2, "raft": "A", "physical_filter": "Cam1-G"}),
348 ({"detector": 2}, {"instrument": "Cam1", "raft": "A", "physical_filter": "Cam1-G"}),
349 ({"raft": "A"}, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}),
350 ({"raft": "A"}, {"instrument": "Cam1", "detector": "Ab", "physical_filter": "Cam1-G"}),
351 )
353 for dataId, kwds in variants:
354 try:
355 flat_id, _ = butler.get("flat", dataId=dataId, collections=coll, **kwds)
356 except Exception as e:
357 raise type(e)(f"{str(e)}: dataId={dataId}, kwds={kwds}") from e
358 self.assertEqual(flat_id, flat2g.id, msg=f"DataId: {dataId}, kwds: {kwds}")
360 # Check that bad combinations raise.
361 variants = (
362 # Inconsistent detector information.
363 (None, {"instrument": "Cam1", "detector": 2, "raft": "B", "physical_filter": "Cam1-G"}),
364 ({"detector": 2}, {"instrument": "Cam1", "raft": "B", "physical_filter": "Cam1-G"}),
365 ({"detector": 12}, {"instrument": "Cam1", "raft": "B", "physical_filter": "Cam1-G"}),
366 ({"raft": "B"}, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}),
367 ({"raft": "B"}, {"instrument": "Cam1", "detector": "Ab", "physical_filter": "Cam1-G"}),
368 # Under-specified.
369 ({"raft": "B"}, {"instrument": "Cam1", "physical_filter": "Cam1-G"}),
370 # Spurious kwargs.
371 (None, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G", "x": "y"}),
372 ({"x": "y"}, {"instrument": "Cam1", "detector": 2, "physical_filter": "Cam1-G"}),
373 )
374 for dataId, kwds in variants:
375 with self.assertRaises((ValueError, LookupError)):
376 butler.get("flat", dataId=dataId, collections=coll, **kwds)
378 def testGetCalibration(self):
379 """Test that `Butler.get` can be used to fetch from
380 `~CollectionType.CALIBRATION` collections if the data ID includes
381 extra dimensions with temporal information.
382 """
383 # Import data to play with.
384 butler = self.makeButler(writeable=True)
385 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
386 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile))
387 # Certify some biases into a CALIBRATION collection.
388 registry = butler.registry
389 registry.registerCollection("calibs", CollectionType.CALIBRATION)
390 t1 = astropy.time.Time("2020-01-01T01:00:00", format="isot", scale="tai")
391 t2 = astropy.time.Time("2020-01-01T02:00:00", format="isot", scale="tai")
392 t3 = astropy.time.Time("2020-01-01T03:00:00", format="isot", scale="tai")
393 bias2a = registry.findDataset("bias", instrument="Cam1", detector=2, collections="imported_g")
394 bias3a = registry.findDataset("bias", instrument="Cam1", detector=3, collections="imported_g")
395 bias2b = registry.findDataset("bias", instrument="Cam1", detector=2, collections="imported_r")
396 bias3b = registry.findDataset("bias", instrument="Cam1", detector=3, collections="imported_r")
397 registry.certify("calibs", [bias2a, bias3a], Timespan(t1, t2))
398 registry.certify("calibs", [bias2b], Timespan(t2, None))
399 registry.certify("calibs", [bias3b], Timespan(t2, t3))
400 # Insert some exposure dimension data.
401 registry.insertDimensionData(
402 "exposure",
403 {
404 "instrument": "Cam1",
405 "id": 3,
406 "obs_id": "three",
407 "timespan": Timespan(t1, t2),
408 "physical_filter": "Cam1-G",
409 "day_obs": 20201114,
410 "seq_num": 55,
411 },
412 {
413 "instrument": "Cam1",
414 "id": 4,
415 "obs_id": "four",
416 "timespan": Timespan(t2, t3),
417 "physical_filter": "Cam1-G",
418 "day_obs": 20211114,
419 "seq_num": 42,
420 },
421 )
422 # Get some biases from raw-like data IDs.
423 bias2a_id, _ = butler.get(
424 "bias", {"instrument": "Cam1", "exposure": 3, "detector": 2}, collections="calibs"
425 )
426 self.assertEqual(bias2a_id, bias2a.id)
427 bias3b_id, _ = butler.get(
428 "bias", {"instrument": "Cam1", "exposure": 4, "detector": 3}, collections="calibs"
429 )
430 self.assertEqual(bias3b_id, bias3b.id)
432 # Get using the kwarg form
433 bias3b_id, _ = butler.get("bias", instrument="Cam1", exposure=4, detector=3, collections="calibs")
434 self.assertEqual(bias3b_id, bias3b.id)
436 # Do it again but using the record information
437 bias2a_id, _ = butler.get(
438 "bias",
439 {"instrument": "Cam1", "exposure.obs_id": "three", "detector.full_name": "Ab"},
440 collections="calibs",
441 )
442 self.assertEqual(bias2a_id, bias2a.id)
443 bias3b_id, _ = butler.get(
444 "bias",
445 {"exposure.obs_id": "four", "detector.full_name": "Ba"},
446 collections="calibs",
447 instrument="Cam1",
448 )
449 self.assertEqual(bias3b_id, bias3b.id)
451 # And again but this time using the alternate value rather than
452 # the primary.
453 bias3b_id, _ = butler.get(
454 "bias", {"exposure": "four", "detector": "Ba"}, collections="calibs", instrument="Cam1"
455 )
456 self.assertEqual(bias3b_id, bias3b.id)
458 # And again but this time using the alternate value rather than
459 # the primary and do it in the keyword arguments.
460 bias3b_id, _ = butler.get(
461 "bias", exposure="four", detector="Ba", collections="calibs", instrument="Cam1"
462 )
463 self.assertEqual(bias3b_id, bias3b.id)
465 # Now with implied record columns
466 bias3b_id, _ = butler.get(
467 "bias",
468 day_obs=20211114,
469 seq_num=42,
470 raft="B",
471 name_in_raft="a",
472 collections="calibs",
473 instrument="Cam1",
474 )
475 self.assertEqual(bias3b_id, bias3b.id)
477 # Allow a fully-specified dataId and unnecessary extra information
478 # that comes from the record.
479 bias3b_id, _ = butler.get(
480 "bias",
481 dataId=dict(
482 exposure=4,
483 day_obs=20211114,
484 seq_num=42,
485 detector=3,
486 instrument="Cam1",
487 ),
488 collections="calibs",
489 )
490 self.assertEqual(bias3b_id, bias3b.id)
492 # Extra but inconsistent record values are a problem.
493 with self.assertRaises(ValueError):
494 bias3b_id, _ = butler.get(
495 "bias",
496 exposure=3,
497 day_obs=20211114,
498 seq_num=42,
499 detector=3,
500 collections="calibs",
501 instrument="Cam1",
502 )
504 # Ensure that spurious kwargs cause an exception.
505 with self.assertRaises(ValueError):
506 butler.get(
507 "bias",
508 {"exposure.obs_id": "four", "immediate": True, "detector.full_name": "Ba"},
509 collections="calibs",
510 instrument="Cam1",
511 )
513 with self.assertRaises(ValueError):
514 butler.get(
515 "bias",
516 day_obs=20211114,
517 seq_num=42,
518 raft="B",
519 name_in_raft="a",
520 collections="calibs",
521 instrument="Cam1",
522 immediate=True,
523 )
525 def testRegistryDefaults(self):
526 """Test that we can default the collections and some data ID keys when
527 constructing a butler.
529 Many tests that use default run already exist in ``test_butler.py``, so
530 that isn't tested here. And while most of this functionality is
531 implemented in `Registry`, we test it here instead of
532 ``daf/butler/tests/registry.py`` because it shouldn't depend on the
533 database backend at all.
534 """
535 butler = self.makeButler(writeable=True)
536 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
537 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile))
538 # Need to actually set defaults later, not at construction, because
539 # we need to import the instrument before we can use it as a default.
540 # Don't set a default instrument value for data IDs, because 'Cam1'
541 # should be inferred by virtue of that being the only value in the
542 # input collections.
543 butler.registry.defaults = RegistryDefaults(collections=["imported_g"])
544 # Use findDataset without collections or instrument.
545 ref = butler.registry.findDataset("flat", detector=2, physical_filter="Cam1-G")
546 # Do the same with Butler.get; this should ultimately invoke a lot of
547 # the same code, so it's a bit circular, but mostly we're checking that
548 # it works at all.
549 dataset_id, _ = butler.get("flat", detector=2, physical_filter="Cam1-G")
550 self.assertEqual(ref.id, dataset_id)
551 # Query for datasets. Test defaulting the data ID in both kwargs and
552 # in the WHERE expression.
553 queried_refs_1 = set(butler.registry.queryDatasets("flat", detector=2, physical_filter="Cam1-G"))
554 self.assertEqual({ref}, queried_refs_1)
555 queried_refs_2 = set(
556 butler.registry.queryDatasets("flat", where="detector=2 AND physical_filter='Cam1-G'")
557 )
558 self.assertEqual({ref}, queried_refs_2)
559 # Query for data IDs with a dataset constraint.
560 queried_data_ids = set(
561 butler.registry.queryDataIds(
562 {"instrument", "detector", "physical_filter"},
563 datasets={"flat"},
564 detector=2,
565 physical_filter="Cam1-G",
566 )
567 )
568 self.assertEqual({ref.dataId}, queried_data_ids)
569 # Add another instrument to the repo, and a dataset that uses it to
570 # the `imported_g` collection.
571 butler.registry.insertDimensionData("instrument", {"name": "Cam2"})
572 camera = DatasetType(
573 "camera",
574 dimensions=butler.registry.dimensions["instrument"].graph,
575 storageClass="Camera",
576 )
577 butler.registry.registerDatasetType(camera)
578 butler.registry.insertDatasets(camera, [{"instrument": "Cam2"}], run="imported_g")
579 # Initialize a new butler with `imported_g` as its default run.
580 # This should not have a default instrument, because there are two.
581 # Pass run instead of collections; this should set both.
582 butler2 = Butler(butler=butler, run="imported_g")
583 self.assertEqual(list(butler2.registry.defaults.collections), ["imported_g"])
584 self.assertEqual(butler2.registry.defaults.run, "imported_g")
585 self.assertFalse(butler2.registry.defaults.dataId)
586 # Initialize a new butler with an instrument default explicitly given.
587 # Set collections instead of run, which should then be None.
588 butler3 = Butler(butler=butler, collections=["imported_g"], instrument="Cam2")
589 self.assertEqual(list(butler3.registry.defaults.collections), ["imported_g"])
590 self.assertIsNone(butler3.registry.defaults.run, None)
591 self.assertEqual(butler3.registry.defaults.dataId.byName(), {"instrument": "Cam2"})
593 def testJson(self):
594 """Test JSON serialization mediated by registry."""
595 butler = self.makeButler(writeable=True)
596 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
597 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", self.datasetsImportFile))
598 # Need to actually set defaults later, not at construction, because
599 # we need to import the instrument before we can use it as a default.
600 # Don't set a default instrument value for data IDs, because 'Cam1'
601 # should be inferred by virtue of that being the only value in the
602 # input collections.
603 butler.registry.defaults = RegistryDefaults(collections=["imported_g"])
604 # Use findDataset without collections or instrument.
605 ref = butler.registry.findDataset("flat", detector=2, physical_filter="Cam1-G")
607 # Transform the ref and dataset type to and from JSON
608 # and check that it can be reconstructed properly
610 # Do it with the ref and a component ref in minimal and standard form
611 compRef = ref.makeComponentRef("wcs")
613 for test_item in (ref, ref.datasetType, compRef, compRef.datasetType):
614 for minimal in (False, True):
615 json_str = test_item.to_json(minimal=minimal)
616 from_json = type(test_item).from_json(json_str, registry=butler.registry)
617 self.assertEqual(from_json, test_item, msg=f"From JSON '{json_str}' using registry")
619 # for minimal=False case also do a test without registry
620 if not minimal:
621 from_json = type(test_item).from_json(json_str, universe=butler.registry.dimensions)
622 self.assertEqual(from_json, test_item, msg=f"From JSON '{json_str}' using universe")
624 def testJsonDimensionRecordsAndHtmlRepresentation(self):
625 # Dimension Records
626 butler = self.makeButler(writeable=True)
627 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "hsc-rc2-subset.yaml"))
629 for dimension in ("detector", "visit"):
630 records = butler.registry.queryDimensionRecords(dimension, instrument="HSC")
631 for r in records:
632 for minimal in (True, False):
633 json_str = r.to_json(minimal=minimal)
634 r_json = type(r).from_json(json_str, registry=butler.registry)
635 self.assertEqual(r_json, r)
636 # check with direct method
637 simple = r.to_simple()
638 fromDirect = type(simple).direct(**json.loads(json_str))
639 self.assertEqual(simple, fromDirect)
640 # Also check equality of each of the components as dicts
641 self.assertEqual(r_json.toDict(), r.toDict())
643 # check the html representation of records
644 r_html = r._repr_html_()
645 self.assertTrue(isinstance(r_html, str))
646 self.assertIn(dimension, r_html)
648 def testWildcardQueries(self):
649 """Test that different collection type queries work."""
651 # Import data to play with.
652 butler = self.makeButler(writeable=True)
653 butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml"))
655 # Create some collections
656 created = {"collection", "u/user/test", "coll3"}
657 for collection in created:
658 butler.registry.registerCollection(collection, type=CollectionType.RUN)
660 collections = butler.registry.queryCollections()
661 self.assertEqual(set(collections), created)
663 expressions = (
664 ("collection", {"collection"}),
665 (..., created),
666 ("*", created),
667 (("collection", "*"), created),
668 ("u/*", {"u/user/test"}),
669 (re.compile("u.*"), {"u/user/test"}),
670 (re.compile(".*oll.*"), {"collection", "coll3"}),
671 ("*oll*", {"collection", "coll3"}),
672 ((re.compile(r".*\d$"), "u/user/test"), {"coll3", "u/user/test"}),
673 ("*[0-9]", {"coll3"}),
674 )
675 for expression, expected in expressions:
676 result = butler.registry.queryCollections(expression)
677 self.assertEqual(set(result), expected)
680class SimpleButlerMixedUUIDTestCase(SimpleButlerTestCase):
681 """Same as SimpleButlerTestCase but uses UUID-based datasets manager and
682 loads datasets from YAML file with integer IDs.
683 """
685 datasetsManager = (
686 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
687 )
688 datasetsImportFile = "datasets.yaml"
689 datasetsIdType = uuid.UUID
692if __name__ == "__main__": 692 ↛ 693line 692 didn't jump to line 693, because the condition on line 692 was never true
693 unittest.main()