Coverage for tests/test_dataset_handle.py: 19%
99 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-09 09:17 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-09 09:17 +0000
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import unittest
24from lsst.daf.butler import DataCoordinate, DimensionUniverse, StorageClassConfig, StorageClassFactory
25from lsst.daf.butler.tests import MetricsExample
26from lsst.pipe.base import InMemoryDatasetHandle
28storageClasses = """
29Integer:
30 pytype: int
31StructuredDataTestDict:
32 pytype: dict
33StructuredDataTestList:
34 pytype: list
35 delegate: lsst.daf.butler.tests.ListDelegate
36 parameters:
37 - slice
38 derivedComponents:
39 counter: Integer
40StructuredDataTest:
41 # Data from a simple Python class
42 pytype: lsst.daf.butler.tests.MetricsExample
43 delegate: lsst.daf.butler.tests.MetricsDelegate
44 # Use YAML formatter by default
45 components:
46 # Components are those supported by get.
47 summary: StructuredDataTestDict
48 output: StructuredDataTestDict
49 data: StructuredDataTestList
50 parameters:
51 - slice
52 derivedComponents:
53 counter: Integer
54MetricsConversion:
55 # Special storage class to test conversions.
56 pytype: lsst.daf.butler.tests.MetricsExampleModel
57 delegate: lsst.daf.butler.tests.MetricsDelegate
58 converters:
59 lsst.daf.butler.tests.MetricsExample: lsst.daf.butler.tests.MetricsExampleModel.from_metrics
60StructuredDataTestListSet:
61 pytype: set
62 converters:
63 list: builtins.set
64"""
67class SpecialThing:
68 """Class known not to have associated StorageClass"""
71class NotCopyable(MetricsExample):
72 """Subclass of metrics that can't be copied."""
74 def __deepcopy__(self, memo=None):
75 raise RuntimeError("Can not be copied")
78class TestDatasetHandle(unittest.TestCase):
79 @classmethod
80 def setUpClass(cls):
81 config = StorageClassConfig.fromYaml(storageClasses)
82 cls.factory = StorageClassFactory()
83 cls.factory.addFromConfig(config)
85 def test_dataset_handle_basic(self):
86 inMemoryDataset = 42
87 hdl = InMemoryDatasetHandle(inMemoryDataset)
89 self.assertEqual(hdl.get(), inMemoryDataset)
91 def test_dataset_handle_copy(self):
92 inMemoryDataset = [1, 2]
93 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=False)
95 retrieved = hdl.get()
96 self.assertEqual(retrieved, inMemoryDataset)
97 retrieved.append(3)
98 self.assertEqual(retrieved, inMemoryDataset)
100 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True)
101 retrieved = hdl.get()
102 self.assertEqual(retrieved, inMemoryDataset)
103 retrieved.append(3)
104 self.assertNotEqual(retrieved, inMemoryDataset)
106 inMemoryDataset = NotCopyable(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4])
107 hdl = InMemoryDatasetHandle(inMemoryDataset)
108 self.assertIs(hdl.get(), inMemoryDataset)
110 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True, storageClass="MetricsConversion")
111 with self.assertRaises(NotImplementedError):
112 hdl.get()
114 def test_dataset_handle_unknown(self):
115 inMemoryDataset = SpecialThing()
116 hdl = InMemoryDatasetHandle(inMemoryDataset)
118 self.assertEqual(hdl.get(), inMemoryDataset)
120 with self.assertRaises(KeyError):
121 # Will not be able to find a matching StorageClass.
122 hdl.get(parameters={"key": "value"})
124 def test_dataset_handle_none(self):
125 hdl = InMemoryDatasetHandle(None)
126 self.assertIsNone(hdl.get())
127 self.assertIsNone(hdl.get(component="comp"))
128 self.assertIsNone(hdl.get(parameters={"something": 42}))
130 def test_dataset_handle_dataid(self):
131 hdl = InMemoryDatasetHandle(42)
132 self.assertEqual(dict(hdl.dataId), {})
134 dataId = DataCoordinate.makeEmpty(DimensionUniverse())
135 hdl = InMemoryDatasetHandle(42, dataId=dataId)
136 self.assertIs(hdl.dataId, dataId)
138 dataId = {"tract": 5, "patch": 2, "instrument": "TestCam"}
139 hdl = InMemoryDatasetHandle(42, **dataId)
140 self.assertEqual(hdl.dataId, dataId)
142 hdl = InMemoryDatasetHandle(42, dataId=dataId, tract=6)
143 self.assertEqual(hdl.dataId["tract"], 6)
145 dataId = DataCoordinate.standardize({}, universe=DimensionUniverse(), instrument="DummyCam")
146 hdl = InMemoryDatasetHandle(42, dataId=dataId, physical_filter="g")
147 self.assertIsInstance(hdl.dataId, DataCoordinate)
148 self.assertEqual(hdl.dataId["physical_filter"], "g")
150 def test_dataset_handle_metric(self):
151 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4])
153 # First with explicit storage class.
154 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest")
155 retrieved = hdl.get()
156 self.assertEqual(retrieved, metric)
158 data = hdl.get(component="data")
159 self.assertEqual(data, metric.data)
161 # Now with implicit storage class.
162 hdl = InMemoryDatasetHandle(metric)
163 data = hdl.get(component="data")
164 self.assertEqual(data, metric.data)
166 # Parameters.
167 data = hdl.get(parameters={"slice": slice(2)})
168 self.assertEqual(data.summary, metric.summary)
169 self.assertEqual(data.data, [1, 2])
171 data = hdl.get(parameters={"slice": slice(2)}, component="data")
172 self.assertEqual(data, [1, 2])
174 # Use parameters in constructor and also override.
175 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest", parameters={"slice": slice(3)})
176 self.assertEqual(hdl.get(component="data"), [1, 2, 3])
177 self.assertEqual(hdl.get(component="counter"), 3)
178 self.assertEqual(hdl.get(component="data", parameters={"slice": slice(1, 3)}), [2, 3])
179 self.assertEqual(hdl.get(component="counter", parameters={"slice": slice(1, 3)}), 2)
181 # Ensure the original has not been modified.
182 self.assertEqual(len(metric.data), 4)
184 def test_handle_conversion(self):
185 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4])
187 # Test conversion with no components or parameters.
188 hdl = InMemoryDatasetHandle(metric)
189 retrieved = hdl.get() # Reset the reference.
190 converted = hdl.get(storageClass="MetricsConversion")
191 self.assertIsNot(type(converted), type(retrieved))
192 self.assertEqual(retrieved, converted)
194 # Again with a full storage class.
195 sc = self.factory.getStorageClass("MetricsConversion")
196 converted2 = hdl.get(storageClass=sc)
197 self.assertEqual(converted2, converted)
199 # Conversion of component.
200 data = hdl.get(component="data", storageClass="StructuredDataTestListSet")
201 self.assertIsInstance(data, set)
202 self.assertEqual(data, set(converted.data))
205if __name__ == "__main__": 205 ↛ 206line 205 didn't jump to line 206, because the condition on line 205 was never true
206 unittest.main()