Coverage for tests/test_dataset_handle.py: 20%
99 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-30 12:09 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-30 12:09 +0000
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import unittest
30from lsst.daf.butler import DataCoordinate, DimensionUniverse, StorageClassConfig, StorageClassFactory
31from lsst.daf.butler.tests import MetricsExample
32from lsst.pipe.base import InMemoryDatasetHandle
34storageClasses = """
35Integer:
36 pytype: int
37StructuredDataTestDict:
38 pytype: dict
39StructuredDataTestList:
40 pytype: list
41 delegate: lsst.daf.butler.tests.ListDelegate
42 parameters:
43 - slice
44 derivedComponents:
45 counter: Integer
46StructuredDataTest:
47 # Data from a simple Python class
48 pytype: lsst.daf.butler.tests.MetricsExample
49 delegate: lsst.daf.butler.tests.MetricsDelegate
50 # Use YAML formatter by default
51 components:
52 # Components are those supported by get.
53 summary: StructuredDataTestDict
54 output: StructuredDataTestDict
55 data: StructuredDataTestList
56 parameters:
57 - slice
58 derivedComponents:
59 counter: Integer
60MetricsConversion:
61 # Special storage class to test conversions.
62 pytype: lsst.daf.butler.tests.MetricsExampleModel
63 delegate: lsst.daf.butler.tests.MetricsDelegate
64 converters:
65 lsst.daf.butler.tests.MetricsExample: lsst.daf.butler.tests.MetricsExampleModel.from_metrics
66StructuredDataTestListSet:
67 pytype: set
68 converters:
69 list: builtins.set
70"""
73class SpecialThing:
74 """Class known not to have associated StorageClass"""
77class NotCopyable(MetricsExample):
78 """Subclass of metrics that can't be copied."""
80 def __deepcopy__(self, memo=None):
81 raise RuntimeError("Can not be copied")
84class TestDatasetHandle(unittest.TestCase):
85 """Test in-memory dataset handle."""
87 @classmethod
88 def setUpClass(cls):
89 cls.storage_class_config = StorageClassConfig.fromYaml(storageClasses)
90 cls.factory = StorageClassFactory()
92 def setUp(self):
93 self.factory.reset()
94 self.factory.addFromConfig(self.storage_class_config)
96 def test_dataset_handle_basic(self):
97 inMemoryDataset = 42
98 hdl = InMemoryDatasetHandle(inMemoryDataset)
100 self.assertEqual(hdl.get(), inMemoryDataset)
102 def test_dataset_handle_copy(self):
103 inMemoryDataset = [1, 2]
104 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=False)
106 retrieved = hdl.get()
107 self.assertEqual(retrieved, inMemoryDataset)
108 retrieved.append(3)
109 self.assertEqual(retrieved, inMemoryDataset)
111 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True)
112 retrieved = hdl.get()
113 self.assertEqual(retrieved, inMemoryDataset)
114 retrieved.append(3)
115 self.assertNotEqual(retrieved, inMemoryDataset)
117 inMemoryDataset = NotCopyable(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4])
118 hdl = InMemoryDatasetHandle(inMemoryDataset)
119 self.assertIs(hdl.get(), inMemoryDataset)
121 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True, storageClass="MetricsConversion")
122 with self.assertRaises(NotImplementedError):
123 hdl.get()
125 def test_dataset_handle_unknown(self):
126 inMemoryDataset = SpecialThing()
127 hdl = InMemoryDatasetHandle(inMemoryDataset)
129 self.assertEqual(hdl.get(), inMemoryDataset)
131 with self.assertRaises(KeyError):
132 # Will not be able to find a matching StorageClass.
133 hdl.get(parameters={"key": "value"})
135 def test_dataset_handle_none(self):
136 hdl = InMemoryDatasetHandle(None)
137 self.assertIsNone(hdl.get())
138 self.assertIsNone(hdl.get(component="comp"))
139 self.assertIsNone(hdl.get(parameters={"something": 42}))
141 def test_dataset_handle_dataid(self):
142 hdl = InMemoryDatasetHandle(42)
143 self.assertEqual(dict(hdl.dataId.required), {})
145 dataId = DataCoordinate.make_empty(DimensionUniverse())
146 hdl = InMemoryDatasetHandle(42, dataId=dataId)
147 self.assertIs(hdl.dataId, dataId)
149 dataId = {"tract": 5, "patch": 2, "instrument": "TestCam"}
150 hdl = InMemoryDatasetHandle(42, **dataId)
151 self.assertEqual(hdl.dataId, dataId)
153 hdl = InMemoryDatasetHandle(42, dataId=dataId, tract=6)
154 self.assertEqual(hdl.dataId["tract"], 6)
156 dataId = DataCoordinate.standardize({}, universe=DimensionUniverse(), instrument="DummyCam")
157 hdl = InMemoryDatasetHandle(42, dataId=dataId, physical_filter="g")
158 self.assertIsInstance(hdl.dataId, DataCoordinate)
159 self.assertEqual(hdl.dataId["physical_filter"], "g")
161 def test_dataset_handle_metric(self):
162 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4])
164 # First with explicit storage class.
165 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest")
166 retrieved = hdl.get()
167 self.assertEqual(retrieved, metric)
169 data = hdl.get(component="data")
170 self.assertEqual(data, metric.data)
172 # Now with implicit storage class.
173 hdl = InMemoryDatasetHandle(metric)
174 data = hdl.get(component="data")
175 self.assertEqual(data, metric.data)
177 # Parameters.
178 data = hdl.get(parameters={"slice": slice(2)})
179 self.assertEqual(data.summary, metric.summary)
180 self.assertEqual(data.data, [1, 2])
182 data = hdl.get(parameters={"slice": slice(2)}, component="data")
183 self.assertEqual(data, [1, 2])
185 # Use parameters in constructor and also override.
186 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest", parameters={"slice": slice(3)})
187 self.assertEqual(hdl.get(component="data"), [1, 2, 3])
188 self.assertEqual(hdl.get(component="counter"), 3)
189 self.assertEqual(hdl.get(component="data", parameters={"slice": slice(1, 3)}), [2, 3])
190 self.assertEqual(hdl.get(component="counter", parameters={"slice": slice(1, 3)}), 2)
192 # Ensure the original has not been modified.
193 self.assertEqual(len(metric.data), 4)
195 def test_handle_conversion(self):
196 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4])
198 # Test conversion with no components or parameters.
199 hdl = InMemoryDatasetHandle(metric)
200 retrieved = hdl.get() # Reset the reference.
201 converted = hdl.get(storageClass="MetricsConversion")
202 self.assertIsNot(type(converted), type(retrieved))
203 self.assertEqual(retrieved, converted)
205 # Again with a full storage class.
206 sc = self.factory.getStorageClass("MetricsConversion")
207 converted2 = hdl.get(storageClass=sc)
208 self.assertEqual(converted2, converted)
210 # Conversion of component.
211 data = hdl.get(component="data", storageClass="StructuredDataTestListSet")
212 self.assertIsInstance(data, set)
213 self.assertEqual(data, set(converted.data))
216if __name__ == "__main__":
217 unittest.main()