Coverage for tests/test_dataset

1# This file is part of pipe_base.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This software is dual licensed under the GNU General Public License and also

10# under a 3-clause BSD license. Recipients may choose which of these licenses

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,

12# respectively. If you choose the GPL option then the following text applies

13# (but note that there is still no warranty even if you opt for BSD instead):

14#

15# This program is free software: you can redistribute it and/or modify

16# it under the terms of the GNU General Public License as published by

17# the Free Software Foundation, either version 3 of the License, or

18# (at your option) any later version.

19#

20# This program is distributed in the hope that it will be useful,

21# but WITHOUT ANY WARRANTY; without even the implied warranty of

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

23# GNU General Public License for more details.

24#

25# You should have received a copy of the GNU General Public License

26# along with this program. If not, see <http://www.gnu.org/licenses/>.

28import unittest

30from lsst.daf.butler import DataCoordinate, DimensionUniverse, StorageClassConfig, StorageClassFactory

31from lsst.daf.butler.tests import MetricsExample

32from lsst.pipe.base import InMemoryDatasetHandle

34storageClasses = """

35Integer:

36 pytype: int

37StructuredDataTestDict:

38 pytype: dict

39StructuredDataTestList:

40 pytype: list

41 delegate: lsst.daf.butler.tests.ListDelegate

42 parameters:

43 - slice

44 derivedComponents:

45 counter: Integer

46StructuredDataTest:

47 # Data from a simple Python class

48 pytype: lsst.daf.butler.tests.MetricsExample

49 delegate: lsst.daf.butler.tests.MetricsDelegate

50 # Use YAML formatter by default

51 components:

52 # Components are those supported by get.

53 summary: StructuredDataTestDict

54 output: StructuredDataTestDict

55 data: StructuredDataTestList

56 parameters:

57 - slice

58 derivedComponents:

59 counter: Integer

60MetricsConversion:

61 # Special storage class to test conversions.

62 pytype: lsst.daf.butler.tests.MetricsExampleModel

63 delegate: lsst.daf.butler.tests.MetricsDelegate

64 converters:

65 lsst.daf.butler.tests.MetricsExample: lsst.daf.butler.tests.MetricsExampleModel.from_metrics

66StructuredDataTestListSet:

67 pytype: set

68 converters:

69 list: builtins.set

70"""

73class SpecialThing:

74 """Class known not to have associated StorageClass."""

77class NotCopyable(MetricsExample):

78 """Subclass of metrics that can't be copied."""

80 def __deepcopy__(self, memo=None):

81 raise RuntimeError("Can not be copied")

84class TestDatasetHandle(unittest.TestCase):

85 """Test in-memory dataset handle."""

87 @classmethod

88 def setUpClass(cls):

89 cls.storage_class_config = StorageClassConfig.fromYaml(storageClasses)

90 cls.factory = StorageClassFactory()

92 def setUp(self):

93 self.factory.reset()

94 self.factory.addFromConfig(self.storage_class_config)

96 def test_dataset_handle_basic(self):

97 inMemoryDataset = 42

98 hdl = InMemoryDatasetHandle(inMemoryDataset)

100 self.assertEqual(hdl.get(), inMemoryDataset)

101

102 def test_dataset_handle_copy(self):

103 inMemoryDataset = [1, 2]

104 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=False)

105

106 retrieved = hdl.get()

107 self.assertEqual(retrieved, inMemoryDataset)

108 retrieved.append(3)

109 self.assertEqual(retrieved, inMemoryDataset)

110

111 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True)

112 retrieved = hdl.get()

113 self.assertEqual(retrieved, inMemoryDataset)

114 retrieved.append(3)

115 self.assertNotEqual(retrieved, inMemoryDataset)

116

117 inMemoryDataset = NotCopyable(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4])

118 hdl = InMemoryDatasetHandle(inMemoryDataset)

119 self.assertIs(hdl.get(), inMemoryDataset)

120

121 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True, storageClass="MetricsConversion")

122 with self.assertRaises(NotImplementedError):

123 hdl.get()

124

125 def test_dataset_handle_unknown(self):

126 inMemoryDataset = SpecialThing()

127 hdl = InMemoryDatasetHandle(inMemoryDataset)

128

129 self.assertEqual(hdl.get(), inMemoryDataset)

130

131 with self.assertRaises(KeyError):

132 # Will not be able to find a matching StorageClass.

133 hdl.get(parameters={"key": "value"})

134

135 def test_dataset_handle_none(self):

136 hdl = InMemoryDatasetHandle(None)

137 self.assertIsNone(hdl.get())

138 self.assertIsNone(hdl.get(component="comp"))

139 self.assertIsNone(hdl.get(parameters={"something": 42}))

140

141 def test_dataset_handle_dataid(self):

142 hdl = InMemoryDatasetHandle(42)

143 self.assertEqual(dict(hdl.dataId.required), {})

144

145 dataId = DataCoordinate.make_empty(DimensionUniverse())

146 hdl = InMemoryDatasetHandle(42, dataId=dataId)

147 self.assertIs(hdl.dataId, dataId)

148

149 dataId = {"tract": 5, "patch": 2, "instrument": "TestCam"}

150 hdl = InMemoryDatasetHandle(42, **dataId)

151 self.assertEqual(hdl.dataId, dataId)

152

153 hdl = InMemoryDatasetHandle(42, dataId=dataId, tract=6)

154 self.assertEqual(hdl.dataId["tract"], 6)

155

156 dataId = DataCoordinate.standardize({}, universe=DimensionUniverse(), instrument="DummyCam")

157 hdl = InMemoryDatasetHandle(42, dataId=dataId, physical_filter="g")

158 self.assertIsInstance(hdl.dataId, DataCoordinate)

159 self.assertEqual(hdl.dataId["physical_filter"], "g")

160

161 def test_dataset_handle_metric(self):

162 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4])

163

164 # First with explicit storage class.

165 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest")

166 retrieved = hdl.get()

167 self.assertEqual(retrieved, metric)

168

169 data = hdl.get(component="data")

170 self.assertEqual(data, metric.data)

171

172 # Now with implicit storage class.

173 hdl = InMemoryDatasetHandle(metric)

174 data = hdl.get(component="data")

175 self.assertEqual(data, metric.data)

176

177 # Parameters.

178 data = hdl.get(parameters={"slice": slice(2)})

179 self.assertEqual(data.summary, metric.summary)

180 self.assertEqual(data.data, [1, 2])

181

182 data = hdl.get(parameters={"slice": slice(2)}, component="data")

183 self.assertEqual(data, [1, 2])

184

185 # Use parameters in constructor and also override.

186 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest", parameters={"slice": slice(3)})

187 self.assertEqual(hdl.get(component="data"), [1, 2, 3])

188 self.assertEqual(hdl.get(component="counter"), 3)

189 self.assertEqual(hdl.get(component="data", parameters={"slice": slice(1, 3)}), [2, 3])

190 self.assertEqual(hdl.get(component="counter", parameters={"slice": slice(1, 3)}), 2)

191

192 # Ensure the original has not been modified.

193 self.assertEqual(len(metric.data), 4)

194

195 def test_handle_conversion(self):

196 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4])

197

198 # Test conversion with no components or parameters.

199 hdl = InMemoryDatasetHandle(metric)

200 retrieved = hdl.get() # Reset the reference.

201 converted = hdl.get(storageClass="MetricsConversion")

202 self.assertIsNot(type(converted), type(retrieved))

203 self.assertEqual(retrieved, converted)

204

205 # Again with a full storage class.

206 sc = self.factory.getStorageClass("MetricsConversion")

207 converted2 = hdl.get(storageClass=sc)

208 self.assertEqual(converted2, converted)

209

210 # Conversion of component.

211 data = hdl.get(component="data", storageClass="StructuredDataTestListSet")

212 self.assertIsInstance(data, set)

213 self.assertEqual(data, set(converted.data))

214

215

216if __name__ == "__main__":

217 unittest.main()

Coverage for tests/test_dataset_handle.py: 20%

99 statements