Coverage for tests/test_dataset_handle.py: 20%

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import unittest 

29 

30from lsst.daf.butler import DataCoordinate, DimensionUniverse, StorageClassConfig, StorageClassFactory 

31from lsst.daf.butler.tests import MetricsExample 

32from lsst.pipe.base import InMemoryDatasetHandle 

33 

34storageClasses = """ 

35Integer: 

36 pytype: int 

37StructuredDataTestDict: 

38 pytype: dict 

39StructuredDataTestList: 

40 pytype: list 

41 delegate: lsst.daf.butler.tests.ListDelegate 

42 parameters: 

43 - slice 

44 derivedComponents: 

45 counter: Integer 

46StructuredDataTest: 

47 # Data from a simple Python class 

48 pytype: lsst.daf.butler.tests.MetricsExample 

49 delegate: lsst.daf.butler.tests.MetricsDelegate 

50 # Use YAML formatter by default 

51 components: 

52 # Components are those supported by get. 

53 summary: StructuredDataTestDict 

54 output: StructuredDataTestDict 

55 data: StructuredDataTestList 

56 parameters: 

57 - slice 

58 derivedComponents: 

59 counter: Integer 

60MetricsConversion: 

61 # Special storage class to test conversions. 

62 pytype: lsst.daf.butler.tests.MetricsExampleModel 

63 delegate: lsst.daf.butler.tests.MetricsDelegate 

64 converters: 

65 lsst.daf.butler.tests.MetricsExample: lsst.daf.butler.tests.MetricsExampleModel.from_metrics 

66StructuredDataTestListSet: 

67 pytype: set 

68 converters: 

69 list: builtins.set 

70""" 

71 

72 

73class SpecialThing: 

74 """Class known not to have associated StorageClass""" 

75 

76 

77class NotCopyable(MetricsExample): 

78 """Subclass of metrics that can't be copied.""" 

79 

80 def __deepcopy__(self, memo=None): 

81 raise RuntimeError("Can not be copied") 

82 

83 

84class TestDatasetHandle(unittest.TestCase): 

85 """Test in-memory dataset handle.""" 

86 

87 @classmethod 

88 def setUpClass(cls): 

89 cls.storage_class_config = StorageClassConfig.fromYaml(storageClasses) 

90 cls.factory = StorageClassFactory() 

91 

92 def setUp(self): 

93 self.factory.reset() 

94 self.factory.addFromConfig(self.storage_class_config) 

95 

96 def test_dataset_handle_basic(self): 

97 inMemoryDataset = 42 

98 hdl = InMemoryDatasetHandle(inMemoryDataset) 

99 

100 self.assertEqual(hdl.get(), inMemoryDataset) 

101 

102 def test_dataset_handle_copy(self): 

103 inMemoryDataset = [1, 2] 

104 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=False) 

105 

106 retrieved = hdl.get() 

107 self.assertEqual(retrieved, inMemoryDataset) 

108 retrieved.append(3) 

109 self.assertEqual(retrieved, inMemoryDataset) 

110 

111 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True) 

112 retrieved = hdl.get() 

113 self.assertEqual(retrieved, inMemoryDataset) 

114 retrieved.append(3) 

115 self.assertNotEqual(retrieved, inMemoryDataset) 

116 

117 inMemoryDataset = NotCopyable(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4]) 

118 hdl = InMemoryDatasetHandle(inMemoryDataset) 

119 self.assertIs(hdl.get(), inMemoryDataset) 

120 

121 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True, storageClass="MetricsConversion") 

122 with self.assertRaises(NotImplementedError): 

123 hdl.get() 

124 

125 def test_dataset_handle_unknown(self): 

126 inMemoryDataset = SpecialThing() 

127 hdl = InMemoryDatasetHandle(inMemoryDataset) 

128 

129 self.assertEqual(hdl.get(), inMemoryDataset) 

130 

131 with self.assertRaises(KeyError): 

132 # Will not be able to find a matching StorageClass. 

133 hdl.get(parameters={"key": "value"}) 

134 

135 def test_dataset_handle_none(self): 

136 hdl = InMemoryDatasetHandle(None) 

137 self.assertIsNone(hdl.get()) 

138 self.assertIsNone(hdl.get(component="comp")) 

139 self.assertIsNone(hdl.get(parameters={"something": 42})) 

140 

141 def test_dataset_handle_dataid(self): 

142 hdl = InMemoryDatasetHandle(42) 

143 self.assertEqual(dict(hdl.dataId.required), {}) 

144 

145 dataId = DataCoordinate.make_empty(DimensionUniverse()) 

146 hdl = InMemoryDatasetHandle(42, dataId=dataId) 

147 self.assertIs(hdl.dataId, dataId) 

148 

149 dataId = {"tract": 5, "patch": 2, "instrument": "TestCam"} 

150 hdl = InMemoryDatasetHandle(42, **dataId) 

151 self.assertEqual(hdl.dataId, dataId) 

152 

153 hdl = InMemoryDatasetHandle(42, dataId=dataId, tract=6) 

154 self.assertEqual(hdl.dataId["tract"], 6) 

155 

156 dataId = DataCoordinate.standardize({}, universe=DimensionUniverse(), instrument="DummyCam") 

157 hdl = InMemoryDatasetHandle(42, dataId=dataId, physical_filter="g") 

158 self.assertIsInstance(hdl.dataId, DataCoordinate) 

159 self.assertEqual(hdl.dataId["physical_filter"], "g") 

160 

161 def test_dataset_handle_metric(self): 

162 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4]) 

163 

164 # First with explicit storage class. 

165 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest") 

166 retrieved = hdl.get() 

167 self.assertEqual(retrieved, metric) 

168 

169 data = hdl.get(component="data") 

170 self.assertEqual(data, metric.data) 

171 

172 # Now with implicit storage class. 

173 hdl = InMemoryDatasetHandle(metric) 

174 data = hdl.get(component="data") 

175 self.assertEqual(data, metric.data) 

176 

177 # Parameters. 

178 data = hdl.get(parameters={"slice": slice(2)}) 

179 self.assertEqual(data.summary, metric.summary) 

180 self.assertEqual(data.data, [1, 2]) 

181 

182 data = hdl.get(parameters={"slice": slice(2)}, component="data") 

183 self.assertEqual(data, [1, 2]) 

184 

185 # Use parameters in constructor and also override. 

186 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest", parameters={"slice": slice(3)}) 

187 self.assertEqual(hdl.get(component="data"), [1, 2, 3]) 

188 self.assertEqual(hdl.get(component="counter"), 3) 

189 self.assertEqual(hdl.get(component="data", parameters={"slice": slice(1, 3)}), [2, 3]) 

190 self.assertEqual(hdl.get(component="counter", parameters={"slice": slice(1, 3)}), 2) 

191 

192 # Ensure the original has not been modified. 

193 self.assertEqual(len(metric.data), 4) 

194 

195 def test_handle_conversion(self): 

196 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4]) 

197 

198 # Test conversion with no components or parameters. 

199 hdl = InMemoryDatasetHandle(metric) 

200 retrieved = hdl.get() # Reset the reference. 

201 converted = hdl.get(storageClass="MetricsConversion") 

202 self.assertIsNot(type(converted), type(retrieved)) 

203 self.assertEqual(retrieved, converted) 

204 

205 # Again with a full storage class. 

206 sc = self.factory.getStorageClass("MetricsConversion") 

207 converted2 = hdl.get(storageClass=sc) 

208 self.assertEqual(converted2, converted) 

209 

210 # Conversion of component. 

211 data = hdl.get(component="data", storageClass="StructuredDataTestListSet") 

212 self.assertIsInstance(data, set) 

213 self.assertEqual(data, set(converted.data)) 

214 

215 

216if __name__ == "__main__": 

217 unittest.main()