Coverage for tests/test_dataset_handle.py: 19%

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23 

24from lsst.daf.butler import DataCoordinate, DimensionUniverse, StorageClassConfig, StorageClassFactory 

25from lsst.daf.butler.tests import MetricsExample 

26from lsst.pipe.base import InMemoryDatasetHandle 

27 

28storageClasses = """ 

29Integer: 

30 pytype: int 

31StructuredDataTestDict: 

32 pytype: dict 

33StructuredDataTestList: 

34 pytype: list 

35 delegate: lsst.daf.butler.tests.ListDelegate 

36 parameters: 

37 - slice 

38 derivedComponents: 

39 counter: Integer 

40StructuredDataTest: 

41 # Data from a simple Python class 

42 pytype: lsst.daf.butler.tests.MetricsExample 

43 delegate: lsst.daf.butler.tests.MetricsDelegate 

44 # Use YAML formatter by default 

45 components: 

46 # Components are those supported by get. 

47 summary: StructuredDataTestDict 

48 output: StructuredDataTestDict 

49 data: StructuredDataTestList 

50 parameters: 

51 - slice 

52 derivedComponents: 

53 counter: Integer 

54MetricsConversion: 

55 # Special storage class to test conversions. 

56 pytype: lsst.daf.butler.tests.MetricsExampleModel 

57 delegate: lsst.daf.butler.tests.MetricsDelegate 

58 converters: 

59 lsst.daf.butler.tests.MetricsExample: lsst.daf.butler.tests.MetricsExampleModel.from_metrics 

60StructuredDataTestListSet: 

61 pytype: set 

62 converters: 

63 list: builtins.set 

64""" 

65 

66 

67class SpecialThing: 

68 """Class known not to have associated StorageClass""" 

69 

70 

71class NotCopyable(MetricsExample): 

72 """Subclass of metrics that can't be copied.""" 

73 

74 def __deepcopy__(self, memo=None): 

75 raise RuntimeError("Can not be copied") 

76 

77 

78class TestDatasetHandle(unittest.TestCase): 

79 """Test in-memory dataset handle.""" 

80 

81 @classmethod 

82 def setUpClass(cls): 

83 config = StorageClassConfig.fromYaml(storageClasses) 

84 cls.factory = StorageClassFactory() 

85 cls.factory.addFromConfig(config) 

86 

87 def test_dataset_handle_basic(self): 

88 inMemoryDataset = 42 

89 hdl = InMemoryDatasetHandle(inMemoryDataset) 

90 

91 self.assertEqual(hdl.get(), inMemoryDataset) 

92 

93 def test_dataset_handle_copy(self): 

94 inMemoryDataset = [1, 2] 

95 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=False) 

96 

97 retrieved = hdl.get() 

98 self.assertEqual(retrieved, inMemoryDataset) 

99 retrieved.append(3) 

100 self.assertEqual(retrieved, inMemoryDataset) 

101 

102 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True) 

103 retrieved = hdl.get() 

104 self.assertEqual(retrieved, inMemoryDataset) 

105 retrieved.append(3) 

106 self.assertNotEqual(retrieved, inMemoryDataset) 

107 

108 inMemoryDataset = NotCopyable(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4]) 

109 hdl = InMemoryDatasetHandle(inMemoryDataset) 

110 self.assertIs(hdl.get(), inMemoryDataset) 

111 

112 hdl = InMemoryDatasetHandle(inMemoryDataset, copy=True, storageClass="MetricsConversion") 

113 with self.assertRaises(NotImplementedError): 

114 hdl.get() 

115 

116 def test_dataset_handle_unknown(self): 

117 inMemoryDataset = SpecialThing() 

118 hdl = InMemoryDatasetHandle(inMemoryDataset) 

119 

120 self.assertEqual(hdl.get(), inMemoryDataset) 

121 

122 with self.assertRaises(KeyError): 

123 # Will not be able to find a matching StorageClass. 

124 hdl.get(parameters={"key": "value"}) 

125 

126 def test_dataset_handle_none(self): 

127 hdl = InMemoryDatasetHandle(None) 

128 self.assertIsNone(hdl.get()) 

129 self.assertIsNone(hdl.get(component="comp")) 

130 self.assertIsNone(hdl.get(parameters={"something": 42})) 

131 

132 def test_dataset_handle_dataid(self): 

133 hdl = InMemoryDatasetHandle(42) 

134 self.assertEqual(dict(hdl.dataId), {}) 

135 

136 dataId = DataCoordinate.makeEmpty(DimensionUniverse()) 

137 hdl = InMemoryDatasetHandle(42, dataId=dataId) 

138 self.assertIs(hdl.dataId, dataId) 

139 

140 dataId = {"tract": 5, "patch": 2, "instrument": "TestCam"} 

141 hdl = InMemoryDatasetHandle(42, **dataId) 

142 self.assertEqual(hdl.dataId, dataId) 

143 

144 hdl = InMemoryDatasetHandle(42, dataId=dataId, tract=6) 

145 self.assertEqual(hdl.dataId["tract"], 6) 

146 

147 dataId = DataCoordinate.standardize({}, universe=DimensionUniverse(), instrument="DummyCam") 

148 hdl = InMemoryDatasetHandle(42, dataId=dataId, physical_filter="g") 

149 self.assertIsInstance(hdl.dataId, DataCoordinate) 

150 self.assertEqual(hdl.dataId["physical_filter"], "g") 

151 

152 def test_dataset_handle_metric(self): 

153 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4]) 

154 

155 # First with explicit storage class. 

156 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest") 

157 retrieved = hdl.get() 

158 self.assertEqual(retrieved, metric) 

159 

160 data = hdl.get(component="data") 

161 self.assertEqual(data, metric.data) 

162 

163 # Now with implicit storage class. 

164 hdl = InMemoryDatasetHandle(metric) 

165 data = hdl.get(component="data") 

166 self.assertEqual(data, metric.data) 

167 

168 # Parameters. 

169 data = hdl.get(parameters={"slice": slice(2)}) 

170 self.assertEqual(data.summary, metric.summary) 

171 self.assertEqual(data.data, [1, 2]) 

172 

173 data = hdl.get(parameters={"slice": slice(2)}, component="data") 

174 self.assertEqual(data, [1, 2]) 

175 

176 # Use parameters in constructor and also override. 

177 hdl = InMemoryDatasetHandle(metric, storageClass="StructuredDataTest", parameters={"slice": slice(3)}) 

178 self.assertEqual(hdl.get(component="data"), [1, 2, 3]) 

179 self.assertEqual(hdl.get(component="counter"), 3) 

180 self.assertEqual(hdl.get(component="data", parameters={"slice": slice(1, 3)}), [2, 3]) 

181 self.assertEqual(hdl.get(component="counter", parameters={"slice": slice(1, 3)}), 2) 

182 

183 # Ensure the original has not been modified. 

184 self.assertEqual(len(metric.data), 4) 

185 

186 def test_handle_conversion(self): 

187 metric = MetricsExample(summary={"a": 1, "b": 2}, output={"c": {"d": 5}}, data=[1, 2, 3, 4]) 

188 

189 # Test conversion with no components or parameters. 

190 hdl = InMemoryDatasetHandle(metric) 

191 retrieved = hdl.get() # Reset the reference. 

192 converted = hdl.get(storageClass="MetricsConversion") 

193 self.assertIsNot(type(converted), type(retrieved)) 

194 self.assertEqual(retrieved, converted) 

195 

196 # Again with a full storage class. 

197 sc = self.factory.getStorageClass("MetricsConversion") 

198 converted2 = hdl.get(storageClass=sc) 

199 self.assertEqual(converted2, converted) 

200 

201 # Conversion of component. 

202 data = hdl.get(component="data", storageClass="StructuredDataTestListSet") 

203 self.assertIsInstance(data, set) 

204 self.assertEqual(data, set(converted.data)) 

205 

206 

207if __name__ == "__main__": 

208 unittest.main()