Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Tests related to the formatter infrastructure. 

23""" 

24 

25import inspect 

26import os.path 

27import unittest 

28 

29from lsst.daf.butler.tests import DatasetTestHelper 

30from lsst.daf.butler import (Formatter, FormatterFactory, StorageClass, DatasetType, Config, 

31 FileDescriptor, Location, DimensionUniverse, DimensionGraph) 

32from lsst.daf.butler.tests.testFormatters import DoNothingFormatter 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class FormatterFactoryTestCase(unittest.TestCase, DatasetTestHelper): 

38 """Tests of the formatter factory infrastructure. 

39 """ 

40 

41 def setUp(self): 

42 self.id = 0 

43 self.factory = FormatterFactory() 

44 

45 # Dummy FileDescriptor for testing getFormatter 

46 self.fileDescriptor = FileDescriptor(Location("/a/b/c", "d"), 

47 StorageClass("DummyStorageClass", dict, None)) 

48 

49 def assertIsFormatter(self, formatter): 

50 """Check that the supplied parameter is either a Formatter instance 

51 or Formatter class.""" 

52 

53 if inspect.isclass(formatter): 

54 self.assertTrue(issubclass(formatter, Formatter), f"Is {formatter} a Formatter") 

55 else: 

56 self.assertIsInstance(formatter, Formatter) 

57 

58 def testFormatter(self): 

59 """Check basic parameter exceptions""" 

60 f = DoNothingFormatter(self.fileDescriptor) 

61 self.assertEqual(f.writeRecipes, {}) 

62 self.assertEqual(f.writeParameters, {}) 

63 self.assertIn("DoNothingFormatter", repr(f)) 

64 

65 with self.assertRaises(TypeError): 

66 DoNothingFormatter() 

67 

68 with self.assertRaises(ValueError): 

69 DoNothingFormatter(self.fileDescriptor, writeParameters={"param1": 0}) 

70 

71 with self.assertRaises(RuntimeError): 

72 DoNothingFormatter(self.fileDescriptor, writeRecipes={"label": "value"}) 

73 

74 with self.assertRaises(NotImplementedError): 

75 f.makeUpdatedLocation(Location("a", "b")) 

76 

77 with self.assertRaises(NotImplementedError): 

78 f.write("str") 

79 

80 def testRegistry(self): 

81 """Check that formatters can be stored in the registry. 

82 """ 

83 formatterTypeName = "lsst.daf.butler.formatters.pexConfig.PexConfigFormatter" 

84 storageClassName = "Image" 

85 self.factory.registerFormatter(storageClassName, formatterTypeName) 

86 f = self.factory.getFormatter(storageClassName, self.fileDescriptor) 

87 self.assertIsFormatter(f) 

88 self.assertEqual(f.name(), formatterTypeName) 

89 self.assertIn(formatterTypeName, str(f)) 

90 self.assertIn(self.fileDescriptor.location.path, str(f)) 

91 

92 fcls = self.factory.getFormatterClass(storageClassName) 

93 self.assertIsFormatter(fcls) 

94 # Defer the import so that we ensure that the infrastructure loaded 

95 # it on demand previously 

96 from lsst.daf.butler.formatters.pexConfig import PexConfigFormatter 

97 self.assertEqual(type(f), PexConfigFormatter) 

98 

99 with self.assertRaises(TypeError): 

100 # Requires a constructor parameter 

101 self.factory.getFormatter(storageClassName) 

102 

103 with self.assertRaises(KeyError): 

104 self.factory.getFormatter("Missing", self.fileDescriptor) 

105 

106 def testRegistryWithStorageClass(self): 

107 """Test that the registry can be given a StorageClass object. 

108 """ 

109 formatterTypeName = "lsst.daf.butler.formatters.yaml.YamlFormatter" 

110 storageClassName = "TestClass" 

111 sc = StorageClass(storageClassName, dict, None) 

112 

113 universe = DimensionUniverse() 

114 datasetType = DatasetType("calexp", universe.empty, sc) 

115 

116 # Store using an instance 

117 self.factory.registerFormatter(sc, formatterTypeName) 

118 

119 # Retrieve using the class 

120 f = self.factory.getFormatter(sc, self.fileDescriptor) 

121 self.assertIsFormatter(f) 

122 self.assertEqual(f.fileDescriptor, self.fileDescriptor) 

123 

124 # Retrieve using the DatasetType 

125 f2 = self.factory.getFormatter(datasetType, self.fileDescriptor) 

126 self.assertIsFormatter(f2) 

127 self.assertEqual(f.name(), f2.name()) 

128 

129 # Class directly 

130 f2cls = self.factory.getFormatterClass(datasetType) 

131 self.assertIsFormatter(f2cls) 

132 

133 # This might defer the import, pytest may have already loaded it 

134 from lsst.daf.butler.formatters.yaml import YamlFormatter 

135 self.assertEqual(type(f), YamlFormatter) 

136 

137 with self.assertRaises(KeyError): 

138 # Attempt to overwrite using a different value 

139 self.factory.registerFormatter(storageClassName, 

140 "lsst.daf.butler.formatters.json.JsonFormatter") 

141 

142 def testRegistryConfig(self): 

143 configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml") 

144 config = Config(configFile) 

145 universe = DimensionUniverse() 

146 self.factory.registerFormatters(config["datastore", "formatters"], universe=universe) 

147 

148 # Create a DatasetRef with and without instrument matching the 

149 # one in the config file. 

150 dimensions = universe.extract(("visit", "physical_filter", "instrument")) 

151 sc = StorageClass("DummySC", dict, None) 

152 refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyHSC", 

153 "physical_filter": "v"}, 

154 conform=False) 

155 refPviHscFmt = self.factory.getFormatterClass(refPviHsc) 

156 self.assertIsFormatter(refPviHscFmt) 

157 self.assertIn("JsonFormatter", refPviHscFmt.name()) 

158 

159 refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyNotHSC", 

160 "physical_filter": "v"}, 

161 conform=False) 

162 refPviNotHscFmt = self.factory.getFormatterClass(refPviNotHsc) 

163 self.assertIsFormatter(refPviNotHscFmt) 

164 self.assertIn("PickleFormatter", refPviNotHscFmt.name()) 

165 

166 # Create a DatasetRef that should fall back to using Dimensions 

167 refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, {"instrument": "DummyHSC", 

168 "physical_filter": "v"}, 

169 conform=False) 

170 refPvixNotHscFmt = self.factory.getFormatterClass(refPvixHsc) 

171 self.assertIsFormatter(refPvixNotHscFmt) 

172 self.assertIn("PickleFormatter", refPvixNotHscFmt.name()) 

173 

174 # Create a DatasetRef that should fall back to using StorageClass 

175 dimensionsNoV = DimensionGraph(universe, names=("physical_filter", "instrument")) 

176 refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, {"instrument": "DummyHSC", 

177 "physical_filter": "v"}, 

178 conform=False) 

179 refPvixNotHscDims_fmt = self.factory.getFormatterClass(refPvixNotHscDims) 

180 self.assertIsFormatter(refPvixNotHscDims_fmt) 

181 self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name()) 

182 

183 # Check that parameters are stored 

184 refParam = self.makeDatasetRef("paramtest", dimensions, sc, {"instrument": "DummyNotHSC", 

185 "physical_filter": "v"}, 

186 conform=False) 

187 lookup, refParam_fmt, kwargs = self.factory.getFormatterClassWithMatch(refParam) 

188 self.assertIn("writeParameters", kwargs) 

189 expected = {"max": 5, "min": 2, "comment": "Additional commentary", "recipe": "recipe1"} 

190 self.assertEqual(kwargs["writeParameters"], expected) 

191 self.assertIn("FormatterTest", refParam_fmt.name()) 

192 

193 f = self.factory.getFormatter(refParam, self.fileDescriptor) 

194 self.assertEqual(f.writeParameters, expected) 

195 

196 f = self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"min": 22, 

197 "extra": 50}) 

198 self.assertEqual(f.writeParameters, {"max": 5, "min": 22, "comment": "Additional commentary", 

199 "extra": 50, "recipe": "recipe1"}) 

200 

201 self.assertIn("recipe1", f.writeRecipes) 

202 self.assertEqual(f.writeParameters["recipe"], "recipe1") 

203 

204 with self.assertRaises(ValueError): 

205 # "new" is not allowed as a write parameter 

206 self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"new": 1}) 

207 

208 with self.assertRaises(RuntimeError): 

209 # "mode" is a required recipe parameter 

210 self.factory.getFormatter(refParam, self.fileDescriptor, writeRecipes={"recipe3": {"notmode": 1}}) 

211 

212 

213if __name__ == "__main__": 213 ↛ 214line 213 didn't jump to line 214, because the condition on line 213 was never true

214 unittest.main()