Coverage for tests/test_formatter.py: 14%

122 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-01 19:55 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Tests related to the formatter infrastructure. 

23""" 

24 

25import inspect 

26import os.path 

27import unittest 

28 

29from lsst.daf.butler.tests import DatasetTestHelper 

30from lsst.daf.butler import (Formatter, FormatterFactory, StorageClass, DatasetType, Config, 

31 FileDescriptor, Location, DataCoordinate, DimensionUniverse, DimensionGraph) 

32from lsst.daf.butler.tests.testFormatters import (DoNothingFormatter, MultipleExtensionsFormatter, 

33 SingleExtensionFormatter) 

34 

35TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

36 

37 

38class FormatterFactoryTestCase(unittest.TestCase, DatasetTestHelper): 

39 """Tests of the formatter factory infrastructure. 

40 """ 

41 

42 def setUp(self): 

43 self.id = 0 

44 self.factory = FormatterFactory() 

45 self.universe = DimensionUniverse() 

46 self.dataId = DataCoordinate.makeEmpty(self.universe) 

47 

48 # Dummy FileDescriptor for testing getFormatter 

49 self.fileDescriptor = FileDescriptor(Location("/a/b/c", "d"), 

50 StorageClass("DummyStorageClass", dict, None)) 

51 

52 def assertIsFormatter(self, formatter): 

53 """Check that the supplied parameter is either a Formatter instance 

54 or Formatter class.""" 

55 

56 if inspect.isclass(formatter): 

57 self.assertTrue(issubclass(formatter, Formatter), f"Is {formatter} a Formatter") 

58 else: 

59 self.assertIsInstance(formatter, Formatter) 

60 

61 def testFormatter(self): 

62 """Check basic parameter exceptions""" 

63 f = DoNothingFormatter(self.fileDescriptor, self.dataId) 

64 self.assertEqual(f.writeRecipes, {}) 

65 self.assertEqual(f.writeParameters, {}) 

66 self.assertIn("DoNothingFormatter", repr(f)) 

67 

68 with self.assertRaises(TypeError): 

69 DoNothingFormatter() 

70 

71 with self.assertRaises(ValueError): 

72 DoNothingFormatter(self.fileDescriptor, self.dataId, writeParameters={"param1": 0}) 

73 

74 with self.assertRaises(RuntimeError): 

75 DoNothingFormatter(self.fileDescriptor, self.dataId, writeRecipes={"label": "value"}) 

76 

77 with self.assertRaises(NotImplementedError): 

78 f.makeUpdatedLocation(Location("a", "b")) 

79 

80 with self.assertRaises(NotImplementedError): 

81 f.write("str") 

82 

83 def testExtensionValidation(self): 

84 """Test extension validation""" 

85 

86 for file, single_ok, multi_ok in (("e.fits", True, True), 

87 ("e.fit", False, True), 

88 ("e.fits.fz", False, True), 

89 ("e.txt", False, False), 

90 ("e.1.4.fits", True, True), 

91 ("e.3.fit", False, True), 

92 ("e.1.4.fits.gz", False, True), 

93 ): 

94 loc = Location("/a/b/c", file) 

95 

96 for formatter, passes in ((SingleExtensionFormatter, single_ok), 

97 (MultipleExtensionsFormatter, multi_ok)): 

98 if passes: 

99 formatter.validateExtension(loc) 

100 else: 

101 with self.assertRaises(ValueError): 

102 formatter.validateExtension(loc) 

103 

104 def testRegistry(self): 

105 """Check that formatters can be stored in the registry. 

106 """ 

107 formatterTypeName = "lsst.daf.butler.tests.deferredFormatter.DeferredFormatter" 

108 storageClassName = "Image" 

109 self.factory.registerFormatter(storageClassName, formatterTypeName) 

110 f = self.factory.getFormatter(storageClassName, self.fileDescriptor, self.dataId) 

111 self.assertIsFormatter(f) 

112 self.assertEqual(f.name(), formatterTypeName) 

113 self.assertIn(formatterTypeName, str(f)) 

114 self.assertIn(self.fileDescriptor.location.path, str(f)) 

115 

116 fcls = self.factory.getFormatterClass(storageClassName) 

117 self.assertIsFormatter(fcls) 

118 # Defer the import so that we ensure that the infrastructure loaded 

119 # it on demand previously 

120 from lsst.daf.butler.tests.deferredFormatter import DeferredFormatter 

121 self.assertEqual(type(f), DeferredFormatter) 

122 

123 with self.assertRaises(TypeError): 

124 # Requires a constructor parameter 

125 self.factory.getFormatter(storageClassName) 

126 

127 with self.assertRaises(KeyError): 

128 self.factory.getFormatter("Missing", self.fileDescriptor) 

129 

130 # Check that a bad formatter path fails 

131 storageClassName = "BadImage" 

132 self.factory.registerFormatter(storageClassName, "lsst.daf.butler.tests.deferredFormatter.Unknown") 

133 with self.assertRaises(ImportError): 

134 self.factory.getFormatter(storageClassName, self.fileDescriptor, self.dataId) 

135 

136 def testRegistryWithStorageClass(self): 

137 """Test that the registry can be given a StorageClass object. 

138 """ 

139 formatterTypeName = "lsst.daf.butler.formatters.yaml.YamlFormatter" 

140 storageClassName = "TestClass" 

141 sc = StorageClass(storageClassName, dict, None) 

142 

143 datasetType = DatasetType("calexp", self.universe.empty, sc) 

144 

145 # Store using an instance 

146 self.factory.registerFormatter(sc, formatterTypeName) 

147 

148 # Retrieve using the class 

149 f = self.factory.getFormatter(sc, self.fileDescriptor, self.dataId) 

150 self.assertIsFormatter(f) 

151 self.assertEqual(f.fileDescriptor, self.fileDescriptor) 

152 

153 # Retrieve using the DatasetType 

154 f2 = self.factory.getFormatter(datasetType, self.fileDescriptor, self.dataId) 

155 self.assertIsFormatter(f2) 

156 self.assertEqual(f.name(), f2.name()) 

157 

158 # Class directly 

159 f2cls = self.factory.getFormatterClass(datasetType) 

160 self.assertIsFormatter(f2cls) 

161 

162 # This might defer the import, pytest may have already loaded it 

163 from lsst.daf.butler.formatters.yaml import YamlFormatter 

164 self.assertEqual(type(f), YamlFormatter) 

165 

166 with self.assertRaises(KeyError): 

167 # Attempt to overwrite using a different value 

168 self.factory.registerFormatter(storageClassName, 

169 "lsst.daf.butler.formatters.json.JsonFormatter") 

170 

171 def testRegistryConfig(self): 

172 configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml") 

173 config = Config(configFile) 

174 self.factory.registerFormatters(config["datastore", "formatters"], universe=self.universe) 

175 

176 # Create a DatasetRef with and without instrument matching the 

177 # one in the config file. 

178 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

179 sc = StorageClass("DummySC", dict, None) 

180 refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyHSC", 

181 "physical_filter": "v"}, 

182 conform=False) 

183 refPviHscFmt = self.factory.getFormatterClass(refPviHsc) 

184 self.assertIsFormatter(refPviHscFmt) 

185 self.assertIn("JsonFormatter", refPviHscFmt.name()) 

186 

187 refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyNotHSC", 

188 "physical_filter": "v"}, 

189 conform=False) 

190 refPviNotHscFmt = self.factory.getFormatterClass(refPviNotHsc) 

191 self.assertIsFormatter(refPviNotHscFmt) 

192 self.assertIn("PickleFormatter", refPviNotHscFmt.name()) 

193 

194 # Create a DatasetRef that should fall back to using Dimensions 

195 refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, {"instrument": "DummyHSC", 

196 "physical_filter": "v"}, 

197 conform=False) 

198 refPvixNotHscFmt = self.factory.getFormatterClass(refPvixHsc) 

199 self.assertIsFormatter(refPvixNotHscFmt) 

200 self.assertIn("PickleFormatter", refPvixNotHscFmt.name()) 

201 

202 # Create a DatasetRef that should fall back to using StorageClass 

203 dimensionsNoV = DimensionGraph(self.universe, names=("physical_filter", "instrument")) 

204 refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, {"instrument": "DummyHSC", 

205 "physical_filter": "v"}, 

206 conform=False) 

207 refPvixNotHscDims_fmt = self.factory.getFormatterClass(refPvixNotHscDims) 

208 self.assertIsFormatter(refPvixNotHscDims_fmt) 

209 self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name()) 

210 

211 # Check that parameters are stored 

212 refParam = self.makeDatasetRef("paramtest", dimensions, sc, {"instrument": "DummyNotHSC", 

213 "physical_filter": "v"}, 

214 conform=False) 

215 lookup, refParam_fmt, kwargs = self.factory.getFormatterClassWithMatch(refParam) 

216 self.assertIn("writeParameters", kwargs) 

217 expected = {"max": 5, "min": 2, "comment": "Additional commentary", "recipe": "recipe1"} 

218 self.assertEqual(kwargs["writeParameters"], expected) 

219 self.assertIn("FormatterTest", refParam_fmt.name()) 

220 

221 f = self.factory.getFormatter(refParam, self.fileDescriptor, self.dataId) 

222 self.assertEqual(f.writeParameters, expected) 

223 

224 f = self.factory.getFormatter(refParam, self.fileDescriptor, self.dataId, 

225 writeParameters={"min": 22, "extra": 50}) 

226 self.assertEqual(f.writeParameters, {"max": 5, "min": 22, "comment": "Additional commentary", 

227 "extra": 50, "recipe": "recipe1"}) 

228 

229 self.assertIn("recipe1", f.writeRecipes) 

230 self.assertEqual(f.writeParameters["recipe"], "recipe1") 

231 

232 with self.assertRaises(ValueError): 

233 # "new" is not allowed as a write parameter 

234 self.factory.getFormatter(refParam, self.fileDescriptor, self.dataId, writeParameters={"new": 1}) 

235 

236 with self.assertRaises(RuntimeError): 

237 # "mode" is a required recipe parameter 

238 self.factory.getFormatter(refParam, self.fileDescriptor, self.dataId, 

239 writeRecipes={"recipe3": {"notmode": 1}}) 

240 

241 

242if __name__ == "__main__": 242 ↛ 243line 242 didn't jump to line 243, because the condition on line 242 was never true

243 unittest.main()