Coverage for tests/test_formatter.py: 10%

121 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-23 09:30 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Tests related to the formatter infrastructure. 

23""" 

24 

25import inspect 

26import os.path 

27import unittest 

28 

29from lsst.daf.butler import ( 

30 Config, 

31 DataCoordinate, 

32 DatasetType, 

33 DimensionGraph, 

34 DimensionUniverse, 

35 FileDescriptor, 

36 Formatter, 

37 FormatterFactory, 

38 Location, 

39 StorageClass, 

40) 

41from lsst.daf.butler.tests import DatasetTestHelper 

42from lsst.daf.butler.tests.testFormatters import ( 

43 DoNothingFormatter, 

44 MultipleExtensionsFormatter, 

45 SingleExtensionFormatter, 

46) 

47 

48TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

49 

50 

51class FormatterFactoryTestCase(unittest.TestCase, DatasetTestHelper): 

52 """Tests of the formatter factory infrastructure.""" 

53 

54 def setUp(self): 

55 self.id = 0 

56 self.factory = FormatterFactory() 

57 self.universe = DimensionUniverse() 

58 self.dataId = DataCoordinate.makeEmpty(self.universe) 

59 

60 # Dummy FileDescriptor for testing getFormatter 

61 self.fileDescriptor = FileDescriptor( 

62 Location("/a/b/c", "d"), StorageClass("DummyStorageClass", dict, None) 

63 ) 

64 

65 def assertIsFormatter(self, formatter): 

66 """Check that the supplied parameter is either a Formatter instance 

67 or Formatter class.""" 

68 

69 if inspect.isclass(formatter): 

70 self.assertTrue(issubclass(formatter, Formatter), f"Is {formatter} a Formatter") 

71 else: 

72 self.assertIsInstance(formatter, Formatter) 

73 

74 def testFormatter(self): 

75 """Check basic parameter exceptions""" 

76 f = DoNothingFormatter(self.fileDescriptor, self.dataId) 

77 self.assertEqual(f.writeRecipes, {}) 

78 self.assertEqual(f.writeParameters, {}) 

79 self.assertIn("DoNothingFormatter", repr(f)) 

80 

81 with self.assertRaises(TypeError): 

82 DoNothingFormatter() 

83 

84 with self.assertRaises(ValueError): 

85 DoNothingFormatter(self.fileDescriptor, self.dataId, writeParameters={"param1": 0}) 

86 

87 with self.assertRaises(RuntimeError): 

88 DoNothingFormatter(self.fileDescriptor, self.dataId, writeRecipes={"label": "value"}) 

89 

90 with self.assertRaises(NotImplementedError): 

91 f.makeUpdatedLocation(Location("a", "b")) 

92 

93 with self.assertRaises(NotImplementedError): 

94 f.write("str") 

95 

96 def testExtensionValidation(self): 

97 """Test extension validation""" 

98 

99 for file, single_ok, multi_ok in ( 

100 ("e.fits", True, True), 

101 ("e.fit", False, True), 

102 ("e.fits.fz", False, True), 

103 ("e.txt", False, False), 

104 ("e.1.4.fits", True, True), 

105 ("e.3.fit", False, True), 

106 ("e.1.4.fits.gz", False, True), 

107 ): 

108 loc = Location("/a/b/c", file) 

109 

110 for formatter, passes in ( 

111 (SingleExtensionFormatter, single_ok), 

112 (MultipleExtensionsFormatter, multi_ok), 

113 ): 

114 if passes: 

115 formatter.validateExtension(loc) 

116 else: 

117 with self.assertRaises(ValueError): 

118 formatter.validateExtension(loc) 

119 

120 def testRegistry(self): 

121 """Check that formatters can be stored in the registry.""" 

122 formatterTypeName = "lsst.daf.butler.tests.deferredFormatter.DeferredFormatter" 

123 storageClassName = "Image" 

124 self.factory.registerFormatter(storageClassName, formatterTypeName) 

125 f = self.factory.getFormatter(storageClassName, self.fileDescriptor, self.dataId) 

126 self.assertIsFormatter(f) 

127 self.assertEqual(f.name(), formatterTypeName) 

128 self.assertIn(formatterTypeName, str(f)) 

129 self.assertIn(self.fileDescriptor.location.path, str(f)) 

130 

131 fcls = self.factory.getFormatterClass(storageClassName) 

132 self.assertIsFormatter(fcls) 

133 # Defer the import so that we ensure that the infrastructure loaded 

134 # it on demand previously 

135 from lsst.daf.butler.tests.deferredFormatter import DeferredFormatter 

136 

137 self.assertEqual(type(f), DeferredFormatter) 

138 

139 with self.assertRaises(TypeError): 

140 # Requires a constructor parameter 

141 self.factory.getFormatter(storageClassName) 

142 

143 with self.assertRaises(KeyError): 

144 self.factory.getFormatter("Missing", self.fileDescriptor) 

145 

146 # Check that a bad formatter path fails 

147 storageClassName = "BadImage" 

148 self.factory.registerFormatter(storageClassName, "lsst.daf.butler.tests.deferredFormatter.Unknown") 

149 with self.assertRaises(ImportError): 

150 self.factory.getFormatter(storageClassName, self.fileDescriptor, self.dataId) 

151 

152 def testRegistryWithStorageClass(self): 

153 """Test that the registry can be given a StorageClass object.""" 

154 formatterTypeName = "lsst.daf.butler.formatters.yaml.YamlFormatter" 

155 storageClassName = "TestClass" 

156 sc = StorageClass(storageClassName, dict, None) 

157 

158 datasetType = DatasetType("calexp", self.universe.empty, sc) 

159 

160 # Store using an instance 

161 self.factory.registerFormatter(sc, formatterTypeName) 

162 

163 # Retrieve using the class 

164 f = self.factory.getFormatter(sc, self.fileDescriptor, self.dataId) 

165 self.assertIsFormatter(f) 

166 self.assertEqual(f.fileDescriptor, self.fileDescriptor) 

167 

168 # Retrieve using the DatasetType 

169 f2 = self.factory.getFormatter(datasetType, self.fileDescriptor, self.dataId) 

170 self.assertIsFormatter(f2) 

171 self.assertEqual(f.name(), f2.name()) 

172 

173 # Class directly 

174 f2cls = self.factory.getFormatterClass(datasetType) 

175 self.assertIsFormatter(f2cls) 

176 

177 # This might defer the import, pytest may have already loaded it 

178 from lsst.daf.butler.formatters.yaml import YamlFormatter 

179 

180 self.assertEqual(type(f), YamlFormatter) 

181 

182 with self.assertRaises(KeyError): 

183 # Attempt to overwrite using a different value 

184 self.factory.registerFormatter(storageClassName, "lsst.daf.butler.formatters.json.JsonFormatter") 

185 

186 def testRegistryConfig(self): 

187 configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml") 

188 config = Config(configFile) 

189 self.factory.registerFormatters(config["datastore", "formatters"], universe=self.universe) 

190 

191 # Create a DatasetRef with and without instrument matching the 

192 # one in the config file. 

193 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

194 constant_dataId = {"physical_filter": "v", "visit": 1} 

195 sc = StorageClass("DummySC", dict, None) 

196 refPviHsc = self.makeDatasetRef( 

197 "pvi", 

198 dimensions, 

199 sc, 

200 {"instrument": "DummyHSC", **constant_dataId}, 

201 ) 

202 refPviHscFmt = self.factory.getFormatterClass(refPviHsc) 

203 self.assertIsFormatter(refPviHscFmt) 

204 self.assertIn("JsonFormatter", refPviHscFmt.name()) 

205 

206 refPviNotHsc = self.makeDatasetRef( 

207 "pvi", 

208 dimensions, 

209 sc, 

210 {"instrument": "DummyNotHSC", **constant_dataId}, 

211 ) 

212 refPviNotHscFmt = self.factory.getFormatterClass(refPviNotHsc) 

213 self.assertIsFormatter(refPviNotHscFmt) 

214 self.assertIn("PickleFormatter", refPviNotHscFmt.name()) 

215 

216 # Create a DatasetRef that should fall back to using Dimensions 

217 refPvixHsc = self.makeDatasetRef( 

218 "pvix", 

219 dimensions, 

220 sc, 

221 {"instrument": "DummyHSC", **constant_dataId}, 

222 ) 

223 refPvixNotHscFmt = self.factory.getFormatterClass(refPvixHsc) 

224 self.assertIsFormatter(refPvixNotHscFmt) 

225 self.assertIn("PickleFormatter", refPvixNotHscFmt.name()) 

226 

227 # Create a DatasetRef that should fall back to using StorageClass 

228 dimensionsNoV = DimensionGraph(self.universe, names=("physical_filter", "instrument")) 

229 refPvixNotHscDims = self.makeDatasetRef( 

230 "pvix", 

231 dimensionsNoV, 

232 sc, 

233 {"instrument": "DummyHSC", "physical_filter": "v"}, 

234 ) 

235 refPvixNotHscDims_fmt = self.factory.getFormatterClass(refPvixNotHscDims) 

236 self.assertIsFormatter(refPvixNotHscDims_fmt) 

237 self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name()) 

238 

239 # Check that parameters are stored 

240 refParam = self.makeDatasetRef( 

241 "paramtest", 

242 dimensions, 

243 sc, 

244 {"instrument": "DummyNotHSC", **constant_dataId}, 

245 ) 

246 lookup, refParam_fmt, kwargs = self.factory.getFormatterClassWithMatch(refParam) 

247 self.assertIn("writeParameters", kwargs) 

248 expected = {"max": 5, "min": 2, "comment": "Additional commentary", "recipe": "recipe1"} 

249 self.assertEqual(kwargs["writeParameters"], expected) 

250 self.assertIn("FormatterTest", refParam_fmt.name()) 

251 

252 f = self.factory.getFormatter(refParam, self.fileDescriptor, self.dataId) 

253 self.assertEqual(f.writeParameters, expected) 

254 

255 f = self.factory.getFormatter( 

256 refParam, self.fileDescriptor, self.dataId, writeParameters={"min": 22, "extra": 50} 

257 ) 

258 self.assertEqual( 

259 f.writeParameters, 

260 {"max": 5, "min": 22, "comment": "Additional commentary", "extra": 50, "recipe": "recipe1"}, 

261 ) 

262 

263 self.assertIn("recipe1", f.writeRecipes) 

264 self.assertEqual(f.writeParameters["recipe"], "recipe1") 

265 

266 with self.assertRaises(ValueError): 

267 # "new" is not allowed as a write parameter 

268 self.factory.getFormatter(refParam, self.fileDescriptor, self.dataId, writeParameters={"new": 1}) 

269 

270 with self.assertRaises(RuntimeError): 

271 # "mode" is a required recipe parameter 

272 self.factory.getFormatter( 

273 refParam, self.fileDescriptor, self.dataId, writeRecipes={"recipe3": {"notmode": 1}} 

274 ) 

275 

276 

277if __name__ == "__main__": 

278 unittest.main()