Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Tests related to the formatter infrastructure. 

23""" 

24 

25import inspect 

26import os.path 

27import unittest 

28 

29from lsst.daf.butler.tests import DatasetTestHelper 

30from lsst.daf.butler import (Formatter, FormatterFactory, StorageClass, DatasetType, Config, 

31 FileDescriptor, Location, DimensionUniverse, DimensionGraph) 

32from lsst.daf.butler.tests.testFormatters import (DoNothingFormatter, MultipleExtensionsFormatter, 

33 SingleExtensionFormatter) 

34 

35TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

36 

37 

38class FormatterFactoryTestCase(unittest.TestCase, DatasetTestHelper): 

39 """Tests of the formatter factory infrastructure. 

40 """ 

41 

42 def setUp(self): 

43 self.id = 0 

44 self.factory = FormatterFactory() 

45 

46 # Dummy FileDescriptor for testing getFormatter 

47 self.fileDescriptor = FileDescriptor(Location("/a/b/c", "d"), 

48 StorageClass("DummyStorageClass", dict, None)) 

49 

50 def assertIsFormatter(self, formatter): 

51 """Check that the supplied parameter is either a Formatter instance 

52 or Formatter class.""" 

53 

54 if inspect.isclass(formatter): 

55 self.assertTrue(issubclass(formatter, Formatter), f"Is {formatter} a Formatter") 

56 else: 

57 self.assertIsInstance(formatter, Formatter) 

58 

59 def testFormatter(self): 

60 """Check basic parameter exceptions""" 

61 f = DoNothingFormatter(self.fileDescriptor) 

62 self.assertEqual(f.writeRecipes, {}) 

63 self.assertEqual(f.writeParameters, {}) 

64 self.assertIn("DoNothingFormatter", repr(f)) 

65 

66 with self.assertRaises(TypeError): 

67 DoNothingFormatter() 

68 

69 with self.assertRaises(ValueError): 

70 DoNothingFormatter(self.fileDescriptor, writeParameters={"param1": 0}) 

71 

72 with self.assertRaises(RuntimeError): 

73 DoNothingFormatter(self.fileDescriptor, writeRecipes={"label": "value"}) 

74 

75 with self.assertRaises(NotImplementedError): 

76 f.makeUpdatedLocation(Location("a", "b")) 

77 

78 with self.assertRaises(NotImplementedError): 

79 f.write("str") 

80 

81 def testExtensionValidation(self): 

82 """Test extension validation""" 

83 

84 for file, single_ok, multi_ok in (("e.fits", True, True), 

85 ("e.fit", False, True), 

86 ("e.fits.fz", False, True), 

87 ("e.txt", False, False), 

88 ("e.1.4.fits", True, True), 

89 ("e.3.fit", False, True), 

90 ("e.1.4.fits.gz", False, True), 

91 ): 

92 loc = Location("/a/b/c", file) 

93 

94 for formatter, passes in ((SingleExtensionFormatter, single_ok), 

95 (MultipleExtensionsFormatter, multi_ok)): 

96 if passes: 

97 formatter.validateExtension(loc) 

98 else: 

99 with self.assertRaises(ValueError): 

100 formatter.validateExtension(loc) 

101 

102 def testRegistry(self): 

103 """Check that formatters can be stored in the registry. 

104 """ 

105 formatterTypeName = "lsst.daf.butler.formatters.pexConfig.PexConfigFormatter" 

106 storageClassName = "Image" 

107 self.factory.registerFormatter(storageClassName, formatterTypeName) 

108 f = self.factory.getFormatter(storageClassName, self.fileDescriptor) 

109 self.assertIsFormatter(f) 

110 self.assertEqual(f.name(), formatterTypeName) 

111 self.assertIn(formatterTypeName, str(f)) 

112 self.assertIn(self.fileDescriptor.location.path, str(f)) 

113 

114 fcls = self.factory.getFormatterClass(storageClassName) 

115 self.assertIsFormatter(fcls) 

116 # Defer the import so that we ensure that the infrastructure loaded 

117 # it on demand previously 

118 from lsst.daf.butler.formatters.pexConfig import PexConfigFormatter 

119 self.assertEqual(type(f), PexConfigFormatter) 

120 

121 with self.assertRaises(TypeError): 

122 # Requires a constructor parameter 

123 self.factory.getFormatter(storageClassName) 

124 

125 with self.assertRaises(KeyError): 

126 self.factory.getFormatter("Missing", self.fileDescriptor) 

127 

128 def testRegistryWithStorageClass(self): 

129 """Test that the registry can be given a StorageClass object. 

130 """ 

131 formatterTypeName = "lsst.daf.butler.formatters.yaml.YamlFormatter" 

132 storageClassName = "TestClass" 

133 sc = StorageClass(storageClassName, dict, None) 

134 

135 universe = DimensionUniverse() 

136 datasetType = DatasetType("calexp", universe.empty, sc) 

137 

138 # Store using an instance 

139 self.factory.registerFormatter(sc, formatterTypeName) 

140 

141 # Retrieve using the class 

142 f = self.factory.getFormatter(sc, self.fileDescriptor) 

143 self.assertIsFormatter(f) 

144 self.assertEqual(f.fileDescriptor, self.fileDescriptor) 

145 

146 # Retrieve using the DatasetType 

147 f2 = self.factory.getFormatter(datasetType, self.fileDescriptor) 

148 self.assertIsFormatter(f2) 

149 self.assertEqual(f.name(), f2.name()) 

150 

151 # Class directly 

152 f2cls = self.factory.getFormatterClass(datasetType) 

153 self.assertIsFormatter(f2cls) 

154 

155 # This might defer the import, pytest may have already loaded it 

156 from lsst.daf.butler.formatters.yaml import YamlFormatter 

157 self.assertEqual(type(f), YamlFormatter) 

158 

159 with self.assertRaises(KeyError): 

160 # Attempt to overwrite using a different value 

161 self.factory.registerFormatter(storageClassName, 

162 "lsst.daf.butler.formatters.json.JsonFormatter") 

163 

164 def testRegistryConfig(self): 

165 configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml") 

166 config = Config(configFile) 

167 universe = DimensionUniverse() 

168 self.factory.registerFormatters(config["datastore", "formatters"], universe=universe) 

169 

170 # Create a DatasetRef with and without instrument matching the 

171 # one in the config file. 

172 dimensions = universe.extract(("visit", "physical_filter", "instrument")) 

173 sc = StorageClass("DummySC", dict, None) 

174 refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyHSC", 

175 "physical_filter": "v"}, 

176 conform=False) 

177 refPviHscFmt = self.factory.getFormatterClass(refPviHsc) 

178 self.assertIsFormatter(refPviHscFmt) 

179 self.assertIn("JsonFormatter", refPviHscFmt.name()) 

180 

181 refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyNotHSC", 

182 "physical_filter": "v"}, 

183 conform=False) 

184 refPviNotHscFmt = self.factory.getFormatterClass(refPviNotHsc) 

185 self.assertIsFormatter(refPviNotHscFmt) 

186 self.assertIn("PickleFormatter", refPviNotHscFmt.name()) 

187 

188 # Create a DatasetRef that should fall back to using Dimensions 

189 refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, {"instrument": "DummyHSC", 

190 "physical_filter": "v"}, 

191 conform=False) 

192 refPvixNotHscFmt = self.factory.getFormatterClass(refPvixHsc) 

193 self.assertIsFormatter(refPvixNotHscFmt) 

194 self.assertIn("PickleFormatter", refPvixNotHscFmt.name()) 

195 

196 # Create a DatasetRef that should fall back to using StorageClass 

197 dimensionsNoV = DimensionGraph(universe, names=("physical_filter", "instrument")) 

198 refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, {"instrument": "DummyHSC", 

199 "physical_filter": "v"}, 

200 conform=False) 

201 refPvixNotHscDims_fmt = self.factory.getFormatterClass(refPvixNotHscDims) 

202 self.assertIsFormatter(refPvixNotHscDims_fmt) 

203 self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name()) 

204 

205 # Check that parameters are stored 

206 refParam = self.makeDatasetRef("paramtest", dimensions, sc, {"instrument": "DummyNotHSC", 

207 "physical_filter": "v"}, 

208 conform=False) 

209 lookup, refParam_fmt, kwargs = self.factory.getFormatterClassWithMatch(refParam) 

210 self.assertIn("writeParameters", kwargs) 

211 expected = {"max": 5, "min": 2, "comment": "Additional commentary", "recipe": "recipe1"} 

212 self.assertEqual(kwargs["writeParameters"], expected) 

213 self.assertIn("FormatterTest", refParam_fmt.name()) 

214 

215 f = self.factory.getFormatter(refParam, self.fileDescriptor) 

216 self.assertEqual(f.writeParameters, expected) 

217 

218 f = self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"min": 22, 

219 "extra": 50}) 

220 self.assertEqual(f.writeParameters, {"max": 5, "min": 22, "comment": "Additional commentary", 

221 "extra": 50, "recipe": "recipe1"}) 

222 

223 self.assertIn("recipe1", f.writeRecipes) 

224 self.assertEqual(f.writeParameters["recipe"], "recipe1") 

225 

226 with self.assertRaises(ValueError): 

227 # "new" is not allowed as a write parameter 

228 self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"new": 1}) 

229 

230 with self.assertRaises(RuntimeError): 

231 # "mode" is a required recipe parameter 

232 self.factory.getFormatter(refParam, self.fileDescriptor, writeRecipes={"recipe3": {"notmode": 1}}) 

233 

234 

235if __name__ == "__main__": 235 ↛ 236line 235 didn't jump to line 236, because the condition on line 235 was never true

236 unittest.main()