Coverage for tests / test_logFormatter.py: 19%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Tests for ButlerLogRecordsFormatter.""" 

29 

30import json 

31import logging 

32import os 

33import sys 

34import tempfile 

35import unittest 

36from logging import FileHandler 

37 

38from lsst.daf.butler import Butler, DatasetRef, DatasetType, FileDataset 

39from lsst.daf.butler.logging import ( 

40 ButlerLogRecordHandler, 

41 ButlerLogRecords, 

42 JsonLogFormatter, 

43 _ButlerLogRecordsModelV1, 

44) 

45from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

46 

47TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

48 

49 

50class ButlerLogRecordsFormatterTestCase(unittest.TestCase): 

51 """Test for ButlerLogRecords put/get.""" 

52 

53 def setUp(self): 

54 self.root = makeTestTempDir(TESTDIR) 

55 Butler.makeRepo(self.root) 

56 

57 self.run = "testrun" 

58 self.butler = Butler.from_config(self.root, run=self.run) 

59 self.enterContext(self.butler) 

60 self.datasetType = DatasetType("test_logs", [], "ButlerLogRecords", universe=self.butler.dimensions) 

61 

62 self.butler.registry.registerDatasetType(self.datasetType) 

63 

64 def tearDown(self): 

65 removeTestTempDir(self.root) 

66 

67 def testButlerLogRecordsFormatter(self): 

68 """Test that we can get and put records handled by 

69 ButlerLogRecordHandler. 

70 """ 

71 handler = ButlerLogRecordHandler() 

72 

73 log = logging.getLogger(self.id()) 

74 log.setLevel(logging.INFO) 

75 log.addHandler(handler) 

76 

77 log.info("An INFO message") 

78 log.debug("A DEBUG message") 

79 log.warning("A WARNING message") 

80 

81 ref = self.butler.put(handler.records, self.datasetType) 

82 records = self.butler.get(ref) 

83 

84 self.assertEqual(records, handler.records) 

85 self.assertEqual(len(records), 2) 

86 

87 def testButlerLogRecordsFormatterExtra(self): 

88 """Test that we can get and put records handled by 

89 ButlerLogRecordHandler with extra JSON data attached. 

90 """ 

91 handler = ButlerLogRecordHandler() 

92 

93 log = logging.getLogger(self.id()) 

94 log.setLevel(logging.INFO) 

95 log.addHandler(handler) 

96 

97 log.info("An INFO message") 

98 log.debug("A DEBUG message") 

99 log.warning("A WARNING message") 

100 

101 extra_data = {"extra1": 1, "extra2": 2} 

102 handler.records.extra = extra_data 

103 ref = self.butler.put(handler.records, self.datasetType) 

104 records = self.butler.get(ref) 

105 

106 self.assertEqual(records, handler.records) 

107 self.assertEqual(len(records), 2) 

108 

109 def testButlerLogRecordsFormatterEmptyExtra(self): 

110 """Test that we can store extra information even with no log 

111 records written. 

112 """ 

113 handler = ButlerLogRecordHandler() 

114 

115 log = logging.getLogger(self.id()) 

116 log.setLevel(logging.INFO) 

117 log.addHandler(handler) 

118 

119 extra_data = {"extra1": 1, "extra2": 2} 

120 handler.records.extra = extra_data 

121 ref = self.butler.put(handler.records, self.datasetType) 

122 records = self.butler.get(ref) 

123 

124 self.assertEqual(records, handler.records) 

125 self.assertEqual(len(records), 0) 

126 

127 @unittest.skipIf( 

128 sys.version_info < (3, 12, 0), 

129 "This test requires NamedTemporaryFile behavior not available in older Python versions.", 

130 ) 

131 def testButlerLogRecordsFormatterV1(self): 

132 """Test that we can read log records stored via the old V1 format 

133 (just a JSON list). 

134 """ 

135 handler = ButlerLogRecordHandler() 

136 

137 log = logging.getLogger(self.id()) 

138 log.setLevel(logging.INFO) 

139 log.addHandler(handler) 

140 

141 log.info("An INFO message") 

142 log.debug("A DEBUG message") 

143 log.warning("A WARNING message") 

144 

145 with tempfile.NamedTemporaryFile( 

146 mode="w", suffix=".json", prefix="butler-log-", delete_on_close=False 

147 ) as tmp: 

148 # We can't use butler.put since that always writes the new format, 

149 # so we write manually and ingest. 

150 tmp.write( 

151 _ButlerLogRecordsModelV1(root=handler.records._records).model_dump_json( 

152 exclude_unset=True, exclude_defaults=True 

153 ) 

154 ) 

155 tmp.close() 

156 ref = DatasetRef(self.datasetType, dataId={}, run=self.run) 

157 dataset = FileDataset(path=os.path.abspath(tmp.name), refs=ref) 

158 self.butler.ingest(dataset, transfer="move") 

159 

160 records = self.butler.get(ref) 

161 self.assertEqual(records, handler.records) 

162 self.assertEqual(len(records), 2) 

163 

164 @unittest.skipIf( 

165 sys.version_info < (3, 12, 0), 

166 "This test requires NamedTemporaryFile behavior not available in older Python versions.", 

167 ) 

168 def testJsonLogRecordsFormatter(self): 

169 """Test that externally created JSON format stream files work.""" 

170 log = logging.getLogger(self.id()) 

171 log.setLevel(logging.INFO) 

172 

173 with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="butler-log-") as tmp: 

174 handler = FileHandler(tmp.name) 

175 handler.setFormatter(JsonLogFormatter()) 

176 log.addHandler(handler) 

177 

178 log.info("An INFO message") 

179 log.debug("A DEBUG message") 

180 log.warning("A WARNING message") 

181 

182 handler.close() 

183 

184 # Now ingest the file. 

185 ref = DatasetRef(self.datasetType, dataId={}, run=self.run) 

186 dataset = FileDataset(path=tmp.name, refs=ref) 

187 self.butler.ingest(dataset, transfer="move") 

188 

189 records = self.butler.get(ref) 

190 self.assertEqual(len(records), 2) 

191 

192 @unittest.skipIf( 

193 sys.version_info < (3, 12, 0), 

194 "This test requires NamedTemporaryFile behavior not available in older Python versions.", 

195 ) 

196 def testJsonLogRecordsFormatterExtra(self): 

197 """Test that externally created JSON format stream files work with 

198 extra JSON data appended. 

199 """ 

200 log = logging.getLogger(self.id()) 

201 log.setLevel(logging.INFO) 

202 

203 with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="butler-log-") as tmp: 

204 handler = FileHandler(tmp.name) 

205 handler.setFormatter(JsonLogFormatter()) 

206 log.addHandler(handler) 

207 

208 log.info("An INFO message") 

209 log.debug("A DEBUG message") 

210 log.warning("A WARNING message") 

211 

212 handler.close() 

213 extra_data = {"extra1": 1, "extra2": 2} 

214 with open(tmp.name, "a") as stream: 

215 ButlerLogRecords.write_streaming_extra(stream, json.dumps(extra_data)) 

216 

217 # Now ingest the file. 

218 ref = DatasetRef(self.datasetType, dataId={}, run=self.run) 

219 dataset = FileDataset(path=tmp.name, refs=ref) 

220 self.butler.ingest(dataset, transfer="move") 

221 

222 records: ButlerLogRecords = self.butler.get(ref) 

223 self.assertEqual(len(records), 2) 

224 self.assertEqual(records.extra, extra_data) 

225 

226 @unittest.skipIf( 

227 sys.version_info < (3, 12, 0), 

228 "This test requires NamedTemporaryFile behavior not available in older Python versions.", 

229 ) 

230 def testJsonLogRecordsFormatterExtraEmpty(self): 

231 """Test that externally created JSON format stream files work with 

232 extra JSON data appended even if no log messages written. 

233 """ 

234 log = logging.getLogger(self.id()) 

235 log.setLevel(logging.INFO) 

236 

237 with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="butler-log-") as tmp: 

238 handler = FileHandler(tmp.name) 

239 handler.setFormatter(JsonLogFormatter()) 

240 log.addHandler(handler) 

241 

242 handler.close() 

243 extra_data = {"extra1": 1, "extra2": 2} 

244 with open(tmp.name, "a") as stream: 

245 ButlerLogRecords.write_streaming_extra(stream, json.dumps(extra_data)) 

246 

247 # Now ingest the file. 

248 ref = DatasetRef(self.datasetType, dataId={}, run=self.run) 

249 dataset = FileDataset(path=tmp.name, refs=ref) 

250 self.butler.ingest(dataset, transfer="move") 

251 

252 records: ButlerLogRecords = self.butler.get(ref) 

253 self.assertEqual(len(records), 0) 

254 self.assertEqual(records.extra, extra_data) 

255 

256 

257if __name__ == "__main__": 

258 unittest.main()