Coverage for python / lsst / daf / butler / _butler_metrics.py: 61%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30from collections.abc import Callable, Iterator 

31from contextlib import contextmanager 

32from typing import Concatenate, ParamSpec 

33 

34from pydantic import BaseModel 

35 

36from lsst.utils.logging import LsstLoggers 

37from lsst.utils.timer import time_this 

38 

39P = ParamSpec("P") 

40 

41 

42class ButlerMetrics(BaseModel): 

43 """Metrics collected during Butler operations.""" 

44 

45 time_in_put: float = 0.0 

46 """Wall-clock time, in seconds, spent in put().""" 

47 

48 time_in_get: float = 0.0 

49 """Wall-clock time, in seconds, spent in get().""" 

50 

51 time_in_ingest: float = 0.0 

52 """Wall-clock time, in seconds, spent in ingest().""" 

53 

54 n_get: int = 0 

55 """Number of datasets retrieved with get().""" 

56 

57 n_put: int = 0 

58 """Number of datasets stored with put().""" 

59 

60 n_ingest: int = 0 

61 """Number of datasets ingested.""" 

62 

63 def reset(self) -> None: 

64 """Reset all metrics.""" 

65 self.time_in_put = 0.0 

66 self.time_in_get = 0.0 

67 self.time_in_ingest = 0.0 

68 self.n_get = 0 

69 self.n_put = 0 

70 self.n_ingest = 0 

71 

72 def increment_get(self, duration: float) -> None: 

73 """Increment time for get(). 

74 

75 Parameters 

76 ---------- 

77 duration : `float` 

78 Duration to add to the get() statistics. 

79 """ 

80 self.time_in_get += duration 

81 self.n_get += 1 

82 

83 def increment_put(self, duration: float) -> None: 

84 """Increment time for put(). 

85 

86 Parameters 

87 ---------- 

88 duration : `float` 

89 Duration to add to the put() statistics. 

90 """ 

91 self.time_in_put += duration 

92 self.n_put += 1 

93 

94 def increment_ingest(self, duration: float, n_datasets: int) -> None: 

95 """Increment time and datasets for ingest(). 

96 

97 Parameters 

98 ---------- 

99 duration : `float` 

100 Duration to add to the ingest() statistics. 

101 n_datasets : `int` 

102 Number of datasets to be ingested for this call. 

103 """ 

104 self.time_in_ingest += duration 

105 self.n_ingest += n_datasets 

106 

107 @contextmanager 

108 def _timer( 

109 self, 

110 handler: Callable[Concatenate[float, P], None], 

111 log: LsstLoggers | None = None, 

112 msg: str | None = None, 

113 *args: P.args, 

114 **kwargs: P.kwargs, 

115 ) -> Iterator[None]: 

116 with time_this(log=log, msg=msg) as timer: 

117 yield 

118 handler(timer.duration, *args, **kwargs) 

119 

120 @contextmanager 

121 def instrument_get(self, log: LsstLoggers | None = None, msg: str | None = None) -> Iterator[None]: 

122 """Run code and increment get statistics. 

123 

124 Parameters 

125 ---------- 

126 log : `logging.Logger` or `None` 

127 Logger to use for any timing information. 

128 msg : `str` or `None` 

129 Any message to be included in log output. 

130 """ 

131 with self._timer(self.increment_get, log=log, msg=msg): 

132 yield 

133 

134 @contextmanager 

135 def instrument_put(self, log: LsstLoggers | None = None, msg: str | None = None) -> Iterator[None]: 

136 """Run code and increment put statistics. 

137 

138 Parameters 

139 ---------- 

140 log : `logging.Logger` or `None` 

141 Logger to use for any timing information. 

142 msg : `str` or `None` 

143 Any message to be included in log output. 

144 """ 

145 with self._timer(self.increment_put, log=log, msg=msg): 

146 yield 

147 

148 @contextmanager 

149 def instrument_ingest( 

150 self, n_datasets: int, log: LsstLoggers | None = None, msg: str | None = None 

151 ) -> Iterator[None]: 

152 """Run code and increment ingest statistics. 

153 

154 Parameters 

155 ---------- 

156 n_datasets : `int` 

157 Number of datasets being ingested. 

158 log : `logging.Logger` or `None` 

159 Logger to use for any timing information. 

160 msg : `str` or `None` 

161 Any message to be included in log output. 

162 """ 

163 with self._timer(self.increment_ingest, n_datasets=n_datasets, log=log, msg=msg): 

164 yield