Coverage for python/lsst/utils/usage.py: 65%

50 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-09 05:52 +0000

1# This file is part of utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Utilities for measuring resource consumption. 

13""" 

14 

15__all__ = ["get_current_mem_usage", "get_peak_mem_usage"] 

16 

17import dataclasses 

18import platform 

19import resource 

20import time 

21from typing import Dict, Tuple, Union 

22 

23import astropy.units as u 

24import psutil 

25 

26 

27def _get_rusage_multiplier() -> int: 

28 """Return the multiplier to use for memory usage returned by getrusage. 

29 

30 Returns 

31 ------- 

32 unit : `int` 

33 The multiplier that should be applied to the memory usage numbers 

34 returned by `resource.getrusage` to convert them to bytes. 

35 """ 

36 system = platform.system().lower() 

37 if system == "darwin": 37 ↛ 39line 37 didn't jump to line 39, because the condition on line 37 was never true

38 # MacOS uses bytes 

39 return 1 

40 elif "solaris" in system or "sunos" in system: 40 ↛ 42line 40 didn't jump to line 42, because the condition on line 40 was never true

41 # Solaris and SunOS use pages 

42 return resource.getpagesize() 

43 else: 

44 # Assume Linux/FreeBSD etc, which use kibibytes 

45 return 1024 

46 

47 

48_RUSAGE_MEMORY_MULTIPLIER = _get_rusage_multiplier() 

49 

50 

51def get_current_mem_usage() -> Tuple[u.Quantity, u.Quantity]: 

52 """Report current memory usage. 

53 

54 Returns 

55 ------- 

56 usage_main : `astropy.units.Quantity` 

57 Current memory usage of the calling process expressed in bytes. 

58 usage_child : `astropy.units.Quantity` 

59 Current memory usage of the child processes (zero if there are none) 

60 expressed in bytes. 

61 

62 Notes 

63 ----- 

64 Function reports current memory usage using resident set size as a proxy. 

65 As such the values it reports are capped at available physical RAM and may 

66 not reflect the actual memory allocated to the process and its children. 

67 """ 

68 proc = psutil.Process() 

69 with proc.oneshot(): 

70 usage_main = proc.memory_info().rss * u.byte 

71 usage_child = sum([child.memory_info().rss for child in proc.children()]) * u.byte 

72 return usage_main, usage_child 

73 

74 

75def get_peak_mem_usage() -> Tuple[u.Quantity, u.Quantity]: 

76 """Report peak memory usage. 

77 

78 Returns 

79 ------- 

80 peak_main: `astropy.units.Quantity` 

81 Peak memory usage (maximum resident set size) of the calling process. 

82 peak_child: `astropy.units.Quantity` 

83 Peak memory usage (resident set size) of the largest child process. 

84 

85 Notes 

86 ----- 

87 Function reports peak memory usage using the maximum resident set size as 

88 a proxy. As such the value it reports is capped at available physical RAM 

89 and may not reflect the actual maximal value. 

90 """ 

91 peak_main = _get_current_rusage().maxResidentSetSize * u.byte 

92 peak_child = _get_current_rusage(for_children=True).maxResidentSetSize * u.byte 

93 return peak_main, peak_child 

94 

95 

96@dataclasses.dataclass(frozen=True) 

97class _UsageInfo: 

98 """Summary of process usage.""" 

99 

100 cpuTime: float 

101 """CPU time in seconds.""" 

102 userTime: float 

103 """User time in seconds.""" 

104 systemTime: float 

105 """System time in seconds.""" 

106 maxResidentSetSize: int 

107 """Maximum resident set size in bytes.""" 

108 minorPageFaults: int 

109 majorPageFaults: int 

110 blockInputs: int 

111 blockOutputs: int 

112 voluntaryContextSwitches: int 

113 involuntaryContextSwitches: int 

114 

115 def dict(self) -> Dict[str, Union[float, int]]: 

116 return dataclasses.asdict(self) 

117 

118 

119def _get_current_rusage(for_children: bool = False) -> _UsageInfo: 

120 """Get information about this (or the child) process. 

121 

122 Parameters 

123 ---------- 

124 for_children : `bool`, optional 

125 Whether the information should be requested for child processes. 

126 Default is for the current process. 

127 

128 Returns 

129 ------- 

130 info : `_UsageInfo` 

131 The information obtained from the process. 

132 """ 

133 who = resource.RUSAGE_CHILDREN if for_children else resource.RUSAGE_SELF 

134 res = resource.getrusage(who) 

135 

136 # Convert the memory usage to bytes. 

137 max_rss = res.ru_maxrss * _RUSAGE_MEMORY_MULTIPLIER 

138 

139 return _UsageInfo( 

140 cpuTime=time.process_time(), 

141 userTime=res.ru_utime, 

142 systemTime=res.ru_stime, 

143 maxResidentSetSize=max_rss, 

144 minorPageFaults=res.ru_minflt, 

145 majorPageFaults=res.ru_majflt, 

146 blockInputs=res.ru_inblock, 

147 blockOutputs=res.ru_oublock, 

148 voluntaryContextSwitches=res.ru_nvcsw, 

149 involuntaryContextSwitches=res.ru_nivcsw, 

150 )