Coverage for python/lsst/verify/tasks/commonMetrics.py: 35%
61 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-27 02:03 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-27 02:03 -0800
1#
2# This file is part of verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Code for measuring metrics that apply to any Task.
25"""
27__all__ = ["TimingMetricConfig", "TimingMetricTask",
28 "MemoryMetricConfig", "MemoryMetricTask",
29 ]
31import resource
32import sys
34import astropy.units as u
36import lsst.pex.config as pexConfig
37from lsst.pipe.base import NoWorkFound
39from lsst.verify import Measurement, Datum
40from lsst.verify.tasks import MetricComputationError, MetadataMetricTask, \
41 MetadataMetricConfig
44class TimeMethodMetricConfig(MetadataMetricConfig):
45 """Common config fields for metrics based on
46 `~lsst.utils.timer.timeMethod`.
48 These fields let metrics distinguish between different methods that have
49 been decorated with `~lsst.utils.timer.timeMethod`.
50 """
51 target = pexConfig.Field(
52 dtype=str,
53 doc="The method to profile, optionally prefixed by one or more tasks "
54 "in the format of `lsst.pipe.base.Task.getFullMetadata()`.")
57# Expose TimingMetricConfig name because config-writers expect it
58TimingMetricConfig = TimeMethodMetricConfig
61class TimingMetricTask(MetadataMetricTask):
62 """A Task that computes a wall-clock time using metadata produced by the
63 `lsst.utils.timer.timeMethod` decorator.
65 Parameters
66 ----------
67 args
68 kwargs
69 Constructor parameters are the same as for
70 `lsst.verify.tasks.MetricTask`.
71 """
73 ConfigClass = TimingMetricConfig
74 _DefaultName = "timingMetric"
76 @classmethod
77 def getInputMetadataKeys(cls, config):
78 """Get search strings for the metadata.
80 Parameters
81 ----------
82 config : ``cls.ConfigClass``
83 Configuration for this task.
85 Returns
86 -------
87 keys : `dict`
88 A dictionary of keys, optionally prefixed by one or more tasks in
89 the format of `lsst.pipe.base.Task.getFullMetadata()`.
91 ``"StartTime"``
92 The key for when the target method started (`str`).
93 ``"EndTime"``
94 The key for when the target method ended (`str`).
95 ``"StartTimestamp"``
96 The key for an ISO 8601-compliant text string where the target
97 method started (`str`).
98 ``"EndTimestamp"``
99 The key for an ISO 8601-compliant text string where the target
100 method ended (`str`).
101 """
102 keyBase = config.target
103 return {"StartTime": keyBase + "StartCpuTime",
104 "EndTime": keyBase + "EndCpuTime",
105 "StartTimestamp": keyBase + "StartUtc",
106 "EndTimestamp": keyBase + "EndUtc",
107 }
109 def makeMeasurement(self, timings):
110 """Compute a wall-clock measurement from metadata provided by
111 `lsst.utils.timer.timeMethod`.
113 Parameters
114 ----------
115 timings : `dict` [`str`, any]
116 A representation of the metadata passed to `run`. The `dict` has
117 the following keys:
119 ``"StartTime"``
120 The time the target method started (`float` or `None`).
121 ``"EndTime"``
122 The time the target method ended (`float` or `None`).
123 ``"StartTimestamp"``, ``"EndTimestamp"``
124 The start and end timestamps, in an ISO 8601-compliant format
125 (`str` or `None`).
127 Returns
128 -------
129 measurement : `lsst.verify.Measurement`
130 The running time of the target method.
132 Raises
133 ------
134 lsst.verify.tasks.MetricComputationError
135 Raised if the timing metadata are invalid.
136 lsst.pipe.base.NoWorkFound
137 Raised if no matching timing metadata found.
138 """
139 if timings["StartTime"] is not None or timings["EndTime"] is not None:
140 try:
141 totalTime = timings["EndTime"] - timings["StartTime"]
142 except TypeError:
143 raise MetricComputationError("Invalid metadata")
144 else:
145 meas = Measurement(self.config.metricName,
146 totalTime * u.second)
147 meas.notes["estimator"] = "utils.timer.timeMethod"
148 if timings["StartTimestamp"]:
149 meas.extras["start"] = Datum(timings["StartTimestamp"])
150 if timings["EndTimestamp"]:
151 meas.extras["end"] = Datum(timings["EndTimestamp"])
152 return meas
153 else:
154 raise NoWorkFound(f"Nothing to do: no timing information for {self.config.target} found.")
157# Expose MemoryMetricConfig name because config-writers expect it
158MemoryMetricConfig = TimeMethodMetricConfig
161class MemoryMetricTask(MetadataMetricTask):
162 """A Task that computes the maximum resident set size using metadata
163 produced by the `lsst.utils.timer.timeMethod` decorator.
165 Parameters
166 ----------
167 args
168 kwargs
169 Constructor parameters are the same as for
170 `lsst.verify.tasks.MetricTask`.
171 """
173 ConfigClass = MemoryMetricConfig
174 _DefaultName = "memoryMetric"
176 @classmethod
177 def getInputMetadataKeys(cls, config):
178 """Get search strings for the metadata.
180 Parameters
181 ----------
182 config : ``cls.ConfigClass``
183 Configuration for this task.
185 Returns
186 -------
187 keys : `dict`
188 A dictionary of keys, optionally prefixed by one or more tasks in
189 the format of `lsst.pipe.base.Task.getFullMetadata()`.
191 ``"EndMemory"``
192 The key for the memory usage at the end of the method (`str`).
193 ``"MetadataVersion"``
194 The key for the task-level metadata version.
195 """
196 keyBase = config.target
197 # Parse keyBase to get just the task prefix, if any; needed to
198 # guarantee that returned keys all point to unique entries.
199 # The following line returns a "."-terminated string if keyBase has a
200 # task prefix, and "" otherwise.
201 taskPrefix = "".join(keyBase.rpartition(".")[0:2])
203 return {"EndMemory": keyBase + "EndMaxResidentSetSize",
204 "MetadataVersion": taskPrefix + "__version__",
205 }
207 def makeMeasurement(self, memory):
208 """Compute a maximum resident set size measurement from metadata
209 provided by `lsst.utils.timer.timeMethod`.
211 Parameters
212 ----------
213 memory : `dict` [`str`, any]
214 A representation of the metadata passed to `run`. Each `dict` has
215 the following keys:
217 ``"EndMemory"``
218 The memory usage at the end of the method (`int` or `None`).
219 ``"MetadataVersion"``
220 The version of the task metadata in which the value was stored
221 (`int` or `None`). `None` is assumed to be version 0.
223 Returns
224 -------
225 measurement : `lsst.verify.Measurement`
226 The maximum memory usage of the target method.
228 Raises
229 ------
230 lsst.verify.tasks.MetricComputationError
231 Raised if the memory metadata are invalid.
232 lsst.pipe.base.NoWorkFound
233 Raised if no matching memory metadata found.
234 """
235 if memory["EndMemory"] is not None:
236 try:
237 maxMemory = int(memory["EndMemory"])
238 version = memory["MetadataVersion"] \
239 if memory["MetadataVersion"] else 0
240 except (ValueError, TypeError) as e:
241 raise MetricComputationError("Invalid metadata") from e
242 else:
243 meas = Measurement(self.config.metricName,
244 self._addUnits(maxMemory, version))
245 meas.notes['estimator'] = 'utils.timer.timeMethod'
246 return meas
247 else:
248 raise NoWorkFound(f"Nothing to do: no memory information for {self.config.target} found.")
250 def _addUnits(self, memory, version):
251 """Represent memory usage in correct units.
253 Parameters
254 ----------
255 memory : `int`
256 The memory usage as returned by `resource.getrusage`, in
257 platform-dependent units.
258 version : `int`
259 The metadata version. If ``0``, ``memory`` is in platform-dependent
260 units. If ``1`` or greater, ``memory`` is in bytes.
262 Returns
263 -------
264 memory : `astropy.units.Quantity`
265 The memory usage in absolute units.
266 """
267 if version >= 1:
268 return memory * u.byte
269 elif sys.platform.startswith('darwin'):
270 # MacOS uses bytes
271 return memory * u.byte
272 elif sys.platform.startswith('sunos') \
273 or sys.platform.startswith('solaris'):
274 # Solaris and SunOS use pages
275 return memory * resource.getpagesize() * u.byte
276 else:
277 # Assume Linux, which uses kibibytes
278 return memory * u.kibibyte