Coverage for python/lsst/verify/tasks/commonMetrics.py: 34%
84 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 03:00 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 03:00 -0700
1#
2# This file is part of verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Code for measuring metrics that apply to any Task.
25"""
27__all__ = ["TimingMetricConfig", "TimingMetricTask",
28 "CpuTimingMetricConfig", "CpuTimingMetricTask",
29 "MemoryMetricConfig", "MemoryMetricTask",
30 ]
32from datetime import datetime
33import resource
34import sys
36import astropy.units as u
38import lsst.pex.config as pexConfig
39from lsst.pipe.base import NoWorkFound
41from lsst.verify import Measurement, Datum
42from lsst.verify.tasks import MetricComputationError, MetadataMetricTask, \
43 MetadataMetricConfig
46class TimeMethodMetricConfig(MetadataMetricConfig):
47 """Common config fields for metrics based on
48 `~lsst.utils.timer.timeMethod`.
50 These fields let metrics distinguish between different methods that have
51 been decorated with `~lsst.utils.timer.timeMethod`.
52 """
53 target = pexConfig.Field(
54 dtype=str,
55 doc="The method to profile, optionally prefixed by one or more tasks "
56 "in the format of `lsst.pipe.base.Task.getFullMetadata()`.")
59# Expose TimingMetricConfig name because config-writers expect it
60TimingMetricConfig = TimeMethodMetricConfig
63class TimingMetricTask(MetadataMetricTask):
64 """A Task that computes a wall-clock time using metadata produced by the
65 `lsst.utils.timer.timeMethod` decorator.
67 Parameters
68 ----------
69 args
70 kwargs
71 Constructor parameters are the same as for
72 `lsst.verify.tasks.MetricTask`.
73 """
75 ConfigClass = TimingMetricConfig
76 _DefaultName = "timingMetric"
78 @classmethod
79 def getInputMetadataKeys(cls, config):
80 """Get search strings for the metadata.
82 Parameters
83 ----------
84 config : ``cls.ConfigClass``
85 Configuration for this task.
87 Returns
88 -------
89 keys : `dict`
90 A dictionary of keys, optionally prefixed by one or more tasks in
91 the format of `lsst.pipe.base.Task.getFullMetadata()`.
93 ``"StartTimestamp"``
94 The key for an ISO 8601-compliant text string where the target
95 method started (`str`).
96 ``"EndTimestamp"``
97 The key for an ISO 8601-compliant text string where the target
98 method ended (`str`).
99 """
100 keyBase = config.target
101 return {"StartTimestamp": keyBase + "StartUtc",
102 "EndTimestamp": keyBase + "EndUtc",
103 }
105 def makeMeasurement(self, timings):
106 """Compute a wall-clock measurement from metadata provided by
107 `lsst.utils.timer.timeMethod`.
109 Parameters
110 ----------
111 timings : `dict` [`str`, any]
112 A representation of the metadata passed to `run`. The `dict` has
113 the following keys:
115 ``"StartTimestamp"``
116 The time the target method started, in an ISO 8601-compliant
117 format (`str` or `None`).
118 ``"EndTimestamp"``
119 The time the target method ended, in an ISO 8601-compliant
120 format (`str` or `None`).
122 Returns
123 -------
124 measurement : `lsst.verify.Measurement`
125 The running time of the target method.
127 Raises
128 ------
129 lsst.verify.tasks.MetricComputationError
130 Raised if the timing metadata are invalid.
131 lsst.pipe.base.NoWorkFound
132 Raised if no matching timing metadata found.
133 """
134 # Use or, not and, so that unpaired keys raise MetricComputationError.
135 if timings["StartTimestamp"] is not None or timings["EndTimestamp"] is not None:
136 try:
137 startTime = datetime.fromisoformat(timings["StartTimestamp"])
138 endTime = datetime.fromisoformat(timings["EndTimestamp"])
139 except (TypeError, ValueError):
140 raise MetricComputationError("Invalid metadata")
141 else:
142 totalTime = (endTime - startTime).total_seconds()
143 meas = Measurement(self.config.metricName,
144 totalTime * u.second)
145 meas.notes["estimator"] = "utils.timer.timeMethod"
146 meas.extras["start"] = Datum(timings["StartTimestamp"])
147 meas.extras["end"] = Datum(timings["EndTimestamp"])
148 return meas
149 else:
150 raise NoWorkFound(f"Nothing to do: no timing information for {self.config.target} found.")
153# Expose CpuTimingMetricConfig name because config-writers expect it
154CpuTimingMetricConfig = TimeMethodMetricConfig
157class CpuTimingMetricTask(MetadataMetricTask):
158 """A Task that computes a CPU time using metadata produced by the
159 `lsst.utils.timer.timeMethod` decorator.
161 Parameters
162 ----------
163 args
164 kwargs
165 Constructor parameters are the same as for
166 `lsst.verify.tasks.MetricTask`.
167 """
169 ConfigClass = CpuTimingMetricConfig
170 _DefaultName = "cpuTimingMetric"
172 @classmethod
173 def getInputMetadataKeys(cls, config):
174 """Get search strings for the metadata.
176 Parameters
177 ----------
178 config : ``cls.ConfigClass``
179 Configuration for this task.
181 Returns
182 -------
183 keys : `dict`
184 A dictionary of keys, optionally prefixed by one or more tasks in
185 the format of `lsst.pipe.base.Task.getFullMetadata()`.
187 ``"StartTime"``
188 The key for when the target method started (`str`).
189 ``"EndTime"``
190 The key for when the target method ended (`str`).
191 ``"StartTimestamp"``
192 The key for an ISO 8601-compliant text string where the target
193 method started (`str`).
194 ``"EndTimestamp"``
195 The key for an ISO 8601-compliant text string where the target
196 method ended (`str`).
197 """
198 keyBase = config.target
199 return {"StartTime": keyBase + "StartCpuTime",
200 "EndTime": keyBase + "EndCpuTime",
201 "StartTimestamp": keyBase + "StartUtc",
202 "EndTimestamp": keyBase + "EndUtc",
203 }
205 def makeMeasurement(self, timings):
206 """Compute a wall-clock measurement from metadata provided by
207 `lsst.utils.timer.timeMethod`.
209 Parameters
210 ----------
211 timings : `dict` [`str`, any]
212 A representation of the metadata passed to `run`. The `dict` has
213 the following keys:
215 ``"StartTime"``
216 The time the target method started (`float` or `None`).
217 ``"EndTime"``
218 The time the target method ended (`float` or `None`).
219 ``"StartTimestamp"``, ``"EndTimestamp"``
220 The start and end timestamps, in an ISO 8601-compliant format
221 (`str` or `None`).
223 Returns
224 -------
225 measurement : `lsst.verify.Measurement`
226 The running time of the target method.
228 Raises
229 ------
230 lsst.verify.tasks.MetricComputationError
231 Raised if the timing metadata are invalid.
232 lsst.pipe.base.NoWorkFound
233 Raised if no matching timing metadata found.
234 """
235 # Use or, not and, so that unpaired keys raise MetricComputationError.
236 if timings["StartTime"] is not None or timings["EndTime"] is not None:
237 try:
238 totalTime = timings["EndTime"] - timings["StartTime"]
239 except TypeError:
240 raise MetricComputationError("Invalid metadata")
241 else:
242 meas = Measurement(self.config.metricName,
243 totalTime * u.second)
244 meas.notes["estimator"] = "utils.timer.timeMethod"
245 if timings["StartTimestamp"]:
246 meas.extras["start"] = Datum(timings["StartTimestamp"])
247 if timings["EndTimestamp"]:
248 meas.extras["end"] = Datum(timings["EndTimestamp"])
249 return meas
250 else:
251 raise NoWorkFound(f"Nothing to do: no timing information for {self.config.target} found.")
254# Expose MemoryMetricConfig name because config-writers expect it
255MemoryMetricConfig = TimeMethodMetricConfig
258class MemoryMetricTask(MetadataMetricTask):
259 """A Task that computes the maximum resident set size using metadata
260 produced by the `lsst.utils.timer.timeMethod` decorator.
262 Parameters
263 ----------
264 args
265 kwargs
266 Constructor parameters are the same as for
267 `lsst.verify.tasks.MetricTask`.
268 """
270 ConfigClass = MemoryMetricConfig
271 _DefaultName = "memoryMetric"
273 @classmethod
274 def getInputMetadataKeys(cls, config):
275 """Get search strings for the metadata.
277 Parameters
278 ----------
279 config : ``cls.ConfigClass``
280 Configuration for this task.
282 Returns
283 -------
284 keys : `dict`
285 A dictionary of keys, optionally prefixed by one or more tasks in
286 the format of `lsst.pipe.base.Task.getFullMetadata()`.
288 ``"EndMemory"``
289 The key for the memory usage at the end of the method (`str`).
290 ``"MetadataVersion"``
291 The key for the task-level metadata version.
292 """
293 keyBase = config.target
294 # Parse keyBase to get just the task prefix, if any; needed to
295 # guarantee that returned keys all point to unique entries.
296 # The following line returns a "."-terminated string if keyBase has a
297 # task prefix, and "" otherwise.
298 taskPrefix = "".join(keyBase.rpartition(".")[0:2])
300 return {"EndMemory": keyBase + "EndMaxResidentSetSize",
301 "MetadataVersion": taskPrefix + "__version__",
302 }
304 def makeMeasurement(self, memory):
305 """Compute a maximum resident set size measurement from metadata
306 provided by `lsst.utils.timer.timeMethod`.
308 Parameters
309 ----------
310 memory : `dict` [`str`, any]
311 A representation of the metadata passed to `run`. Each `dict` has
312 the following keys:
314 ``"EndMemory"``
315 The memory usage at the end of the method (`int` or `None`).
316 ``"MetadataVersion"``
317 The version of the task metadata in which the value was stored
318 (`int` or `None`). `None` is assumed to be version 0.
320 Returns
321 -------
322 measurement : `lsst.verify.Measurement`
323 The maximum memory usage of the target method.
325 Raises
326 ------
327 lsst.verify.tasks.MetricComputationError
328 Raised if the memory metadata are invalid.
329 lsst.pipe.base.NoWorkFound
330 Raised if no matching memory metadata found.
331 """
332 if memory["EndMemory"] is not None:
333 try:
334 maxMemory = int(memory["EndMemory"])
335 version = memory["MetadataVersion"] \
336 if memory["MetadataVersion"] else 0
337 except (ValueError, TypeError) as e:
338 raise MetricComputationError("Invalid metadata") from e
339 else:
340 meas = Measurement(self.config.metricName,
341 self._addUnits(maxMemory, version))
342 meas.notes['estimator'] = 'utils.timer.timeMethod'
343 return meas
344 else:
345 raise NoWorkFound(f"Nothing to do: no memory information for {self.config.target} found.")
347 def _addUnits(self, memory, version):
348 """Represent memory usage in correct units.
350 Parameters
351 ----------
352 memory : `int`
353 The memory usage as returned by `resource.getrusage`, in
354 platform-dependent units.
355 version : `int`
356 The metadata version. If ``0``, ``memory`` is in platform-dependent
357 units. If ``1`` or greater, ``memory`` is in bytes.
359 Returns
360 -------
361 memory : `astropy.units.Quantity`
362 The memory usage in absolute units.
363 """
364 if version >= 1:
365 return memory * u.byte
366 elif sys.platform.startswith('darwin'):
367 # MacOS uses bytes
368 return memory * u.byte
369 elif sys.platform.startswith('sunos') \
370 or sys.platform.startswith('solaris'):
371 # Solaris and SunOS use pages
372 return memory * resource.getpagesize() * u.byte
373 else:
374 # Assume Linux, which uses kibibytes
375 return memory * u.kibibyte