Coverage for python/lsst/verify/tasks/commonMetrics.py: 37%
77 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-24 09:24 +0000
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-24 09:24 +0000
1#
2# This file is part of verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Code for measuring metrics that apply to any Task.
25"""
27__all__ = ["TimingMetricConfig", "TimingMetricTask",
28 "MemoryMetricConfig", "MemoryMetricTask",
29 ]
31import resource
32import sys
33import warnings
35import astropy.units as u
37import lsst.pex.config as pexConfig
39from lsst.verify import Measurement, Datum
40from lsst.verify.gen2tasks.metricRegistry import registerMultiple
41from lsst.verify.tasks import MetricComputationError, MetadataMetricTask, \
42 MetadataMetricConfig
45class TimeMethodMetricConfig(MetadataMetricConfig):
46 """Common config fields for metrics based on
47 `~lsst.utils.timer.timeMethod`.
49 These fields let metrics distinguish between different methods that have
50 been decorated with `~lsst.utils.timer.timeMethod`.
51 """
52 target = pexConfig.Field(
53 dtype=str,
54 doc="The method to profile, optionally prefixed by one or more tasks "
55 "in the format of `lsst.pipe.base.Task.getFullMetadata()`.")
56 metric = pexConfig.Field(
57 dtype=str,
58 optional=True,
59 doc="The fully qualified name of the metric to store the "
60 "profiling information.",
61 deprecated="This field has been replaced by connections.package and "
62 "connections.metric. It will be removed along "
63 "with daf_persistence."
64 )
66 def validate(self):
67 super().validate()
69 if self.metric:
70 if self.metric != self.connections.package \
71 + "." + self.connections.metric:
72 warnings.warn(
73 "config.metric is deprecated; set connections.package "
74 "and connections.metric instead.",
75 FutureWarning)
76 try:
77 self.connections.package, self.connections.metric \
78 = self.metric.split(".")
79 except ValueError:
80 self.connections.package = ""
81 self.connections.metric = self.metric
84# Expose TimingMetricConfig name because config-writers expect it
85TimingMetricConfig = TimeMethodMetricConfig
88@registerMultiple("timing")
89class TimingMetricTask(MetadataMetricTask):
90 """A Task that computes a wall-clock time using metadata produced by the
91 `lsst.utils.timer.timeMethod` decorator.
93 Parameters
94 ----------
95 args
96 kwargs
97 Constructor parameters are the same as for
98 `lsst.verify.tasks.MetricTask`.
99 """
101 ConfigClass = TimingMetricConfig
102 _DefaultName = "timingMetric"
104 @classmethod
105 def getInputMetadataKeys(cls, config):
106 """Get search strings for the metadata.
108 Parameters
109 ----------
110 config : ``cls.ConfigClass``
111 Configuration for this task.
113 Returns
114 -------
115 keys : `dict`
116 A dictionary of keys, optionally prefixed by one or more tasks in
117 the format of `lsst.pipe.base.Task.getFullMetadata()`.
119 ``"StartTime"``
120 The key for when the target method started (`str`).
121 ``"EndTime"``
122 The key for when the target method ended (`str`).
123 ``"StartTimestamp"``
124 The key for an ISO 8601-compliant text string where the target
125 method started (`str`).
126 ``"EndTimestamp"``
127 The key for an ISO 8601-compliant text string where the target
128 method ended (`str`).
129 """
130 keyBase = config.target
131 return {"StartTime": keyBase + "StartCpuTime",
132 "EndTime": keyBase + "EndCpuTime",
133 "StartTimestamp": keyBase + "StartUtc",
134 "EndTimestamp": keyBase + "EndUtc",
135 }
137 def makeMeasurement(self, timings):
138 """Compute a wall-clock measurement from metadata provided by
139 `lsst.utils.timer.timeMethod`.
141 Parameters
142 ----------
143 timings : `dict` [`str`, any]
144 A representation of the metadata passed to `run`. The `dict` has
145 the following keys:
147 ``"StartTime"``
148 The time the target method started (`float` or `None`).
149 ``"EndTime"``
150 The time the target method ended (`float` or `None`).
151 ``"StartTimestamp"``, ``"EndTimestamp"``
152 The start and end timestamps, in an ISO 8601-compliant format
153 (`str` or `None`).
155 Returns
156 -------
157 measurement : `lsst.verify.Measurement` or `None`
158 The running time of the target method.
160 Raises
161 ------
162 MetricComputationError
163 Raised if the timing metadata are invalid.
164 """
165 if timings["StartTime"] is not None or timings["EndTime"] is not None:
166 try:
167 totalTime = timings["EndTime"] - timings["StartTime"]
168 except TypeError:
169 raise MetricComputationError("Invalid metadata")
170 else:
171 meas = Measurement(self.config.metricName,
172 totalTime * u.second)
173 meas.notes["estimator"] = "utils.timer.timeMethod"
174 if timings["StartTimestamp"]:
175 meas.extras["start"] = Datum(timings["StartTimestamp"])
176 if timings["EndTimestamp"]:
177 meas.extras["end"] = Datum(timings["EndTimestamp"])
178 return meas
179 else:
180 self.log.info("Nothing to do: no timing information for %s found.",
181 self.config.target)
182 return None
185# Expose MemoryMetricConfig name because config-writers expect it
186MemoryMetricConfig = TimeMethodMetricConfig
189@registerMultiple("memory")
190class MemoryMetricTask(MetadataMetricTask):
191 """A Task that computes the maximum resident set size using metadata
192 produced by the `lsst.utils.timer.timeMethod` decorator.
194 Parameters
195 ----------
196 args
197 kwargs
198 Constructor parameters are the same as for
199 `lsst.verify.tasks.MetricTask`.
200 """
202 ConfigClass = MemoryMetricConfig
203 _DefaultName = "memoryMetric"
205 @classmethod
206 def getInputMetadataKeys(cls, config):
207 """Get search strings for the metadata.
209 Parameters
210 ----------
211 config : ``cls.ConfigClass``
212 Configuration for this task.
214 Returns
215 -------
216 keys : `dict`
217 A dictionary of keys, optionally prefixed by one or more tasks in
218 the format of `lsst.pipe.base.Task.getFullMetadata()`.
220 ``"EndMemory"``
221 The key for the memory usage at the end of the method (`str`).
222 ``"MetadataVersion"``
223 The key for the task-level metadata version.
224 """
225 keyBase = config.target
226 # Parse keyBase to get just the task prefix, if any; needed to
227 # guarantee that returned keys all point to unique entries.
228 # The following line returns a "."-terminated string if keyBase has a
229 # task prefix, and "" otherwise.
230 taskPrefix = "".join(keyBase.rpartition(".")[0:2])
232 return {"EndMemory": keyBase + "EndMaxResidentSetSize",
233 "MetadataVersion": taskPrefix + "__version__",
234 }
236 def makeMeasurement(self, memory):
237 """Compute a maximum resident set size measurement from metadata
238 provided by `lsst.utils.timer.timeMethod`.
240 Parameters
241 ----------
242 memory : `dict` [`str`, any]
243 A representation of the metadata passed to `run`. Each `dict` has
244 the following keys:
246 ``"EndMemory"``
247 The memory usage at the end of the method (`int` or `None`).
248 ``"MetadataVersion"``
249 The version of the task metadata in which the value was stored
250 (`int` or `None`). `None` is assumed to be version 0.
252 Returns
253 -------
254 measurement : `lsst.verify.Measurement` or `None`
255 The maximum memory usage of the target method.
257 Raises
258 ------
259 MetricComputationError
260 Raised if the memory metadata are invalid.
261 """
262 if memory["EndMemory"] is not None:
263 try:
264 maxMemory = int(memory["EndMemory"])
265 version = memory["MetadataVersion"] \
266 if memory["MetadataVersion"] else 0
267 except (ValueError, TypeError) as e:
268 raise MetricComputationError("Invalid metadata") from e
269 else:
270 meas = Measurement(self.config.metricName,
271 self._addUnits(maxMemory, version))
272 meas.notes['estimator'] = 'utils.timer.timeMethod'
273 return meas
274 else:
275 self.log.info("Nothing to do: no memory information for %s found.",
276 self.config.target)
277 return None
279 def _addUnits(self, memory, version):
280 """Represent memory usage in correct units.
282 Parameters
283 ----------
284 memory : `int`
285 The memory usage as returned by `resource.getrusage`, in
286 platform-dependent units.
287 version : `int`
288 The metadata version. If ``0``, ``memory`` is in platform-dependent
289 units. If ``1`` or greater, ``memory`` is in bytes.
291 Returns
292 -------
293 memory : `astropy.units.Quantity`
294 The memory usage in absolute units.
295 """
296 if version >= 1:
297 return memory * u.byte
298 elif sys.platform.startswith('darwin'):
299 # MacOS uses bytes
300 return memory * u.byte
301 elif sys.platform.startswith('sunos') \
302 or sys.platform.startswith('solaris'):
303 # Solaris and SunOS use pages
304 return memory * resource.getpagesize() * u.byte
305 else:
306 # Assume Linux, which uses kibibytes
307 return memory * u.kibibyte