Coverage for python/lsst/verify/job.py: 26%
97 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-15 02:11 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-15 02:11 -0800
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21__all__ = ['Job']
23import json
24import os
26from .blobset import BlobSet
27from .jobmetadata import Metadata
28from .jsonmixin import JsonSerializationMixin
29from .measurementset import MeasurementSet
30from .metricset import MetricSet
31from .specset import SpecificationSet
32from . import squash
35class Job(JsonSerializationMixin):
36 r"""Container for `~lsst.verify.Measurement`\ s, `~lsst.verify.Blob` \s,
37 and `~lsst.verify.Metadata` associated with a pipeline run.
39 Parameters
40 ----------
41 measurements : `MeasurementSet` or `list` of `Measurement`\ s, optional
42 `Measurement`\ s to report in the Job.
43 metrics : `list` of `Metric`\ s or a `MetricSet`, optional
44 Optional list of `Metric`\ s, or a `MetricSet`.
45 specs : `SpecificationSet` or `list` of `Specification`\ s, optional
46 Optional specification information.
47 meta : `dict`, optional
48 Optional dictionary of metadata key-value entries.
49 """
51 def __init__(self, measurements=None, metrics=None, specs=None,
52 meta=None):
53 if isinstance(measurements, MeasurementSet):
54 self._meas_set = measurements
55 else:
56 self._meas_set = MeasurementSet(measurements)
58 if isinstance(metrics, MetricSet):
59 self._metric_set = metrics
60 else:
61 self._metric_set = MetricSet(metrics)
63 if isinstance(specs, SpecificationSet):
64 self._spec_set = specs
65 else:
66 self._spec_set = SpecificationSet(specs)
68 # Create metadata last so it has access to the measurement set
69 self._meta = Metadata(self._meas_set, data=meta)
71 @classmethod
72 def load_metrics_package(cls, package_name_or_path='verify_metrics',
73 subset=None, measurements=None, meta=None):
74 r"""Create a Job with metrics and specifications pre-loaded from a
75 Verification Framework metrics package, such as
76 :ref:`verify_metrics <verify-metrics-package>`.
78 Parameters
79 ----------
80 package_name_or_path : `str`, optional
81 Name of an EUPS package that hosts metric and specification
82 definition YAML files **or** the file path to a metrics package.
83 ``'verify_metrics'`` is the default package, and is where metrics
84 and specifications are defined for most LSST Science Pipelines
85 packages.
86 subset : `str`, optional
87 If set, only metrics and specification for this package are loaded.
88 For example, if ``subset='validate_drp'``, only ``validate_drp``
89 metrics are loaded. This argument is equivalent to the
90 `MetricSet.subset` method. Default is `None`.
91 measurements : `MeasurementSet` or `list` of `Measurement`\ s, optional
92 Measurements to report in the Job.
93 meta : `dict`, optional
94 Optional dictionary of metadata key-value entries to include
95 in the Job.
97 Returns
98 -------
99 job : `Job`
100 `Job` instance.
101 """
102 metrics = MetricSet.load_metrics_package(
103 package_name_or_path=package_name_or_path,
104 subset=subset)
105 specs = SpecificationSet.load_metrics_package(
106 package_name_or_path=package_name_or_path,
107 subset=subset)
108 instance = cls(measurements=measurements, metrics=metrics, specs=specs,
109 meta=meta)
110 return instance
112 @classmethod
113 def deserialize(cls, measurements=None, blobs=None,
114 metrics=None, specs=None, meta=None):
115 """Deserialize a Verification Framework Job from a JSON serialization.
117 Parameters
118 ----------
119 measurements : `list`, optional
120 List of serialized `Measurement` objects.
121 blobs : `list`, optional
122 List of serialized `Blob` objects.
123 metrics : `list`, optional
124 List of serialized `Metric` objects.
125 specs : `list`, optional
126 List of serialized specification objects.
127 meta : `dict`, optional
128 Dictionary of key-value metadata entries.
130 Returns
131 -------
132 job : `Job`
133 `Job` instance built from serialized data.
135 Examples
136 --------
137 Together, `Job.json` and `Job.deserialize` allow a verification job to
138 be serialized and later re-instantiated.
140 >>> import json
141 >>> job = Job()
142 >>> json_str = json.dumps(job.json)
143 >>> json_obj = json.loads(json_str)
144 >>> new_job = Job.deserialize(**json_obj)
145 """
146 blob_set = BlobSet.deserialize(blobs)
147 metric_set = MetricSet.deserialize(metrics)
148 spec_set = SpecificationSet.deserialize(specs)
149 meas_set = MeasurementSet.deserialize(
150 measurements=measurements,
151 blob_set=blob_set,
152 metric_set=metric_set)
154 instance = cls(measurements=meas_set,
155 metrics=metric_set,
156 specs=spec_set,
157 meta=meta)
158 return instance
160 @property
161 def measurements(self):
162 """Measurements associated with the pipeline verification job
163 (`MeasurementSet`).
164 """
165 return self._meas_set
167 @property
168 def metrics(self):
169 """Metrics associated with the pipeline verification job (`MetricSet`).
170 """
171 return self._metric_set
173 @property
174 def specs(self):
175 """Specifications associated with the pipeline verifification job
176 (`SpecificationSet`).
177 """
178 return self._spec_set
180 @property
181 def meta(self):
182 """Metadata mapping (`Metadata`)."""
183 return self._meta
185 @property
186 def json(self):
187 """`Job` data as a JSON-serialiable `dict`."""
188 # Gather blobs from all measurements
189 blob_set = BlobSet()
190 for name, measurement in self._meas_set.items():
191 for blob_name, blob in measurement.blobs.items():
192 if (str(name) == blob_name) and (len(blob) == 0):
193 # Don't serialize empty 'extras' blobs
194 continue
195 blob_set.insert(blob)
197 doc = JsonSerializationMixin.jsonify_dict({
198 'measurements': self._meas_set,
199 'blobs': blob_set,
200 'metrics': self._metric_set,
201 'specs': self._spec_set,
202 'meta': self._meta
203 })
204 return doc
206 def __eq__(self, other):
207 if self.measurements != other.measurements:
208 return False
210 if self.metrics != other.metrics:
211 return False
213 if self.specs != other.specs:
214 return False
216 if self.meta != other.meta:
217 return False
219 return True
221 def __ne__(self, other):
222 return not self.__eq__(other)
224 def __iadd__(self, other):
225 """Merge another Job into this one.
227 Parameters
228 ----------
229 other : `Job`
230 Job instance to be merged into this one.
232 Returns
233 -------
234 self : `Job`
235 This `Job` instance.
236 """
237 self.measurements.update(other.measurements)
238 self.metrics.update(other.metrics)
239 self.specs.update(other.specs)
240 self.meta.update(other.meta)
241 return self
243 def reload_metrics_package(self, package_name_or_path='verify_metrics',
244 subset=None):
245 """Load a metrics package and add metric and specification definitions
246 to the Job, as well as the collected measurements.
248 Parameters
249 ----------
250 package_name_or_path : `str`, optional
251 Name of an EUPS package that hosts metric and specification
252 definition YAML files **or** the file path to a metrics package.
253 ``'verify_metrics'`` is the default package, and is where metrics
254 and specifications are defined for most packages.
255 subset : `str`, optional
256 If set, only metrics and specification for this package are loaded.
257 For example, if ``subset='validate_drp'``, only ``validate_drp``
258 metrics are included in the `MetricSet`. This argument is
259 equivalent to the `MetricSet.subset` method. Default is `None`.
261 Notes
262 -----
263 This method is useful for loading metric and specification definitions
264 into a job that was created without this information. In addition
265 to being added to `Job.metrics`, metrics are also attached to
266 `Job.measurements` items. This ensures that measurement values are
267 normalized into the units of the metric definition when a Job is
268 serialized.
270 See also
271 --------
272 lsst.verify.MeasurementSet.refresh_metrics
273 """
274 metrics = MetricSet.load_metrics_package(
275 package_name_or_path=package_name_or_path,
276 subset=subset)
277 specs = SpecificationSet.load_metrics_package(
278 package_name_or_path=package_name_or_path,
279 subset=subset)
281 self.metrics.update(metrics)
282 self.specs.update(specs)
284 # Insert mertics into measurements
285 self.measurements.refresh_metrics(metrics)
287 def write(self, filename):
288 """Write a JSON serialization to the filesystem.
290 Parameters
291 ----------
292 filename : `str`
293 Name of the JSON file (including directories). This name
294 should be unique among all task executions in a pipeline. The
295 recommended extension is ``'.verify.json'``. This convention is
296 used by post-processing tools to discover verification framework
297 outputs.
298 """
299 dirname = os.path.dirname(filename)
300 if len(dirname) > 0:
301 if not os.path.isdir(dirname):
302 os.makedirs(dirname)
304 with open(filename, 'w') as f:
305 json.dump(self.json, f, indent=2, sort_keys=True)
307 def dispatch(self, api_user=None, api_password=None,
308 api_url='https://squash-restful-api.lsst.codes',
309 **kwargs):
310 """POST the job to SQUASH, LSST Data Management's metric dashboard.
312 Parameters
313 ----------
314 api_url : `str`, optional
315 Root URL of the SQUASH API server.
316 api_user : `str`, optional
317 API username.
318 api_password : `str`, optional
319 API password.
320 **kwargs : optional
321 Additional keyword arguments passed to `lsst.verify.squash.post`.
323 Returns
324 -------
325 output : response
326 The response from the POST request to the SQuaSH API
327 """
328 full_json_doc = self.json
329 # subset JSON to just the 'job' fields; no metrics and specs
330 job_json = {k: full_json_doc[k]
331 for k in ('measurements', 'blobs', 'meta')}
333 access_token = squash.get_access_token(api_url, api_user,
334 api_password)
336 return squash.post(api_url, 'job', json_doc=job_json,
337 access_token=access_token, **kwargs)
339 def report(self, name=None, spec_tags=None, metric_tags=None):
340 r"""Create a verification report that lists the pass/fail status of
341 measurements against specifications in this job.
343 In a Jupyter notebook, this report can be shown as an inline table.
345 Parameters
346 ----------
347 name : `str` or `lsst.verify.Name`, optional
348 A package or metric name to subset specifications by. When set,
349 only measurement and specification combinations belonging to that
350 package or metric are included in the report.
351 spec_tags : sequence of `str`, optional
352 A set of specification tag strings. when given, only
353 specifications that have all the given tags are included in the
354 report. For example, ``spec_tags=['LPM-17', 'minimum']``.
355 metric_tags : sequence of `str`, optional
356 A set of metric tag strings. When given, only specifications
357 belonging to metrics that posess **all** given tags are included
358 in the report. For example,
359 ``metric_tags=['LPM-17', 'photometry']`` selects sepifications
360 that have both the ``'LPM-17'`` and ``'photometry'`` tags.
362 Returns
363 -------
364 report : `lsst.verify.Report`
365 Report instance. In a Jupyter notebook, you can view the report
366 by calling `Report.show`.
368 See also
369 --------
370 lsst.verify.SpecificationSet.report
372 Notes
373 -----
374 This method uses the `lsst.verify.SpecificationSet.report` API to
375 create the `lsst.verify.Report`, automatically inserting the `Job`\ 's
376 measurements and metadata for filtering specifiation tests.
378 In a Jupyter notebook environment, use the `lsst.verify.Report.show`
379 method to view an interactive HTML table.
381 .. code-block:: python
383 job = lsst.verify.Job()
384 # ...
385 report = job.report()
386 report.show()
387 """
388 report = self.specs.report(self.measurements, meta=self.meta,
389 name=name, metric_tags=metric_tags,
390 spec_tags=spec_tags, metrics=self.metrics)
391 return report