Coverage for python/lsst/verify/job.py: 28%

97 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-07-09 06:23 -0700

1# This file is part of verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21__all__ = ['Job'] 

22 

23import json 

24import os 

25 

26from .blobset import BlobSet 

27from .jobmetadata import Metadata 

28from .jsonmixin import JsonSerializationMixin 

29from .measurementset import MeasurementSet 

30from .metricset import MetricSet 

31from .specset import SpecificationSet 

32from . import squash 

33 

34 

35class Job(JsonSerializationMixin): 

36 r"""Container for `~lsst.verify.Measurement`\ s, `~lsst.verify.Blob` \s, 

37 and `~lsst.verify.Metadata` associated with a pipeline run. 

38 

39 Parameters 

40 ---------- 

41 measurements : `MeasurementSet` or `list` of `Measurement`\ s, optional 

42 `Measurement`\ s to report in the Job. 

43 metrics : `list` of `Metric`\ s or a `MetricSet`, optional 

44 Optional list of `Metric`\ s, or a `MetricSet`. 

45 specs : `SpecificationSet` or `list` of `Specification`\ s, optional 

46 Optional specification information. 

47 meta : `dict`, optional 

48 Optional dictionary of metadata key-value entries. 

49 """ 

50 

51 def __init__(self, measurements=None, metrics=None, specs=None, 

52 meta=None): 

53 if isinstance(measurements, MeasurementSet): 

54 self._meas_set = measurements 

55 else: 

56 self._meas_set = MeasurementSet(measurements) 

57 

58 if isinstance(metrics, MetricSet): 

59 self._metric_set = metrics 

60 else: 

61 self._metric_set = MetricSet(metrics) 

62 

63 if isinstance(specs, SpecificationSet): 

64 self._spec_set = specs 

65 else: 

66 self._spec_set = SpecificationSet(specs) 

67 

68 # Create metadata last so it has access to the measurement set 

69 self._meta = Metadata(self._meas_set, data=meta) 

70 

71 @classmethod 

72 def load_metrics_package(cls, package_name_or_path='verify_metrics', 

73 subset=None, measurements=None, meta=None): 

74 r"""Create a Job with metrics and specifications pre-loaded from a 

75 Verification Framework metrics package, such as 

76 :ref:`verify_metrics <verify-metrics-package>`. 

77 

78 Parameters 

79 ---------- 

80 package_name_or_path : `str`, optional 

81 Name of an EUPS package that hosts metric and specification 

82 definition YAML files **or** the file path to a metrics package. 

83 ``'verify_metrics'`` is the default package, and is where metrics 

84 and specifications are defined for most LSST Science Pipelines 

85 packages. 

86 subset : `str`, optional 

87 If set, only metrics and specification for this package are loaded. 

88 For example, if ``subset='validate_drp'``, only ``validate_drp`` 

89 metrics are loaded. This argument is equivalent to the 

90 `MetricSet.subset` method. Default is `None`. 

91 measurements : `MeasurementSet` or `list` of `Measurement`\ s, optional 

92 Measurements to report in the Job. 

93 meta : `dict`, optional 

94 Optional dictionary of metadata key-value entries to include 

95 in the Job. 

96 

97 Returns 

98 ------- 

99 job : `Job` 

100 `Job` instance. 

101 """ 

102 metrics = MetricSet.load_metrics_package( 

103 package_name_or_path=package_name_or_path, 

104 subset=subset) 

105 specs = SpecificationSet.load_metrics_package( 

106 package_name_or_path=package_name_or_path, 

107 subset=subset) 

108 instance = cls(measurements=measurements, metrics=metrics, specs=specs, 

109 meta=meta) 

110 return instance 

111 

112 @classmethod 

113 def deserialize(cls, measurements=None, blobs=None, 

114 metrics=None, specs=None, meta=None): 

115 """Deserialize a Verification Framework Job from a JSON serialization. 

116 

117 Parameters 

118 ---------- 

119 measurements : `list`, optional 

120 List of serialized `Measurement` objects. 

121 blobs : `list`, optional 

122 List of serialized `Blob` objects. 

123 metrics : `list`, optional 

124 List of serialized `Metric` objects. 

125 specs : `list`, optional 

126 List of serialized specification objects. 

127 meta : `dict`, optional 

128 Dictionary of key-value metadata entries. 

129 

130 Returns 

131 ------- 

132 job : `Job` 

133 `Job` instance built from serialized data. 

134 

135 Examples 

136 -------- 

137 Together, `Job.json` and `Job.deserialize` allow a verification job to 

138 be serialized and later re-instantiated. 

139 

140 >>> import json 

141 >>> job = Job() 

142 >>> json_str = json.dumps(job.json) 

143 >>> json_obj = json.loads(json_str) 

144 >>> new_job = Job.deserialize(**json_obj) 

145 """ 

146 blob_set = BlobSet.deserialize(blobs) 

147 metric_set = MetricSet.deserialize(metrics) 

148 spec_set = SpecificationSet.deserialize(specs) 

149 meas_set = MeasurementSet.deserialize( 

150 measurements=measurements, 

151 blob_set=blob_set, 

152 metric_set=metric_set) 

153 

154 instance = cls(measurements=meas_set, 

155 metrics=metric_set, 

156 specs=spec_set, 

157 meta=meta) 

158 return instance 

159 

160 @property 

161 def measurements(self): 

162 """Measurements associated with the pipeline verification job 

163 (`MeasurementSet`). 

164 """ 

165 return self._meas_set 

166 

167 @property 

168 def metrics(self): 

169 """Metrics associated with the pipeline verification job (`MetricSet`). 

170 """ 

171 return self._metric_set 

172 

173 @property 

174 def specs(self): 

175 """Specifications associated with the pipeline verifification job 

176 (`SpecificationSet`). 

177 """ 

178 return self._spec_set 

179 

180 @property 

181 def meta(self): 

182 """Metadata mapping (`Metadata`).""" 

183 return self._meta 

184 

185 @property 

186 def json(self): 

187 """`Job` data as a JSON-serialiable `dict`.""" 

188 # Gather blobs from all measurements 

189 blob_set = BlobSet() 

190 for name, measurement in self._meas_set.items(): 

191 for blob_name, blob in measurement.blobs.items(): 

192 if (str(name) == blob_name) and (len(blob) == 0): 

193 # Don't serialize empty 'extras' blobs 

194 continue 

195 blob_set.insert(blob) 

196 

197 doc = JsonSerializationMixin.jsonify_dict({ 

198 'measurements': self._meas_set, 

199 'blobs': blob_set, 

200 'metrics': self._metric_set, 

201 'specs': self._spec_set, 

202 'meta': self._meta 

203 }) 

204 return doc 

205 

206 def __eq__(self, other): 

207 if self.measurements != other.measurements: 

208 return False 

209 

210 if self.metrics != other.metrics: 

211 return False 

212 

213 if self.specs != other.specs: 

214 return False 

215 

216 if self.meta != other.meta: 

217 return False 

218 

219 return True 

220 

221 def __ne__(self, other): 

222 return not self.__eq__(other) 

223 

224 def __iadd__(self, other): 

225 """Merge another Job into this one. 

226 

227 Parameters 

228 ---------- 

229 other : `Job` 

230 Job instance to be merged into this one. 

231 

232 Returns 

233 ------- 

234 self : `Job` 

235 This `Job` instance. 

236 """ 

237 self.measurements.update(other.measurements) 

238 self.metrics.update(other.metrics) 

239 self.specs.update(other.specs) 

240 self.meta.update(other.meta) 

241 return self 

242 

243 def reload_metrics_package(self, package_name_or_path='verify_metrics', 

244 subset=None): 

245 """Load a metrics package and add metric and specification definitions 

246 to the Job, as well as the collected measurements. 

247 

248 Parameters 

249 ---------- 

250 package_name_or_path : `str`, optional 

251 Name of an EUPS package that hosts metric and specification 

252 definition YAML files **or** the file path to a metrics package. 

253 ``'verify_metrics'`` is the default package, and is where metrics 

254 and specifications are defined for most packages. 

255 subset : `str`, optional 

256 If set, only metrics and specification for this package are loaded. 

257 For example, if ``subset='validate_drp'``, only ``validate_drp`` 

258 metrics are included in the `MetricSet`. This argument is 

259 equivalent to the `MetricSet.subset` method. Default is `None`. 

260 

261 Notes 

262 ----- 

263 This method is useful for loading metric and specification definitions 

264 into a job that was created without this information. In addition 

265 to being added to `Job.metrics`, metrics are also attached to 

266 `Job.measurements` items. This ensures that measurement values are 

267 normalized into the units of the metric definition when a Job is 

268 serialized. 

269 

270 See also 

271 -------- 

272 lsst.verify.MeasurementSet.refresh_metrics 

273 """ 

274 metrics = MetricSet.load_metrics_package( 

275 package_name_or_path=package_name_or_path, 

276 subset=subset) 

277 specs = SpecificationSet.load_metrics_package( 

278 package_name_or_path=package_name_or_path, 

279 subset=subset) 

280 

281 self.metrics.update(metrics) 

282 self.specs.update(specs) 

283 

284 # Insert mertics into measurements 

285 self.measurements.refresh_metrics(metrics) 

286 

287 def write(self, filename): 

288 """Write a JSON serialization to the filesystem. 

289 

290 Parameters 

291 ---------- 

292 filename : `str` 

293 Name of the JSON file (including directories). This name 

294 should be unique among all task executions in a pipeline. The 

295 recommended extension is ``'.verify.json'``. This convention is 

296 used by post-processing tools to discover verification framework 

297 outputs. 

298 """ 

299 dirname = os.path.dirname(filename) 

300 if len(dirname) > 0: 

301 if not os.path.isdir(dirname): 

302 os.makedirs(dirname) 

303 

304 with open(filename, 'w') as f: 

305 json.dump(self.json, f, indent=2, sort_keys=True) 

306 

307 def dispatch(self, api_user=None, api_password=None, 

308 api_url='https://squash-restful-api.lsst.codes', 

309 **kwargs): 

310 """POST the job to SQUASH, LSST Data Management's metric dashboard. 

311 

312 Parameters 

313 ---------- 

314 api_url : `str`, optional 

315 Root URL of the SQUASH API server. 

316 api_user : `str`, optional 

317 API username. 

318 api_password : `str`, optional 

319 API password. 

320 **kwargs : optional 

321 Additional keyword arguments passed to `lsst.verify.squash.post`. 

322 

323 Returns 

324 ------- 

325 output : response 

326 The response from the POST request to the SQuaSH API 

327 """ 

328 full_json_doc = self.json 

329 # subset JSON to just the 'job' fields; no metrics and specs 

330 job_json = {k: full_json_doc[k] 

331 for k in ('measurements', 'blobs', 'meta')} 

332 

333 access_token = squash.get_access_token(api_url, api_user, 

334 api_password) 

335 

336 return squash.post(api_url, 'job', json_doc=job_json, 

337 access_token=access_token, **kwargs) 

338 

339 def report(self, name=None, spec_tags=None, metric_tags=None): 

340 r"""Create a verification report that lists the pass/fail status of 

341 measurements against specifications in this job. 

342 

343 In a Jupyter notebook, this report can be shown as an inline table. 

344 

345 Parameters 

346 ---------- 

347 name : `str` or `lsst.verify.Name`, optional 

348 A package or metric name to subset specifications by. When set, 

349 only measurement and specification combinations belonging to that 

350 package or metric are included in the report. 

351 spec_tags : sequence of `str`, optional 

352 A set of specification tag strings. when given, only 

353 specifications that have all the given tags are included in the 

354 report. For example, ``spec_tags=['LPM-17', 'minimum']``. 

355 metric_tags : sequence of `str`, optional 

356 A set of metric tag strings. When given, only specifications 

357 belonging to metrics that posess **all** given tags are included 

358 in the report. For example, 

359 ``metric_tags=['LPM-17', 'photometry']`` selects sepifications 

360 that have both the ``'LPM-17'`` and ``'photometry'`` tags. 

361 

362 Returns 

363 ------- 

364 report : `lsst.verify.Report` 

365 Report instance. In a Jupyter notebook, you can view the report 

366 by calling `Report.show`. 

367 

368 See also 

369 -------- 

370 lsst.verify.SpecificationSet.report 

371 

372 Notes 

373 ----- 

374 This method uses the `lsst.verify.SpecificationSet.report` API to 

375 create the `lsst.verify.Report`, automatically inserting the `Job`\ 's 

376 measurements and metadata for filtering specifiation tests. 

377 

378 In a Jupyter notebook environment, use the `lsst.verify.Report.show` 

379 method to view an interactive HTML table. 

380 

381 .. code-block:: python 

382 

383 job = lsst.verify.Job() 

384 # ... 

385 report = job.report() 

386 report.show() 

387 """ 

388 report = self.specs.report(self.measurements, meta=self.meta, 

389 name=name, metric_tags=metric_tags, 

390 spec_tags=spec_tags, metrics=self.metrics) 

391 return report