Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Upload LSST Science Pipelines Verification `~lsst.verify.Job` datasets to 

22the SQUASH dashboard. 

23 

24Job JSON files can be created by `lsst.verify.Job.write` or 

25`lsst.verify.output_quantities`. A `~lsst.verify.Job` dataset consists of 

26metric measurements, associated blobs, and pipeline execution metadata. 

27Individual LSST Science Pipelines tasks typically write separate JSON datasets. 

28This command can collect and combine multiple Job JSON datasets into a single 

29Job upload. 

30 

31**Configuration** 

32 

33dispatch_verify.py is configurable from both the command line and environment 

34variables. See the argument documenation for environment variable equivalents. 

35Command line settings override environment variable configuration. 

36 

37**Metadata and environment** 

38 

39dispatch_verify.py can enrich Verification Job metadata with information 

40from the environment. Currently dispatch_verify.py supports the Jenkins CI 

41and the LSST Data Facility (LDF) execution environments. 

42 

43In the Jenkins CI execution environment (``--env=jenkins``) the 

44following environment variables are consumed: 

45 

46- ``BUILD_ID``: ID in the CI system 

47- ``BUILD_URL``: CI page with information about the build 

48- ``PRODUCT``: the name of the product built, e.g. 'validate_drp' 

49- ``dataset``: the name of the dataset processed, e.g. 'validation_data_cfht' 

50- ``label``: the name of the platform where it runs 

51- ``refs``: the branches run by Jenkins, e.g. 'tickets/DM-12345 master' 

52 

53If ``--lsstsw`` is used, additional Git branch information is included with 

54Science Pipelines package metadata. 

55 

56In the LSST Data Facility execution environment (``--env=ldf``) the following 

57environment variables are consumed: 

58 

59- ``DATASET``: the name of the dataset processed, e.g 'HSC RC2' 

60- ``DATASET_REPO_URL``: a reference URL with information about the dataset 

61- ``RUN_ID``: ID of the run in the LDF environment 

62- ``RUN_ID_URL``: a reference URL with information about the run 

63- ``VERSION_TAG``: the version tag of the LSST software used, e.g. 'w_2018_18' 

64 

65Note: currently it is not possible to gather Science Pipelines package metadata 

66in the LDF environment, thus if ``--env=ldf`` is used ``--ignore-lsstsw`` is 

67aslo used by default in this environment. 

68""" 

69# For determining what is documented in Sphinx 

70__all__ = ['build_argparser', 'main', 'insert_lsstsw_metadata', 

71 'insert_extra_package_metadata', 'insert_env_metadata', 

72 'Configuration'] 

73 

74import argparse 

75import os 

76import json 

77import getpass 

78 

79try: 

80 import git 

81except ImportError: 

82 # GitPython is not a standard Stack package; skip gracefully if unavailable 

83 git = None 

84 

85import lsst.log 

86from lsst.verify import Job 

87from lsst.verify.metadata.lsstsw import LsstswRepos 

88from lsst.verify.metadata.eupsmanifest import Manifest 

89from lsst.verify.metadata.jenkinsci import get_jenkins_env 

90from lsst.verify.metadata.ldf import get_ldf_env 

91 

92 

93def build_argparser(): 

94 parser = argparse.ArgumentParser( 

95 description=__doc__, 

96 formatter_class=argparse.RawDescriptionHelpFormatter, 

97 epilog='More information is available at https://pipelines.lsst.io.') 

98 

99 parser.add_argument( 

100 'json_paths', 

101 nargs='+', 

102 metavar='json', 

103 help='Verification job JSON file, or files. When multiple JSON ' 

104 'files are present, their measurements, blobs, and metadata ' 

105 'are merged.') 

106 parser.add_argument( 

107 '--test', 

108 default=False, 

109 action='store_true', 

110 help='Run this command without uploading to the SQUASH service. ' 

111 'The JSON payload is printed to standard out.') 

112 parser.add_argument( 

113 '--write', 

114 metavar='PATH', 

115 dest='output_filepath', 

116 help='Write the merged and enriched Job JSON dataset to the given ' 

117 'path.') 

118 parser.add_argument( 

119 '--show', 

120 dest='show_json', 

121 action='store_true', 

122 default=False, 

123 help='Print the assembled Job JSON to standard output.') 

124 parser.add_argument( 

125 '--ignore-blobs', 

126 dest='ignore_blobs', 

127 action='store_true', 

128 default=False, 

129 help='Ignore data blobs even if they are available in the verification' 

130 'job.') 

131 

132 env_group = parser.add_argument_group('Environment arguments') 

133 env_group.add_argument( 

134 '--env', 

135 dest='env_name', 

136 choices=Configuration.allowed_env, 

137 help='Name of the environment where the verification job is being ' 

138 'run. In some environments display_verify.py will gather ' 

139 'additional metadata automatically:\n' 

140 '\n' 

141 'jenkins\n' 

142 ' For the Jenkins CI (https://ci.lsst.codes)' 

143 ' environment.\n' 

144 'ldf\n' 

145 ' For the LSST Data Facility environment. \n' 

146 '\n' 

147 'Equivalent to the $VERIFY_ENV environment variable.') 

148 env_group.add_argument( 

149 '--lsstsw', 

150 dest='lsstsw', 

151 metavar='PATH', 

152 help='lsstsw directory path. If available, Stack package versions are ' 

153 'read from lsstsw. Equivalent to the ``$LSSTSW`` environment ' 

154 'variable. Disabled with ``--ignore-lsstsw.``') 

155 env_group.add_argument( 

156 '--package-repos', 

157 dest='extra_package_paths', 

158 nargs='*', 

159 metavar='PATH', 

160 help='Paths to additional Stack package Git repositories. These ' 

161 'packages are tracked in Job metadata, like lsstsw-based ' 

162 'packages.') 

163 env_group.add_argument( 

164 '--ignore-lsstsw', 

165 dest='ignore_lsstsw', 

166 action='store_true', 

167 default=False, 

168 help='Ignore lsstsw metadata even if it is available (for example, ' 

169 'the ``$LSSTSW`` variable is set).') 

170 

171 api_group = parser.add_argument_group('SQUASH API arguments') 

172 api_group.add_argument( 

173 '--url', 

174 dest='api_url', 

175 metavar='URL', 

176 help='Root URL of the SQUASH API. Equivalent to the ``$SQUASH_URL`` ' 

177 'environment variable.') 

178 api_group.add_argument( 

179 '--user', 

180 dest='api_user', 

181 metavar='USER', 

182 help='Username for SQUASH API. Equivalent to the $SQUASH_USER ' 

183 'environment variable.') 

184 api_group.add_argument( 

185 '--password', 

186 dest='api_password', 

187 metavar='PASSWORD', 

188 help='Password for SQUASH API. Equivalent to the ``$SQUASH_PASSWORD`` ' 

189 'environment variable. If neither is set, you will be prompted.') 

190 return parser 

191 

192 

193def main(): 

194 """Entrypoint for the ``dispatch_verify.py`` command line executable. 

195 """ 

196 log = lsst.log.Log.getLogger('verify.bin.dispatchverify.main') 

197 

198 parser = build_argparser() 

199 args = parser.parse_args() 

200 config = Configuration(args) 

201 log.debug(str(config)) 

202 

203 # Parse all Job JSON 

204 jobs = [] 

205 for json_path in config.json_paths: 

206 log.info('Loading {0}'.format(json_path)) 

207 with open(json_path) as fp: 

208 json_data = json.load(fp) 

209 # Ignore blobs from the verification jobs 

210 if config.ignore_blobs: 

211 log.info('Ignoring blobs from Job JSON {0}'.format(json_path)) 

212 json_data = delete_blobs(json_data) 

213 job = Job.deserialize(**json_data) 

214 jobs.append(job) 

215 

216 # Merge all Jobs into one 

217 job = jobs.pop(0) 

218 if len(jobs) > 0: 

219 log.info('Merging verification Job JSON.') 

220 for other_job in jobs: 

221 job += other_job 

222 

223 # Ensure all measurements have a metric so that units are normalized 

224 log.info('Refreshing metric definitions from verify_metrics') 

225 job.reload_metrics_package('verify_metrics') 

226 

227 # Insert package metadata from lsstsw 

228 if not config.ignore_lsstsw: 

229 log.info('Inserting lsstsw package metadata from ' 

230 '{0}.'.format(config.lsstsw)) 

231 job = insert_lsstsw_metadata(job, config) 

232 

233 # Insert metadata from additional specified packages 

234 if config.extra_package_paths is not None: 

235 job = insert_extra_package_metadata(job, config) 

236 

237 # Add environment variable metadata from the Jenkins CI environment 

238 if config.env_name == 'jenkins': 

239 log.info('Inserting Jenkins CI environment metadata.') 

240 jenkins_metadata = get_jenkins_env() 

241 job = insert_env_metadata(job, 'jenkins', jenkins_metadata) 

242 elif config.env_name == 'ldf': 

243 log.info('Inserting LSST Data Facility environment metadata.') 

244 ldf_metadata = get_ldf_env() 

245 job = insert_env_metadata(job, 'ldf', ldf_metadata) 

246 

247 # Upload job 

248 if not config.test: 

249 log.info('Uploading Job JSON to {0}.'.format(config.api_url)) 

250 job.dispatch(api_user=config.api_user, 

251 api_password=config.api_password, 

252 api_url=config.api_url) 

253 

254 if config.show_json: 

255 print(json.dumps(job.json, 

256 sort_keys=True, indent=4, separators=(',', ': '))) 

257 

258 # Write a json file 

259 if config.output_filepath is not None: 

260 log.info('Writing Job JSON to {0}.'.format(config.output_filepath)) 

261 job.write(config.output_filepath) 

262 

263 

264def delete_blobs(json_data): 

265 """Delete data blobs from the Job JSON 

266 """ 

267 if 'blobs' in json_data: 

268 del json_data['blobs'] 

269 return json_data 

270 

271 

272def insert_lsstsw_metadata(job, config): 

273 """Insert metadata for lsstsw-based packages into ``Job.meta['packages']``. 

274 """ 

275 lsstsw_repos = LsstswRepos(config.lsstsw) 

276 

277 with open(lsstsw_repos.manifest_path) as fp: 

278 manifest = Manifest(fp) 

279 

280 packages = {} 

281 for package_name, manifest_item in manifest.items(): 

282 package_doc = { 

283 'name': package_name, 

284 'git_branch': lsstsw_repos.get_package_branch(package_name), 

285 'git_url': lsstsw_repos.get_package_repo_url(package_name), 

286 'git_sha': manifest_item.git_sha, 

287 'eups_version': manifest_item.version 

288 } 

289 packages[package_name] = package_doc 

290 

291 if 'packages' in job.meta: 

292 # Extend packages entry 

293 job.meta['packages'].update(packages) 

294 else: 

295 # Create new packages entry 

296 job.meta['packages'] = packages 

297 return job 

298 

299 

300def insert_extra_package_metadata(job, config): 

301 """Insert metadata for extra packages ('--package-repos') into 

302 ``Job.meta['packages']``. 

303 """ 

304 log = lsst.log.Log.getLogger( 

305 'verify.bin.dispatchverify.insert_extra_package_metadata') 

306 

307 if 'packages' not in job.meta: 

308 job.meta['packages'] = dict() 

309 

310 for package_path in config.extra_package_paths: 

311 log.info('Inserting extra package metadata: {0}'.format(package_path)) 

312 package_name = package_path.split(os.sep)[-1] 

313 

314 package = {'name': package_name} 

315 

316 if git is not None: 

317 git_repo = git.Repo(package_path) 

318 package['git_sha'] = git_repo.active_branch.commit.hexsha 

319 package['git_branch'] = git_repo.active_branch.name 

320 package['git_url'] = git_repo.remotes.origin.url 

321 

322 if package_name in job.meta['packages']: 

323 # Update pre-existing package metadata 

324 job.meta['packages'][package_name].update(package) 

325 else: 

326 # Create new package metadata 

327 job.meta['packages'][package_name] = package 

328 

329 return job 

330 

331 

332def insert_env_metadata(job, env_name, metadata): 

333 """Insert environment metadata into the Job. 

334 """ 

335 metadata.update({'env_name': env_name}) 

336 job.meta['env'] = metadata 

337 

338 return job 

339 

340 

341class Configuration(object): 

342 """Configuration for dispatch_verify.py that reconciles command line and 

343 environment variable arguments. 

344 

345 Configuration is validated for completeness and certain errors. 

346 

347 Parameters 

348 ---------- 

349 args : `argparse.Namespace` 

350 Parsed command line arguments, produced by `parse_args`. 

351 """ 

352 

353 allowed_env = ('jenkins', 'ldf') 

354 

355 def __init__(self, args): 

356 self.json_paths = args.json_paths 

357 

358 self.test = args.test 

359 

360 self.output_filepath = args.output_filepath 

361 

362 self.show_json = args.show_json 

363 

364 self.env_name = args.env_name or os.getenv('VERIFY_ENV') 

365 if self.env_name is not None and self.env_name not in self.allowed_env: 

366 message = '$VERIFY_ENV not one of {0!s}'.format(self.allowed_env) 

367 raise RuntimeError(message) 

368 

369 self.ignore_blobs = args.ignore_blobs 

370 

371 self.ignore_lsstsw = args.ignore_lsstsw 

372 

373 # Make sure --ignore-lsstw is used in the LDF environment 

374 if self.env_name == 'ldf': 

375 self.ignore_lsstsw = True 

376 

377 self.lsstsw = args.lsstsw or os.getenv('LSSTSW') 

378 if self.lsstsw is not None: 

379 self.lsstsw = os.path.abspath(self.lsstsw) 

380 if not self.ignore_lsstsw and not self.lsstsw: 

381 message = 'lsstsw directory not found at {0}'.format(self.lsstsw) 

382 raise RuntimeError(message) 

383 

384 if args.extra_package_paths is not None: 

385 self.extra_package_paths = [os.path.abspath(p) 

386 for p in args.extra_package_paths] 

387 else: 

388 self.extra_package_paths = [] 

389 for path in self.extra_package_paths: 

390 if not os.path.isdir(path): 

391 message = 'Package directory not found: {0}'.format(path) 

392 raise RuntimeError(message) 

393 

394 default_url = 'https://squash.lsst.codes/dashboard/api' 

395 self.api_url = args.api_url or os.getenv('SQUASH_URL', default_url) 

396 

397 self.api_user = args.api_user or os.getenv('SQUASH_USER') 

398 if not self.test and self.api_user is None: 

399 message = '--user or $SQUASH_USER configuration required' 

400 raise RuntimeError(message) 

401 

402 self.api_password = (args.api_password 

403 or os.getenv('SQUASH_password')) 

404 if not self.test and self.api_password is None: 

405 # If password hasn't been set, prompt for it. 

406 self.api_password = getpass.getpass(prompt="SQuaSH password: ") 

407 

408 def __str__(self): 

409 configs = { 

410 'json_paths': self.json_paths, 

411 'test': self.test, 

412 'output_filepath': self.output_filepath, 

413 'show_json': self.show_json, 

414 'ignore_blobs': self.ignore_blobs, 

415 'env': self.env_name, 

416 'ignore_lsstsw': self.ignore_lsstsw, 

417 'lsstsw': self.lsstsw, 

418 'extra_package_paths': self.extra_package_paths, 

419 'api_url': self.api_url, 

420 'api_user': self.api_user, 

421 } 

422 if self.api_password is None: 

423 configs['api_password'] = None 

424 else: 

425 configs['api_password'] = '*' * len(self.api_password) 

426 

427 return json.dumps(configs, 

428 sort_keys=True, indent=4, separators=(',', ': '))