Coverage for python/lsst/verify/bin/dispatchverify.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Upload LSST Science Pipelines Verification `~lsst.verify.Job` datasets to
22the SQUASH dashboard.
24Job JSON files can be created by `lsst.verify.Job.write` or
25`lsst.verify.output_quantities`. A `~lsst.verify.Job` dataset consists of
26metric measurements, associated blobs, and pipeline execution metadata.
27Individual LSST Science Pipelines tasks typically write separate JSON datasets.
28This command can collect and combine multiple Job JSON datasets into a single
29Job upload.
31**Configuration**
33dispatch_verify.py is configurable from both the command line and environment
34variables. See the argument documenation for environment variable equivalents.
35Command line settings override environment variable configuration.
37**Metadata and environment**
39dispatch_verify.py can enrich Verification Job metadata with information
40from the environment. Currently dispatch_verify.py supports the Jenkins CI
41and the LSST Data Facility (LDF) execution environments.
43In the Jenkins CI execution environment (``--env=jenkins``) the
44following environment variables are consumed:
46- ``BUILD_ID``: ID in the CI system
47- ``BUILD_URL``: CI page with information about the build
48- ``PRODUCT``: the name of the product built, e.g. 'validate_drp'
49- ``dataset``: the name of the dataset processed, e.g. 'validation_data_cfht'
50- ``label``: the name of the platform where it runs
51- ``refs``: the branches run by Jenkins, e.g. 'tickets/DM-12345 master'
53If ``--lsstsw`` is used, additional Git branch information is included with
54Science Pipelines package metadata.
56In the LSST Data Facility execution environment (``--env=ldf``) the following
57environment variables are consumed:
59- ``DATASET``: the name of the dataset processed, e.g 'HSC RC2'
60- ``DATASET_REPO_URL``: a reference URL with information about the dataset
61- ``RUN_ID``: ID of the run in the LDF environment
62- ``RUN_ID_URL``: a reference URL with information about the run
63- ``VERSION_TAG``: the version tag of the LSST software used, e.g. 'w_2018_18'
65Note: currently it is not possible to gather Science Pipelines package metadata
66in the LDF environment, thus if ``--env=ldf`` is used ``--ignore-lsstsw`` is
67aslo used by default in this environment.
68"""
69# For determining what is documented in Sphinx
70__all__ = ['build_argparser', 'main', 'insert_lsstsw_metadata',
71 'insert_extra_package_metadata', 'insert_env_metadata',
72 'Configuration']
74import argparse
75import os
76import json
77import getpass
79try:
80 import git
81except ImportError:
82 # GitPython is not a standard Stack package; skip gracefully if unavailable
83 git = None
85import lsst.log
86from lsst.verify import Job
87from lsst.verify.metadata.lsstsw import LsstswRepos
88from lsst.verify.metadata.eupsmanifest import Manifest
89from lsst.verify.metadata.jenkinsci import get_jenkins_env
90from lsst.verify.metadata.ldf import get_ldf_env
93def build_argparser():
94 parser = argparse.ArgumentParser(
95 description=__doc__,
96 formatter_class=argparse.RawDescriptionHelpFormatter,
97 epilog='More information is available at https://pipelines.lsst.io.')
99 parser.add_argument(
100 'json_paths',
101 nargs='+',
102 metavar='json',
103 help='Verification job JSON file, or files. When multiple JSON '
104 'files are present, their measurements, blobs, and metadata '
105 'are merged.')
106 parser.add_argument(
107 '--test',
108 default=False,
109 action='store_true',
110 help='Run this command without uploading to the SQUASH service. '
111 'The JSON payload is printed to standard out.')
112 parser.add_argument(
113 '--write',
114 metavar='PATH',
115 dest='output_filepath',
116 help='Write the merged and enriched Job JSON dataset to the given '
117 'path.')
118 parser.add_argument(
119 '--show',
120 dest='show_json',
121 action='store_true',
122 default=False,
123 help='Print the assembled Job JSON to standard output.')
124 parser.add_argument(
125 '--ignore-blobs',
126 dest='ignore_blobs',
127 action='store_true',
128 default=False,
129 help='Ignore data blobs even if they are available in the verification'
130 'job.')
132 env_group = parser.add_argument_group('Environment arguments')
133 env_group.add_argument(
134 '--env',
135 dest='env_name',
136 choices=Configuration.allowed_env,
137 help='Name of the environment where the verification job is being '
138 'run. In some environments display_verify.py will gather '
139 'additional metadata automatically:\n'
140 '\n'
141 'jenkins\n'
142 ' For the Jenkins CI (https://ci.lsst.codes)'
143 ' environment.\n'
144 'ldf\n'
145 ' For the LSST Data Facility environment. \n'
146 '\n'
147 'Equivalent to the $VERIFY_ENV environment variable.')
148 env_group.add_argument(
149 '--lsstsw',
150 dest='lsstsw',
151 metavar='PATH',
152 help='lsstsw directory path. If available, Stack package versions are '
153 'read from lsstsw. Equivalent to the ``$LSSTSW`` environment '
154 'variable. Disabled with ``--ignore-lsstsw.``')
155 env_group.add_argument(
156 '--package-repos',
157 dest='extra_package_paths',
158 nargs='*',
159 metavar='PATH',
160 help='Paths to additional Stack package Git repositories. These '
161 'packages are tracked in Job metadata, like lsstsw-based '
162 'packages.')
163 env_group.add_argument(
164 '--ignore-lsstsw',
165 dest='ignore_lsstsw',
166 action='store_true',
167 default=False,
168 help='Ignore lsstsw metadata even if it is available (for example, '
169 'the ``$LSSTSW`` variable is set).')
171 api_group = parser.add_argument_group('SQUASH API arguments')
172 api_group.add_argument(
173 '--url',
174 dest='api_url',
175 metavar='URL',
176 help='Root URL of the SQUASH API. Equivalent to the ``$SQUASH_URL`` '
177 'environment variable.')
178 api_group.add_argument(
179 '--user',
180 dest='api_user',
181 metavar='USER',
182 help='Username for SQUASH API. Equivalent to the $SQUASH_USER '
183 'environment variable.')
184 api_group.add_argument(
185 '--password',
186 dest='api_password',
187 metavar='PASSWORD',
188 help='Password for SQUASH API. Equivalent to the ``$SQUASH_PASSWORD`` '
189 'environment variable. If neither is set, you will be prompted.')
190 return parser
193def main():
194 """Entrypoint for the ``dispatch_verify.py`` command line executable.
195 """
196 log = lsst.log.Log.getLogger('verify.bin.dispatchverify.main')
198 parser = build_argparser()
199 args = parser.parse_args()
200 config = Configuration(args)
201 log.debug(str(config))
203 # Parse all Job JSON
204 jobs = []
205 for json_path in config.json_paths:
206 log.info('Loading {0}'.format(json_path))
207 with open(json_path) as fp:
208 json_data = json.load(fp)
209 # Ignore blobs from the verification jobs
210 if config.ignore_blobs:
211 log.info('Ignoring blobs from Job JSON {0}'.format(json_path))
212 json_data = delete_blobs(json_data)
213 job = Job.deserialize(**json_data)
214 jobs.append(job)
216 # Merge all Jobs into one
217 job = jobs.pop(0)
218 if len(jobs) > 0:
219 log.info('Merging verification Job JSON.')
220 for other_job in jobs:
221 job += other_job
223 # Ensure all measurements have a metric so that units are normalized
224 log.info('Refreshing metric definitions from verify_metrics')
225 job.reload_metrics_package('verify_metrics')
227 # Insert package metadata from lsstsw
228 if not config.ignore_lsstsw:
229 log.info('Inserting lsstsw package metadata from '
230 '{0}.'.format(config.lsstsw))
231 job = insert_lsstsw_metadata(job, config)
233 # Insert metadata from additional specified packages
234 if config.extra_package_paths is not None:
235 job = insert_extra_package_metadata(job, config)
237 # Add environment variable metadata from the Jenkins CI environment
238 if config.env_name == 'jenkins':
239 log.info('Inserting Jenkins CI environment metadata.')
240 jenkins_metadata = get_jenkins_env()
241 job = insert_env_metadata(job, 'jenkins', jenkins_metadata)
242 elif config.env_name == 'ldf':
243 log.info('Inserting LSST Data Facility environment metadata.')
244 ldf_metadata = get_ldf_env()
245 job = insert_env_metadata(job, 'ldf', ldf_metadata)
247 # Upload job
248 if not config.test:
249 log.info('Uploading Job JSON to {0}.'.format(config.api_url))
250 job.dispatch(api_user=config.api_user,
251 api_password=config.api_password,
252 api_url=config.api_url)
254 if config.show_json:
255 print(json.dumps(job.json,
256 sort_keys=True, indent=4, separators=(',', ': ')))
258 # Write a json file
259 if config.output_filepath is not None:
260 log.info('Writing Job JSON to {0}.'.format(config.output_filepath))
261 job.write(config.output_filepath)
264def delete_blobs(json_data):
265 """Delete data blobs from the Job JSON
266 """
267 if 'blobs' in json_data:
268 del json_data['blobs']
269 return json_data
272def insert_lsstsw_metadata(job, config):
273 """Insert metadata for lsstsw-based packages into ``Job.meta['packages']``.
274 """
275 lsstsw_repos = LsstswRepos(config.lsstsw)
277 with open(lsstsw_repos.manifest_path) as fp:
278 manifest = Manifest(fp)
280 packages = {}
281 for package_name, manifest_item in manifest.items():
282 package_doc = {
283 'name': package_name,
284 'git_branch': lsstsw_repos.get_package_branch(package_name),
285 'git_url': lsstsw_repos.get_package_repo_url(package_name),
286 'git_sha': manifest_item.git_sha,
287 'eups_version': manifest_item.version
288 }
289 packages[package_name] = package_doc
291 if 'packages' in job.meta:
292 # Extend packages entry
293 job.meta['packages'].update(packages)
294 else:
295 # Create new packages entry
296 job.meta['packages'] = packages
297 return job
300def insert_extra_package_metadata(job, config):
301 """Insert metadata for extra packages ('--package-repos') into
302 ``Job.meta['packages']``.
303 """
304 log = lsst.log.Log.getLogger(
305 'verify.bin.dispatchverify.insert_extra_package_metadata')
307 if 'packages' not in job.meta:
308 job.meta['packages'] = dict()
310 for package_path in config.extra_package_paths:
311 log.info('Inserting extra package metadata: {0}'.format(package_path))
312 package_name = package_path.split(os.sep)[-1]
314 package = {'name': package_name}
316 if git is not None:
317 git_repo = git.Repo(package_path)
318 package['git_sha'] = git_repo.active_branch.commit.hexsha
319 package['git_branch'] = git_repo.active_branch.name
320 package['git_url'] = git_repo.remotes.origin.url
322 if package_name in job.meta['packages']:
323 # Update pre-existing package metadata
324 job.meta['packages'][package_name].update(package)
325 else:
326 # Create new package metadata
327 job.meta['packages'][package_name] = package
329 return job
332def insert_env_metadata(job, env_name, metadata):
333 """Insert environment metadata into the Job.
334 """
335 metadata.update({'env_name': env_name})
336 job.meta['env'] = metadata
338 return job
341class Configuration(object):
342 """Configuration for dispatch_verify.py that reconciles command line and
343 environment variable arguments.
345 Configuration is validated for completeness and certain errors.
347 Parameters
348 ----------
349 args : `argparse.Namespace`
350 Parsed command line arguments, produced by `parse_args`.
351 """
353 allowed_env = ('jenkins', 'ldf')
355 def __init__(self, args):
356 self.json_paths = args.json_paths
358 self.test = args.test
360 self.output_filepath = args.output_filepath
362 self.show_json = args.show_json
364 self.env_name = args.env_name or os.getenv('VERIFY_ENV')
365 if self.env_name is not None and self.env_name not in self.allowed_env:
366 message = '$VERIFY_ENV not one of {0!s}'.format(self.allowed_env)
367 raise RuntimeError(message)
369 self.ignore_blobs = args.ignore_blobs
371 self.ignore_lsstsw = args.ignore_lsstsw
373 # Make sure --ignore-lsstw is used in the LDF environment
374 if self.env_name == 'ldf':
375 self.ignore_lsstsw = True
377 self.lsstsw = args.lsstsw or os.getenv('LSSTSW')
378 if self.lsstsw is not None:
379 self.lsstsw = os.path.abspath(self.lsstsw)
380 if not self.ignore_lsstsw and not self.lsstsw:
381 message = 'lsstsw directory not found at {0}'.format(self.lsstsw)
382 raise RuntimeError(message)
384 if args.extra_package_paths is not None:
385 self.extra_package_paths = [os.path.abspath(p)
386 for p in args.extra_package_paths]
387 else:
388 self.extra_package_paths = []
389 for path in self.extra_package_paths:
390 if not os.path.isdir(path):
391 message = 'Package directory not found: {0}'.format(path)
392 raise RuntimeError(message)
394 default_url = 'https://squash.lsst.codes/dashboard/api'
395 self.api_url = args.api_url or os.getenv('SQUASH_URL', default_url)
397 self.api_user = args.api_user or os.getenv('SQUASH_USER')
398 if not self.test and self.api_user is None:
399 message = '--user or $SQUASH_USER configuration required'
400 raise RuntimeError(message)
402 self.api_password = (args.api_password or
403 os.getenv('SQUASH_password'))
404 if not self.test and self.api_password is None:
405 # If password hasn't been set, prompt for it.
406 self.api_password = getpass.getpass(prompt="SQuaSH password: ")
408 def __str__(self):
409 configs = {
410 'json_paths': self.json_paths,
411 'test': self.test,
412 'output_filepath': self.output_filepath,
413 'show_json': self.show_json,
414 'ignore_blobs': self.ignore_blobs,
415 'env': self.env_name,
416 'ignore_lsstsw': self.ignore_lsstsw,
417 'lsstsw': self.lsstsw,
418 'extra_package_paths': self.extra_package_paths,
419 'api_url': self.api_url,
420 'api_user': self.api_user,
421 }
422 if self.api_password is None:
423 configs['api_password'] = None
424 else:
425 configs['api_password'] = '*' * len(self.api_password)
427 return json.dumps(configs,
428 sort_keys=True, indent=4, separators=(',', ': '))