Coverage for python/lsst/ap/verify/ap_verify.py : 35%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Command-line program for running and analyzing AP pipeline.
26In addition to containing ap_verify's main function, this module manages
27command-line argument parsing.
28"""
30__all__ = ["runApVerify", "runIngestion"]
32import argparse
33import re
35import lsst.log
37from .dataset import Dataset
38from .ingestion import ingestDataset
39from .metrics import MetricsParser, computeMetrics
40from .pipeline_driver import ApPipeParser, runApPipe
41from .workspace import Workspace
44class _InputOutputParser(argparse.ArgumentParser):
45 """An argument parser for program-wide input and output.
47 This parser is not complete, and is designed to be passed to another parser
48 using the `parent` parameter.
49 """
51 def __init__(self):
52 # Help and documentation will be handled by main program's parser
53 argparse.ArgumentParser.__init__(self, add_help=False)
54 self.add_argument('--dataset', action=_DatasetAction, choices=Dataset.getSupportedDatasets(),
55 required=True, help='The source of data to pass through the pipeline.')
56 self.add_argument('--output', required=True,
57 help='The location of the workspace to use for pipeline repositories.')
60class _ApVerifyParser(argparse.ArgumentParser):
61 """An argument parser for data needed by the main ap_verify program.
62 """
64 def __init__(self):
65 argparse.ArgumentParser.__init__(
66 self,
67 description='Executes the LSST DM AP pipeline and analyzes its performance using metrics.',
68 epilog='',
69 parents=[_InputOutputParser(), ApPipeParser(), MetricsParser()],
70 add_help=True)
73class _IngestOnlyParser(argparse.ArgumentParser):
74 """An argument parser for data needed by dataset ingestion.
75 """
77 def __init__(self):
78 argparse.ArgumentParser.__init__(
79 self,
80 description='Ingests a dataset into a pair of Butler repositories.'
81 'The program will create a data repository in <OUTPUT>/ingested and a calib repository '
82 'in <OUTPUT>/calibingested. '
83 'These repositories may be used directly by ap_verify.py by '
84 'passing the same --output argument, or by other programs that accept '
85 'Butler repositories as input.',
86 epilog='',
87 parents=[_InputOutputParser()],
88 add_help=True)
91class _FormattedType:
92 """An argparse type converter that requires strings in a particular format.
94 Leaves the input as a string if it matches, else raises `argparse.ArgumentTypeError`.
96 Parameters
97 ----------
98 fmt : `str`
99 A regular expression that values must satisfy to be accepted. The *entire* string must match the
100 expression in order to pass.
101 msg : `str`
102 An error string to display for invalid values. The first "%s" shall be filled with the
103 invalid argument.
104 """
105 def __init__(self, fmt, msg='"%s" does not have the expected format.'):
106 fullFormat = fmt
107 if not fullFormat.startswith('^'):
108 fullFormat = '^' + fullFormat
109 if not fullFormat.endswith('$'):
110 fullFormat += '$'
111 self._format = re.compile(fullFormat)
112 self._message = msg
114 def __call__(self, value):
115 if self._format.match(value):
116 return value
117 else:
118 raise argparse.ArgumentTypeError(self._message % value)
121class _DatasetAction(argparse.Action):
122 """A converter for dataset arguments.
124 Not an argparse type converter so that the ``choices`` parameter can be
125 expressed using strings; ``choices`` checks happen after type conversion
126 but before actions.
127 """
128 def __call__(self, _parser, namespace, values, _option_string=None):
129 setattr(namespace, self.dest, Dataset(values))
132def runApVerify(cmdLine=None):
133 """Execute the AP pipeline while handling metrics.
135 This is the main function for ``ap_verify``, and handles logging,
136 command-line argument parsing, pipeline execution, and metrics
137 generation.
139 Parameters
140 ----------
141 cmdLine : `list` of `str`
142 an optional command line used to execute `runApVerify` from other
143 Python code. If `None`, `sys.argv` will be used.
145 Returns
146 -------
147 nFailed : `int`
148 The number of data IDs that were not successfully processed, up to 127,
149 or 127 if the task runner framework failed.
150 """
151 lsst.log.configure()
152 log = lsst.log.Log.getLogger('ap.verify.ap_verify.main')
153 # TODO: what is LSST's policy on exceptions escaping into main()?
154 args = _ApVerifyParser().parse_args(args=cmdLine)
155 log.debug('Command-line arguments: %s', args)
157 workspace = Workspace(args.output)
158 ingestDataset(args.dataset, workspace)
160 log.info('Running pipeline...')
161 apPipeResults = runApPipe(workspace, args)
162 computeMetrics(workspace, apPipeResults.parsedCmd.id, args)
164 return _getCmdLineExitStatus(apPipeResults.resultList)
167def _getCmdLineExitStatus(resultList):
168 """Return the exit status following the conventions of
169 :ref:`running a CmdLineTask from the command line
170 <command-line-task-argument-reference>`.
172 Parameters
173 ----------
174 resultList : `list` [`Struct`] or `None`
175 A list of `Struct`, as returned by `ApPipeTask.parseAndRun`. Each
176 element must contain at least an ``exitStatus`` member.
178 Returns
179 -------
180 exitStatus : `int`
181 The number of failed runs in ``resultList``, up to 127, or 127 if
182 ``resultList`` is `None`.
183 """
184 if resultList:
185 # ApPipeTaskRunner does not override default results handling, exitStatus always defined
186 return min(127, sum(((res.exitStatus != 0) for res in resultList)))
187 else:
188 return 127
191def runIngestion(cmdLine=None):
192 """Ingest a dataset, but do not process it.
194 This is the main function for ``ingest_dataset``, and handles logging,
195 command-line argument parsing, and ingestion.
197 Parameters
198 ----------
199 cmdLine : `list` of `str`
200 an optional command line used to execute `runIngestion` from other
201 Python code. If `None`, `sys.argv` will be used.
202 """
203 lsst.log.configure()
204 log = lsst.log.Log.getLogger('ap.verify.ap_verify.ingest')
205 # TODO: what is LSST's policy on exceptions escaping into main()?
206 args = _IngestOnlyParser().parse_args(args=cmdLine)
207 log.debug('Command-line arguments: %s', args)
209 workspace = Workspace(args.output)
210 ingestDataset(args.dataset, workspace)