Coverage for python/lsst/ap/verify/ap_verify.py : 30%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Command-line program for running and analyzing AP pipeline.
26In addition to containing ap_verify's main function, this module manages
27command-line argument parsing.
28"""
30__all__ = ["runApVerify", "runIngestion"]
32import argparse
33import re
35import lsst.log
37from .dataset import Dataset
38from .ingestion import ingestDataset, ingestDatasetGen3
39from .metrics import MetricsParser, computeMetrics
40from .pipeline_driver import ApPipeParser, runApPipeGen2, runApPipeGen3
41from .workspace import WorkspaceGen2, WorkspaceGen3
44class _InputOutputParser(argparse.ArgumentParser):
45 """An argument parser for program-wide input and output.
47 This parser is not complete, and is designed to be passed to another parser
48 using the `parent` parameter.
49 """
51 def __init__(self):
52 # Help and documentation will be handled by main program's parser
53 argparse.ArgumentParser.__init__(self, add_help=False)
54 self.add_argument('--dataset', action=_DatasetAction, choices=Dataset.getSupportedDatasets(),
55 required=True, help='The source of data to pass through the pipeline.')
56 self.add_argument('--output', required=True,
57 help='The location of the workspace to use for pipeline repositories.')
59 gen23 = self.add_mutually_exclusive_group()
60 # Because store_true and store_false use the same dest, add explicit
61 # default to avoid ambiguity.
62 gen23.add_argument('--gen2', dest='useGen3', action='store_false', default=False,
63 help='Handle the ap_verify dataset using the Gen 2 framework (default).')
64 gen23.add_argument('--gen3', dest='useGen3', action='store_true', default=False,
65 help='Handle the ap_verify dataset using the Gen 3 framework.')
68class _ProcessingParser(argparse.ArgumentParser):
69 """An argument parser for general run-time characteristics.
71 This parser is not complete, and is designed to be passed to another parser
72 using the `parent` parameter.
73 """
75 def __init__(self):
76 # Help and documentation will be handled by main program's parser
77 argparse.ArgumentParser.__init__(self, add_help=False)
78 self.add_argument("-j", "--processes", default=1, type=int,
79 help="Number of processes to use.")
82class _ApVerifyParser(argparse.ArgumentParser):
83 """An argument parser for data needed by the main ap_verify program.
84 """
86 def __init__(self):
87 argparse.ArgumentParser.__init__(
88 self,
89 description='Executes the LSST DM AP pipeline and analyzes its performance using metrics.',
90 epilog='',
91 parents=[_InputOutputParser(), _ProcessingParser(), ApPipeParser(), MetricsParser()],
92 add_help=True)
95class _IngestOnlyParser(argparse.ArgumentParser):
96 """An argument parser for data needed by dataset ingestion.
97 """
99 def __init__(self):
100 argparse.ArgumentParser.__init__(
101 self,
102 description='Ingests an ap_verify dataset into a pair of Butler repositories. '
103 'The program will create repository(ies) appropriate for --gen2 or --gen3 '
104 'in subdirectories of <OUTPUT>. '
105 'These repositories may be used directly by ap_verify.py by '
106 'passing the same --output argument, or by other programs that accept '
107 'Butler repositories as input.',
108 epilog='',
109 parents=[_InputOutputParser(), _ProcessingParser()],
110 add_help=True)
113class _FormattedType:
114 """An argparse type converter that requires strings in a particular format.
116 Leaves the input as a string if it matches, else raises `argparse.ArgumentTypeError`.
118 Parameters
119 ----------
120 fmt : `str`
121 A regular expression that values must satisfy to be accepted. The *entire* string must match the
122 expression in order to pass.
123 msg : `str`
124 An error string to display for invalid values. The first "%s" shall be filled with the
125 invalid argument.
126 """
127 def __init__(self, fmt, msg='"%s" does not have the expected format.'):
128 fullFormat = fmt
129 if not fullFormat.startswith('^'):
130 fullFormat = '^' + fullFormat
131 if not fullFormat.endswith('$'):
132 fullFormat += '$'
133 self._format = re.compile(fullFormat)
134 self._message = msg
136 def __call__(self, value):
137 if self._format.match(value):
138 return value
139 else:
140 raise argparse.ArgumentTypeError(self._message % value)
143class _DatasetAction(argparse.Action):
144 """A converter for dataset arguments.
146 Not an argparse type converter so that the ``choices`` parameter can be
147 expressed using strings; ``choices`` checks happen after type conversion
148 but before actions.
149 """
150 def __call__(self, _parser, namespace, values, _option_string=None):
151 setattr(namespace, self.dest, Dataset(values))
154def runApVerify(cmdLine=None):
155 """Execute the AP pipeline while handling metrics.
157 This is the main function for ``ap_verify``, and handles logging,
158 command-line argument parsing, pipeline execution, and metrics
159 generation.
161 Parameters
162 ----------
163 cmdLine : `list` of `str`
164 an optional command line used to execute `runApVerify` from other
165 Python code. If `None`, `sys.argv` will be used.
167 Returns
168 -------
169 nFailed : `int`
170 The number of data IDs that were not successfully processed, up to 127,
171 or 127 if the task runner framework failed.
172 """
173 lsst.log.configure()
174 log = lsst.log.Log.getLogger('ap.verify.ap_verify.main')
175 # TODO: what is LSST's policy on exceptions escaping into main()?
176 args = _ApVerifyParser().parse_args(args=cmdLine)
177 log.debug('Command-line arguments: %s', args)
179 if args.useGen3:
180 workspace = WorkspaceGen3(args.output)
181 ingestDatasetGen3(args.dataset, workspace, processes=args.processes)
182 log.info('Running pipeline...')
183 # Gen 3 pipeline includes both AP and metrics
184 return runApPipeGen3(workspace, args, processes=args.processes)
185 else:
186 workspace = WorkspaceGen2(args.output)
187 ingestDataset(args.dataset, workspace)
188 log.info('Running pipeline...')
189 apPipeResults = runApPipeGen2(workspace, args, processes=args.processes)
190 computeMetrics(workspace, apPipeResults.parsedCmd.id, args)
191 return _getCmdLineExitStatus(apPipeResults.resultList)
194def _getCmdLineExitStatus(resultList):
195 """Return the exit status following the conventions of
196 :ref:`running a CmdLineTask from the command line
197 <command-line-task-argument-reference>`.
199 Parameters
200 ----------
201 resultList : `list` [`Struct`] or `None`
202 A list of `Struct`, as returned by `ApPipeTask.parseAndRun`. Each
203 element must contain at least an ``exitStatus`` member.
205 Returns
206 -------
207 exitStatus : `int`
208 The number of failed runs in ``resultList``, up to 127, or 127 if
209 ``resultList`` is `None`.
210 """
211 if resultList:
212 # ApPipeTaskRunner does not override default results handling, exitStatus always defined
213 return min(127, sum(((res.exitStatus != 0) for res in resultList)))
214 else:
215 return 127
218def runIngestion(cmdLine=None):
219 """Ingest a dataset, but do not process it.
221 This is the main function for ``ingest_dataset``, and handles logging,
222 command-line argument parsing, and ingestion.
224 Parameters
225 ----------
226 cmdLine : `list` of `str`
227 an optional command line used to execute `runIngestion` from other
228 Python code. If `None`, `sys.argv` will be used.
229 """
230 lsst.log.configure()
231 log = lsst.log.Log.getLogger('ap.verify.ap_verify.ingest')
232 # TODO: what is LSST's policy on exceptions escaping into main()?
233 args = _IngestOnlyParser().parse_args(args=cmdLine)
234 log.debug('Command-line arguments: %s', args)
236 if args.useGen3:
237 workspace = WorkspaceGen3(args.output)
238 ingestDatasetGen3(args.dataset, workspace, processes=args.processes)
239 else:
240 workspace = WorkspaceGen2(args.output)
241 ingestDataset(args.dataset, workspace)