Coverage for python/lsst/ap/verify/ap_verify.py: 37%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Command-line program for running and analyzing AP pipeline.
26In addition to containing ap_verify's main function, this module manages
27command-line argument parsing.
28"""
30__all__ = ["runApVerify", "runIngestion"]
32import argparse
33import re
34import warnings
35import sys
36import logging
38import lsst.log
40from .dataset import Dataset
41from .ingestion import ingestDataset, ingestDatasetGen3
42from .metrics import MetricsParser, computeMetrics
43from .pipeline_driver import ApPipeParser, runApPipeGen2, runApPipeGen3
44from .workspace import WorkspaceGen2, WorkspaceGen3
46_LOG = logging.getLogger(__name__)
49def _configure_logger():
50 """Configure Python logging.
52 Does basic Python logging configuration and
53 forwards LSST logger to Python logging.
54 """
55 logging.basicConfig(level=logging.INFO, stream=sys.stdout)
56 lsst.log.configure_pylog_MDC("DEBUG", MDC_class=None)
59class _InputOutputParser(argparse.ArgumentParser):
60 """An argument parser for program-wide input and output.
62 This parser is not complete, and is designed to be passed to another parser
63 using the `parent` parameter.
64 """
66 def __init__(self):
67 # Help and documentation will be handled by main program's parser
68 argparse.ArgumentParser.__init__(self, add_help=False)
69 self.add_argument('--dataset', action=_DatasetAction,
70 required=True, help='The source of data to pass through the pipeline.')
71 self.add_argument('--output', required=True,
72 help='The location of the workspace to use for pipeline repositories.')
74 gen23 = self.add_mutually_exclusive_group()
75 # Because store_true and store_false use the same dest, add explicit
76 # default to avoid ambiguity.
77 gen23.add_argument('--gen2', dest='useGen3', action='store_false', default=True,
78 help='Handle the ap_verify dataset using the Gen 2 framework (default).')
79 gen23.add_argument('--gen3', dest='useGen3', action='store_true', default=True,
80 help='Handle the ap_verify dataset using the Gen 3 framework (default).')
83class _ProcessingParser(argparse.ArgumentParser):
84 """An argument parser for general run-time characteristics.
86 This parser is not complete, and is designed to be passed to another parser
87 using the `parent` parameter.
88 """
90 def __init__(self):
91 # Help and documentation will be handled by main program's parser
92 argparse.ArgumentParser.__init__(self, add_help=False)
93 self.add_argument("-j", "--processes", default=1, type=int,
94 help="Number of processes to use.")
97class _ApVerifyParser(argparse.ArgumentParser):
98 """An argument parser for data needed by the main ap_verify program.
99 """
101 def __init__(self):
102 argparse.ArgumentParser.__init__(
103 self,
104 description='Executes the LSST DM AP pipeline and analyzes its performance using metrics.',
105 epilog='',
106 parents=[_InputOutputParser(), _ProcessingParser(), ApPipeParser(), MetricsParser()],
107 add_help=True)
109 def parse_args(self, args=None, namespace=None):
110 namespace = super().parse_args(args, namespace)
111 # Code duplication; too hard to implement at shared _InputOutputParser level
112 if not namespace.useGen3:
113 warnings.warn("The --gen2 flag is deprecated; it will be removed after release 23.",
114 category=FutureWarning)
115 return namespace
118class _IngestOnlyParser(argparse.ArgumentParser):
119 """An argument parser for data needed by dataset ingestion.
120 """
122 def __init__(self):
123 argparse.ArgumentParser.__init__(
124 self,
125 description='Ingests an ap_verify dataset into a pair of Butler repositories. '
126 'The program will create repository(ies) appropriate for --gen2 or --gen3 '
127 'in subdirectories of <OUTPUT>. '
128 'These repositories may be used directly by ap_verify.py by '
129 'passing the same --output argument, or by other programs that accept '
130 'Butler repositories as input.',
131 epilog='',
132 parents=[_InputOutputParser(), _ProcessingParser()],
133 add_help=True)
135 def parse_args(self, args=None, namespace=None):
136 namespace = super().parse_args(args, namespace)
137 # Code duplication; too hard to implement at shared _InputOutputParser level
138 if not namespace.useGen3:
139 warnings.warn("The --gen2 flag is deprecated; it will be removed after release 23.",
140 category=FutureWarning)
141 return namespace
144class _FormattedType:
145 """An argparse type converter that requires strings in a particular format.
147 Leaves the input as a string if it matches, else raises `argparse.ArgumentTypeError`.
149 Parameters
150 ----------
151 fmt : `str`
152 A regular expression that values must satisfy to be accepted. The *entire* string must match the
153 expression in order to pass.
154 msg : `str`
155 An error string to display for invalid values. The first "%s" shall be filled with the
156 invalid argument.
157 """
158 def __init__(self, fmt, msg='"%s" does not have the expected format.'):
159 fullFormat = fmt
160 if not fullFormat.startswith('^'):
161 fullFormat = '^' + fullFormat
162 if not fullFormat.endswith('$'):
163 fullFormat += '$'
164 self._format = re.compile(fullFormat)
165 self._message = msg
167 def __call__(self, value):
168 if self._format.match(value):
169 return value
170 else:
171 raise argparse.ArgumentTypeError(self._message % value)
174class _DatasetAction(argparse.Action):
175 """A converter for dataset arguments.
177 Not an argparse type converter so that the ``choices`` parameter can be
178 expressed using strings; ``choices`` checks happen after type conversion
179 but before actions.
180 """
181 def __call__(self, _parser, namespace, values, _option_string=None):
182 setattr(namespace, self.dest, Dataset(values))
185def runApVerify(cmdLine=None):
186 """Execute the AP pipeline while handling metrics.
188 This is the main function for ``ap_verify``, and handles logging,
189 command-line argument parsing, pipeline execution, and metrics
190 generation.
192 Parameters
193 ----------
194 cmdLine : `list` of `str`
195 an optional command line used to execute `runApVerify` from other
196 Python code. If `None`, `sys.argv` will be used.
198 Returns
199 -------
200 nFailed : `int`
201 The number of data IDs that were not successfully processed, up to 127,
202 or 127 if the task runner framework failed.
203 """
204 _configure_logger()
205 log = _LOG.getChild('main')
206 # TODO: what is LSST's policy on exceptions escaping into main()?
207 args = _ApVerifyParser().parse_args(args=cmdLine)
208 log.debug('Command-line arguments: %s', args)
210 if args.useGen3:
211 workspace = WorkspaceGen3(args.output)
212 ingestDatasetGen3(args.dataset, workspace, processes=args.processes)
213 log.info('Running pipeline...')
214 # Gen 3 pipeline includes both AP and metrics
215 return runApPipeGen3(workspace, args, processes=args.processes)
216 else:
217 workspace = WorkspaceGen2(args.output)
218 ingestDataset(args.dataset, workspace)
219 log.info('Running pipeline...')
220 apPipeResults = runApPipeGen2(workspace, args, processes=args.processes)
221 computeMetrics(workspace, apPipeResults.parsedCmd.id, args)
222 return _getCmdLineExitStatus(apPipeResults.resultList)
225def _getCmdLineExitStatus(resultList):
226 """Return the exit status following the conventions of
227 :ref:`running a CmdLineTask from the command line
228 <command-line-task-argument-reference>`.
230 Parameters
231 ----------
232 resultList : `list` [`Struct`] or `None`
233 A list of `Struct`, as returned by `ApPipeTask.parseAndRun`. Each
234 element must contain at least an ``exitStatus`` member.
236 Returns
237 -------
238 exitStatus : `int`
239 The number of failed runs in ``resultList``, up to 127, or 127 if
240 ``resultList`` is `None`.
241 """
242 if resultList:
243 # ApPipeTaskRunner does not override default results handling, exitStatus always defined
244 return min(127, sum(((res.exitStatus != 0) for res in resultList)))
245 else:
246 return 127
249def runIngestion(cmdLine=None):
250 """Ingest a dataset, but do not process it.
252 This is the main function for ``ingest_dataset``, and handles logging,
253 command-line argument parsing, and ingestion.
255 Parameters
256 ----------
257 cmdLine : `list` of `str`
258 an optional command line used to execute `runIngestion` from other
259 Python code. If `None`, `sys.argv` will be used.
260 """
261 _configure_logger()
262 log = _LOG.getChild('ingest')
263 # TODO: what is LSST's policy on exceptions escaping into main()?
264 args = _IngestOnlyParser().parse_args(args=cmdLine)
265 log.debug('Command-line arguments: %s', args)
267 if args.useGen3:
268 workspace = WorkspaceGen3(args.output)
269 ingestDatasetGen3(args.dataset, workspace, processes=args.processes)
270 else:
271 workspace = WorkspaceGen2(args.output)
272 ingestDataset(args.dataset, workspace)