Coverage for python/lsst/ap/verify/ap_verify.py: 28%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Command-line program for running and analyzing AP pipeline.
26In addition to containing ap_verify's main function, this module manages
27command-line argument parsing.
28"""
30__all__ = ["runApVerify", "runIngestion"]
32import argparse
33import re
34import warnings
36import lsst.log
38from .dataset import Dataset
39from .ingestion import ingestDataset, ingestDatasetGen3
40from .metrics import MetricsParser, computeMetrics
41from .pipeline_driver import ApPipeParser, runApPipeGen2, runApPipeGen3
42from .workspace import WorkspaceGen2, WorkspaceGen3
45class _InputOutputParser(argparse.ArgumentParser):
46 """An argument parser for program-wide input and output.
48 This parser is not complete, and is designed to be passed to another parser
49 using the `parent` parameter.
50 """
52 def __init__(self):
53 # Help and documentation will be handled by main program's parser
54 argparse.ArgumentParser.__init__(self, add_help=False)
55 self.add_argument('--dataset', action=_DatasetAction,
56 required=True, help='The source of data to pass through the pipeline.')
57 self.add_argument('--output', required=True,
58 help='The location of the workspace to use for pipeline repositories.')
60 gen23 = self.add_mutually_exclusive_group()
61 # Because store_true and store_false use the same dest, add explicit
62 # default to avoid ambiguity.
63 gen23.add_argument('--gen2', dest='useGen3', action='store_false', default=True,
64 help='Handle the ap_verify dataset using the Gen 2 framework (default).')
65 gen23.add_argument('--gen3', dest='useGen3', action='store_true', default=True,
66 help='Handle the ap_verify dataset using the Gen 3 framework (default).')
69class _ProcessingParser(argparse.ArgumentParser):
70 """An argument parser for general run-time characteristics.
72 This parser is not complete, and is designed to be passed to another parser
73 using the `parent` parameter.
74 """
76 def __init__(self):
77 # Help and documentation will be handled by main program's parser
78 argparse.ArgumentParser.__init__(self, add_help=False)
79 self.add_argument("-j", "--processes", default=1, type=int,
80 help="Number of processes to use.")
83class _ApVerifyParser(argparse.ArgumentParser):
84 """An argument parser for data needed by the main ap_verify program.
85 """
87 def __init__(self):
88 argparse.ArgumentParser.__init__(
89 self,
90 description='Executes the LSST DM AP pipeline and analyzes its performance using metrics.',
91 epilog='',
92 parents=[_InputOutputParser(), _ProcessingParser(), ApPipeParser(), MetricsParser()],
93 add_help=True)
95 def parse_args(self, args=None, namespace=None):
96 namespace = super().parse_args(args, namespace)
97 # Code duplication; too hard to implement at shared _InputOutputParser level
98 if not namespace.useGen3:
99 warnings.warn("The --gen2 flag is deprecated; it will be removed after release 23.",
100 category=FutureWarning)
101 return namespace
104class _IngestOnlyParser(argparse.ArgumentParser):
105 """An argument parser for data needed by dataset ingestion.
106 """
108 def __init__(self):
109 argparse.ArgumentParser.__init__(
110 self,
111 description='Ingests an ap_verify dataset into a pair of Butler repositories. '
112 'The program will create repository(ies) appropriate for --gen2 or --gen3 '
113 'in subdirectories of <OUTPUT>. '
114 'These repositories may be used directly by ap_verify.py by '
115 'passing the same --output argument, or by other programs that accept '
116 'Butler repositories as input.',
117 epilog='',
118 parents=[_InputOutputParser(), _ProcessingParser()],
119 add_help=True)
121 def parse_args(self, args=None, namespace=None):
122 namespace = super().parse_args(args, namespace)
123 # Code duplication; too hard to implement at shared _InputOutputParser level
124 if not namespace.useGen3:
125 warnings.warn("The --gen2 flag is deprecated; it will be removed after release 23.",
126 category=FutureWarning)
127 return namespace
130class _FormattedType:
131 """An argparse type converter that requires strings in a particular format.
133 Leaves the input as a string if it matches, else raises `argparse.ArgumentTypeError`.
135 Parameters
136 ----------
137 fmt : `str`
138 A regular expression that values must satisfy to be accepted. The *entire* string must match the
139 expression in order to pass.
140 msg : `str`
141 An error string to display for invalid values. The first "%s" shall be filled with the
142 invalid argument.
143 """
144 def __init__(self, fmt, msg='"%s" does not have the expected format.'):
145 fullFormat = fmt
146 if not fullFormat.startswith('^'):
147 fullFormat = '^' + fullFormat
148 if not fullFormat.endswith('$'):
149 fullFormat += '$'
150 self._format = re.compile(fullFormat)
151 self._message = msg
153 def __call__(self, value):
154 if self._format.match(value):
155 return value
156 else:
157 raise argparse.ArgumentTypeError(self._message % value)
160class _DatasetAction(argparse.Action):
161 """A converter for dataset arguments.
163 Not an argparse type converter so that the ``choices`` parameter can be
164 expressed using strings; ``choices`` checks happen after type conversion
165 but before actions.
166 """
167 def __call__(self, _parser, namespace, values, _option_string=None):
168 setattr(namespace, self.dest, Dataset(values))
171def runApVerify(cmdLine=None):
172 """Execute the AP pipeline while handling metrics.
174 This is the main function for ``ap_verify``, and handles logging,
175 command-line argument parsing, pipeline execution, and metrics
176 generation.
178 Parameters
179 ----------
180 cmdLine : `list` of `str`
181 an optional command line used to execute `runApVerify` from other
182 Python code. If `None`, `sys.argv` will be used.
184 Returns
185 -------
186 nFailed : `int`
187 The number of data IDs that were not successfully processed, up to 127,
188 or 127 if the task runner framework failed.
189 """
190 lsst.log.configure()
191 log = lsst.log.Log.getLogger('ap.verify.ap_verify.main')
192 # TODO: what is LSST's policy on exceptions escaping into main()?
193 args = _ApVerifyParser().parse_args(args=cmdLine)
194 log.debug('Command-line arguments: %s', args)
196 if args.useGen3:
197 workspace = WorkspaceGen3(args.output)
198 ingestDatasetGen3(args.dataset, workspace, processes=args.processes)
199 log.info('Running pipeline...')
200 # Gen 3 pipeline includes both AP and metrics
201 return runApPipeGen3(workspace, args, processes=args.processes)
202 else:
203 workspace = WorkspaceGen2(args.output)
204 ingestDataset(args.dataset, workspace)
205 log.info('Running pipeline...')
206 apPipeResults = runApPipeGen2(workspace, args, processes=args.processes)
207 computeMetrics(workspace, apPipeResults.parsedCmd.id, args)
208 return _getCmdLineExitStatus(apPipeResults.resultList)
211def _getCmdLineExitStatus(resultList):
212 """Return the exit status following the conventions of
213 :ref:`running a CmdLineTask from the command line
214 <command-line-task-argument-reference>`.
216 Parameters
217 ----------
218 resultList : `list` [`Struct`] or `None`
219 A list of `Struct`, as returned by `ApPipeTask.parseAndRun`. Each
220 element must contain at least an ``exitStatus`` member.
222 Returns
223 -------
224 exitStatus : `int`
225 The number of failed runs in ``resultList``, up to 127, or 127 if
226 ``resultList`` is `None`.
227 """
228 if resultList:
229 # ApPipeTaskRunner does not override default results handling, exitStatus always defined
230 return min(127, sum(((res.exitStatus != 0) for res in resultList)))
231 else:
232 return 127
235def runIngestion(cmdLine=None):
236 """Ingest a dataset, but do not process it.
238 This is the main function for ``ingest_dataset``, and handles logging,
239 command-line argument parsing, and ingestion.
241 Parameters
242 ----------
243 cmdLine : `list` of `str`
244 an optional command line used to execute `runIngestion` from other
245 Python code. If `None`, `sys.argv` will be used.
246 """
247 lsst.log.configure()
248 log = lsst.log.Log.getLogger('ap.verify.ap_verify.ingest')
249 # TODO: what is LSST's policy on exceptions escaping into main()?
250 args = _IngestOnlyParser().parse_args(args=cmdLine)
251 log.debug('Command-line arguments: %s', args)
253 if args.useGen3:
254 workspace = WorkspaceGen3(args.output)
255 ingestDatasetGen3(args.dataset, workspace, processes=args.processes)
256 else:
257 workspace = WorkspaceGen2(args.output)
258 ingestDataset(args.dataset, workspace)