Coverage for python/lsst/ap/verify/ap_verify.py : 31%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Command-line program for running and analyzing AP pipeline.
26In addition to containing ap_verify's main function, this module manages
27command-line argument parsing.
28"""
30__all__ = ["runApVerify", "runIngestion"]
32import argparse
33import re
35import lsst.log
37from .dataset import Dataset
38from .ingestion import ingestDataset, ingestDatasetGen3
39from .metrics import MetricsParser, computeMetrics
40from .pipeline_driver import ApPipeParser, runApPipe
41from .workspace import WorkspaceGen2, WorkspaceGen3
44class _InputOutputParser(argparse.ArgumentParser):
45 """An argument parser for program-wide input and output.
47 This parser is not complete, and is designed to be passed to another parser
48 using the `parent` parameter.
49 """
51 def __init__(self):
52 # Help and documentation will be handled by main program's parser
53 argparse.ArgumentParser.__init__(self, add_help=False)
54 self.add_argument('--dataset', action=_DatasetAction, choices=Dataset.getSupportedDatasets(),
55 required=True, help='The source of data to pass through the pipeline.')
56 self.add_argument('--output', required=True,
57 help='The location of the workspace to use for pipeline repositories.')
59 gen23 = self.add_mutually_exclusive_group()
60 # Because store_true and store_false use the same dest, add explicit
61 # default to avoid ambiguity.
62 gen23.add_argument('--gen2', dest='useGen3', action='store_false', default=False,
63 help='Handle the ap_verify dataset using the Gen 2 framework (default).')
64 gen23.add_argument('--gen3', dest='useGen3', action='store_true', default=False,
65 help='Handle the ap_verify dataset using the Gen 3 framework.')
68class _ApVerifyParser(argparse.ArgumentParser):
69 """An argument parser for data needed by the main ap_verify program.
70 """
72 def __init__(self):
73 argparse.ArgumentParser.__init__(
74 self,
75 description='Executes the LSST DM AP pipeline and analyzes its performance using metrics.',
76 epilog='',
77 parents=[_InputOutputParser(), ApPipeParser(), MetricsParser()],
78 add_help=True)
81class _IngestOnlyParser(argparse.ArgumentParser):
82 """An argument parser for data needed by dataset ingestion.
83 """
85 def __init__(self):
86 argparse.ArgumentParser.__init__(
87 self,
88 description='Ingests an ap_verify dataset into a pair of Butler repositories. '
89 'The program will create repository(ies) appropriate for --gen2 or --gen3 '
90 'in subdirectories of <OUTPUT>. '
91 'These repositories may be used directly by ap_verify.py by '
92 'passing the same --output argument, or by other programs that accept '
93 'Butler repositories as input.',
94 epilog='',
95 parents=[_InputOutputParser()],
96 add_help=True)
99class _FormattedType:
100 """An argparse type converter that requires strings in a particular format.
102 Leaves the input as a string if it matches, else raises `argparse.ArgumentTypeError`.
104 Parameters
105 ----------
106 fmt : `str`
107 A regular expression that values must satisfy to be accepted. The *entire* string must match the
108 expression in order to pass.
109 msg : `str`
110 An error string to display for invalid values. The first "%s" shall be filled with the
111 invalid argument.
112 """
113 def __init__(self, fmt, msg='"%s" does not have the expected format.'):
114 fullFormat = fmt
115 if not fullFormat.startswith('^'):
116 fullFormat = '^' + fullFormat
117 if not fullFormat.endswith('$'):
118 fullFormat += '$'
119 self._format = re.compile(fullFormat)
120 self._message = msg
122 def __call__(self, value):
123 if self._format.match(value):
124 return value
125 else:
126 raise argparse.ArgumentTypeError(self._message % value)
129class _DatasetAction(argparse.Action):
130 """A converter for dataset arguments.
132 Not an argparse type converter so that the ``choices`` parameter can be
133 expressed using strings; ``choices`` checks happen after type conversion
134 but before actions.
135 """
136 def __call__(self, _parser, namespace, values, _option_string=None):
137 setattr(namespace, self.dest, Dataset(values))
140def runApVerify(cmdLine=None):
141 """Execute the AP pipeline while handling metrics.
143 This is the main function for ``ap_verify``, and handles logging,
144 command-line argument parsing, pipeline execution, and metrics
145 generation.
147 Parameters
148 ----------
149 cmdLine : `list` of `str`
150 an optional command line used to execute `runApVerify` from other
151 Python code. If `None`, `sys.argv` will be used.
153 Returns
154 -------
155 nFailed : `int`
156 The number of data IDs that were not successfully processed, up to 127,
157 or 127 if the task runner framework failed.
158 """
159 lsst.log.configure()
160 log = lsst.log.Log.getLogger('ap.verify.ap_verify.main')
161 # TODO: what is LSST's policy on exceptions escaping into main()?
162 args = _ApVerifyParser().parse_args(args=cmdLine)
163 log.debug('Command-line arguments: %s', args)
165 workspace = WorkspaceGen2(args.output)
166 ingestDataset(args.dataset, workspace)
168 log.info('Running pipeline...')
169 apPipeResults = runApPipe(workspace, args)
170 computeMetrics(workspace, apPipeResults.parsedCmd.id, args)
172 return _getCmdLineExitStatus(apPipeResults.resultList)
175def _getCmdLineExitStatus(resultList):
176 """Return the exit status following the conventions of
177 :ref:`running a CmdLineTask from the command line
178 <command-line-task-argument-reference>`.
180 Parameters
181 ----------
182 resultList : `list` [`Struct`] or `None`
183 A list of `Struct`, as returned by `ApPipeTask.parseAndRun`. Each
184 element must contain at least an ``exitStatus`` member.
186 Returns
187 -------
188 exitStatus : `int`
189 The number of failed runs in ``resultList``, up to 127, or 127 if
190 ``resultList`` is `None`.
191 """
192 if resultList:
193 # ApPipeTaskRunner does not override default results handling, exitStatus always defined
194 return min(127, sum(((res.exitStatus != 0) for res in resultList)))
195 else:
196 return 127
199def runIngestion(cmdLine=None):
200 """Ingest a dataset, but do not process it.
202 This is the main function for ``ingest_dataset``, and handles logging,
203 command-line argument parsing, and ingestion.
205 Parameters
206 ----------
207 cmdLine : `list` of `str`
208 an optional command line used to execute `runIngestion` from other
209 Python code. If `None`, `sys.argv` will be used.
210 """
211 lsst.log.configure()
212 log = lsst.log.Log.getLogger('ap.verify.ap_verify.ingest')
213 # TODO: what is LSST's policy on exceptions escaping into main()?
214 args = _IngestOnlyParser().parse_args(args=cmdLine)
215 log.debug('Command-line arguments: %s', args)
217 if args.useGen3:
218 workspace = WorkspaceGen3(args.output)
219 ingestDatasetGen3(args.dataset, workspace)
220 else:
221 workspace = WorkspaceGen2(args.output)
222 ingestDataset(args.dataset, workspace)