Coverage for python/lsst/ap/verify/ap_verify.py: 59%
48 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-07 02:39 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-07 02:39 -0700
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Command-line program for running and analyzing AP pipeline.
26In addition to containing ap_verify's main function, this module manages
27command-line argument parsing.
28"""
30__all__ = ["runApVerify", "runIngestion"]
32import argparse
33import sys
34import logging
36import lsst.log
38from .dataset import Dataset
39from .ingestion import ingestDatasetGen3
40from .pipeline_driver import ApPipeParser, runApPipeGen3
41from .workspace import WorkspaceGen3
43_LOG = logging.getLogger(__name__)
46def _configure_logger():
47 """Configure Python logging.
49 Does basic Python logging configuration and
50 forwards LSST logger to Python logging.
51 """
52 logging.basicConfig(level=logging.INFO, stream=sys.stdout)
53 lsst.log.configure_pylog_MDC("DEBUG", MDC_class=None)
56class _InputOutputParser(argparse.ArgumentParser):
57 """An argument parser for program-wide input and output.
59 This parser is not complete, and is designed to be passed to another parser
60 using the `parent` parameter.
61 """
63 def __init__(self):
64 # Help and documentation will be handled by main program's parser
65 argparse.ArgumentParser.__init__(self, add_help=False)
66 self.add_argument('--dataset', action=_DatasetAction,
67 required=True, help='The source of data to pass through the pipeline.')
68 self.add_argument('--output', required=True,
69 help='The location of the workspace to use for pipeline repositories.')
72class _ProcessingParser(argparse.ArgumentParser):
73 """An argument parser for general run-time characteristics.
75 This parser is not complete, and is designed to be passed to another parser
76 using the `parent` parameter.
77 """
79 def __init__(self):
80 # Help and documentation will be handled by main program's parser
81 argparse.ArgumentParser.__init__(self, add_help=False)
82 self.add_argument("-j", "--processes", default=1, type=int,
83 help="Number of processes to use.")
86class _ApVerifyParser(argparse.ArgumentParser):
87 """An argument parser for data needed by the main ap_verify program.
88 """
90 def __init__(self):
91 argparse.ArgumentParser.__init__(
92 self,
93 description='Executes the LSST DM AP pipeline and analyzes its performance using metrics.',
94 epilog='',
95 parents=[_InputOutputParser(), _ProcessingParser(), ApPipeParser(), ],
96 add_help=True)
99class _IngestOnlyParser(argparse.ArgumentParser):
100 """An argument parser for data needed by dataset ingestion.
101 """
103 def __init__(self):
104 argparse.ArgumentParser.__init__(
105 self,
106 description='Ingests an ap_verify dataset into a repository. '
107 'The program will create a repository in the ``repo`` subdirectory of <OUTPUT>. '
108 'These repositories may be used directly by ap_verify.py by '
109 'passing the same --output argument, or by other programs that accept '
110 'Butler repositories as input.',
111 epilog='',
112 parents=[_InputOutputParser(), _ProcessingParser()],
113 add_help=True)
116class _DatasetAction(argparse.Action):
117 """A converter for dataset arguments.
119 Not an argparse type converter so that the ``choices`` parameter can be
120 expressed using strings; ``choices`` checks happen after type conversion
121 but before actions.
122 """
123 def __call__(self, _parser, namespace, values, _option_string=None):
124 setattr(namespace, self.dest, Dataset(values))
127def runApVerify(cmdLine=None):
128 """Execute the AP pipeline while handling metrics.
130 This is the main function for ``ap_verify``, and handles logging,
131 command-line argument parsing, pipeline execution, and metrics
132 generation.
134 Parameters
135 ----------
136 cmdLine : `list` of `str`
137 an optional command line used to execute `runApVerify` from other
138 Python code. If `None`, `sys.argv` will be used.
140 Returns
141 -------
142 nFailed : `int`
143 The number of data IDs that were not successfully processed, up to 127,
144 or 127 if the task runner framework failed.
145 """
146 _configure_logger()
147 log = _LOG.getChild('main')
148 # TODO: what is LSST's policy on exceptions escaping into main()?
149 args = _ApVerifyParser().parse_args(args=cmdLine)
150 log.debug('Command-line arguments: %s', args)
152 workspace = WorkspaceGen3(args.output)
153 ingestDatasetGen3(args.dataset, workspace, processes=args.processes)
154 log.info('Running pipeline...')
155 # Gen 3 pipeline includes both AP and metrics
156 return runApPipeGen3(workspace, args, processes=args.processes)
159def runIngestion(cmdLine=None):
160 """Ingest a dataset, but do not process it.
162 This is the main function for ``ingest_dataset``, and handles logging,
163 command-line argument parsing, and ingestion.
165 Parameters
166 ----------
167 cmdLine : `list` of `str`
168 an optional command line used to execute `runIngestion` from other
169 Python code. If `None`, `sys.argv` will be used.
170 """
171 _configure_logger()
172 log = _LOG.getChild('ingest')
173 # TODO: what is LSST's policy on exceptions escaping into main()?
174 args = _IngestOnlyParser().parse_args(args=cmdLine)
175 log.debug('Command-line arguments: %s', args)
177 workspace = WorkspaceGen3(args.output)
178 ingestDatasetGen3(args.dataset, workspace, processes=args.processes)