Coverage for python / lsst / ap / verify / ap_verify.py: 44%

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Command-line program for running and analyzing AP pipeline. 

25 

26In addition to containing ap_verify's main function, this module manages 

27command-line argument parsing. 

28""" 

29 

30__all__ = ["runApVerify", "runIngestion"] 

31 

32import argparse 

33import os 

34import sys 

35import logging 

36 

37import lsst.log 

38 

39from .dataset import Dataset 

40from .ingestion import ingestDatasetGen3, IngestionParser 

41from .pipeline_driver import ApPipeParser, runApPipeGen3, _getPipelineFile 

42from .workspace import WorkspaceGen3 

43 

44_LOG = logging.getLogger(__name__) 

45 

46 

47def _configure_logger(): 

48 """Configure Python logging. 

49 

50 Does basic Python logging configuration and 

51 forwards LSST logger to Python logging. 

52 """ 

53 logging.basicConfig(level=logging.INFO, stream=sys.stdout) 

54 lsst.log.configure_pylog_MDC("DEBUG", MDC_class=None) 

55 

56 

57class _InputOutputParser(argparse.ArgumentParser): 

58 """An argument parser for program-wide input and output. 

59 

60 This parser is not complete, and is designed to be passed to another parser 

61 using the `parent` parameter. 

62 """ 

63 

64 def __init__(self): 

65 # Help and documentation will be handled by main program's parser 

66 argparse.ArgumentParser.__init__(self, add_help=False) 

67 self.add_argument('--dataset', action=_DatasetAction, 

68 required=True, help='The source of data to pass through the pipeline.') 

69 self.add_argument('--output', required=True, 

70 help='The location of the workspace to use for pipeline repositories.') 

71 

72 

73class _ProcessingParser(argparse.ArgumentParser): 

74 """An argument parser for general run-time characteristics. 

75 

76 This parser is not complete, and is designed to be passed to another parser 

77 using the `parent` parameter. 

78 """ 

79 

80 def __init__(self): 

81 # Help and documentation will be handled by main program's parser 

82 argparse.ArgumentParser.__init__(self, add_help=False) 

83 self.add_argument("-j", "--processes", default=1, type=int, 

84 help="Number of processes to use.") 

85 

86 

87class _ApVerifyParser(argparse.ArgumentParser): 

88 """An argument parser for data needed by the main ap_verify program. 

89 """ 

90 

91 def __init__(self): 

92 argparse.ArgumentParser.__init__( 

93 self, 

94 description='Executes the LSST DM AP pipeline and analyzes its performance using metrics.', 

95 epilog='', 

96 parents=[IngestionParser(), _InputOutputParser(), _ProcessingParser(), ApPipeParser(), ], 

97 add_help=True) 

98 

99 

100class _IngestOnlyParser(argparse.ArgumentParser): 

101 """An argument parser for data needed by dataset ingestion. 

102 """ 

103 

104 def __init__(self): 

105 argparse.ArgumentParser.__init__( 

106 self, 

107 description='Ingests an ap_verify dataset into a repository. ' 

108 'The program will create a repository in the ``repo`` subdirectory of <OUTPUT>. ' 

109 'These repositories may be used directly by ap_verify.py by ' 

110 'passing the same --output argument, or by other programs that accept ' 

111 'Butler repositories as input.', 

112 epilog='', 

113 parents=[IngestionParser(), _InputOutputParser(), _ProcessingParser()], 

114 add_help=True) 

115 

116 

117class _DatasetAction(argparse.Action): 

118 """A converter for dataset arguments. 

119 

120 Not an argparse type converter so that the ``choices`` parameter can be 

121 expressed using strings; ``choices`` checks happen after type conversion 

122 but before actions. 

123 """ 

124 def __call__(self, _parser, namespace, values, _option_string=None): 

125 setattr(namespace, self.dest, Dataset(values)) 

126 

127 

128def runApVerify(cmdLine=None): 

129 """Execute the AP pipeline while handling metrics. 

130 

131 This is the main function for ``ap_verify``, and handles logging, 

132 command-line argument parsing, pipeline execution, and metrics 

133 generation. 

134 

135 Parameters 

136 ---------- 

137 cmdLine : `list` of `str` 

138 an optional command line used to execute `runApVerify` from other 

139 Python code. If `None`, `sys.argv` will be used. 

140 

141 Returns 

142 ------- 

143 nFailed : `int` 

144 The number of data IDs that were not successfully processed, up to 127, 

145 or 127 if the task runner framework failed. 

146 """ 

147 _configure_logger() 

148 log = _LOG.getChild('main') 

149 # TODO: what is LSST's policy on exceptions escaping into main()? 

150 args = _ApVerifyParser().parse_args(args=cmdLine) 

151 log.debug('Command-line arguments: %s', args) 

152 

153 workspace = WorkspaceGen3(args.output) 

154 # Set the pipeline name as extra parameter for SasquatchDatastore to used 

155 pipelineFile = _getPipelineFile(workspace, args) 

156 extra = dict(vars(args)['extra']) 

157 if 'pipeline' not in extra.keys(): 

158 extra['pipeline'] = os.path.basename(pipelineFile) 

159 

160 ingestDatasetGen3( 

161 args.dataset, workspace, args.namespace, args.restProxyUrl, 

162 extra=extra, processes=args.processes) 

163 log.info('Running pipeline...') 

164 # Gen 3 pipeline includes both AP and metrics 

165 return runApPipeGen3(workspace, args, processes=args.processes) 

166 

167 

168def runIngestion(cmdLine=None): 

169 """Ingest a dataset, but do not process it. 

170 

171 This is the main function for ``ingest_dataset``, and handles logging, 

172 command-line argument parsing, and ingestion. 

173 

174 Parameters 

175 ---------- 

176 cmdLine : `list` of `str` 

177 an optional command line used to execute `runIngestion` from other 

178 Python code. If `None`, `sys.argv` will be used. 

179 """ 

180 _configure_logger() 

181 log = _LOG.getChild('ingest') 

182 # TODO: what is LSST's policy on exceptions escaping into main()? 

183 args = _IngestOnlyParser().parse_args(args=cmdLine) 

184 log.debug('Command-line arguments: %s', args) 

185 

186 workspace = WorkspaceGen3(args.output) 

187 ingestDatasetGen3(args.dataset, workspace, processes=args.processes)