Coverage for python/lsst/ap/verify/ap_verify.py: 37%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

91 statements  

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Command-line program for running and analyzing AP pipeline. 

25 

26In addition to containing ap_verify's main function, this module manages 

27command-line argument parsing. 

28""" 

29 

30__all__ = ["runApVerify", "runIngestion"] 

31 

32import argparse 

33import re 

34import warnings 

35import sys 

36import logging 

37 

38import lsst.log 

39 

40from .dataset import Dataset 

41from .ingestion import ingestDataset, ingestDatasetGen3 

42from .metrics import MetricsParser, computeMetrics 

43from .pipeline_driver import ApPipeParser, runApPipeGen2, runApPipeGen3 

44from .workspace import WorkspaceGen2, WorkspaceGen3 

45 

46_LOG = logging.getLogger(__name__) 

47 

48 

49def _configure_logger(): 

50 """Configure Python logging. 

51 

52 Does basic Python logging configuration and 

53 forwards LSST logger to Python logging. 

54 """ 

55 logging.basicConfig(level=logging.INFO, stream=sys.stdout) 

56 lsst.log.configure_pylog_MDC("DEBUG", MDC_class=None) 

57 

58 

59class _InputOutputParser(argparse.ArgumentParser): 

60 """An argument parser for program-wide input and output. 

61 

62 This parser is not complete, and is designed to be passed to another parser 

63 using the `parent` parameter. 

64 """ 

65 

66 def __init__(self): 

67 # Help and documentation will be handled by main program's parser 

68 argparse.ArgumentParser.__init__(self, add_help=False) 

69 self.add_argument('--dataset', action=_DatasetAction, 

70 required=True, help='The source of data to pass through the pipeline.') 

71 self.add_argument('--output', required=True, 

72 help='The location of the workspace to use for pipeline repositories.') 

73 

74 gen23 = self.add_mutually_exclusive_group() 

75 # Because store_true and store_false use the same dest, add explicit 

76 # default to avoid ambiguity. 

77 gen23.add_argument('--gen2', dest='useGen3', action='store_false', default=True, 

78 help='Handle the ap_verify dataset using the Gen 2 framework (default).') 

79 gen23.add_argument('--gen3', dest='useGen3', action='store_true', default=True, 

80 help='Handle the ap_verify dataset using the Gen 3 framework (default).') 

81 

82 

83class _ProcessingParser(argparse.ArgumentParser): 

84 """An argument parser for general run-time characteristics. 

85 

86 This parser is not complete, and is designed to be passed to another parser 

87 using the `parent` parameter. 

88 """ 

89 

90 def __init__(self): 

91 # Help and documentation will be handled by main program's parser 

92 argparse.ArgumentParser.__init__(self, add_help=False) 

93 self.add_argument("-j", "--processes", default=1, type=int, 

94 help="Number of processes to use.") 

95 

96 

97class _ApVerifyParser(argparse.ArgumentParser): 

98 """An argument parser for data needed by the main ap_verify program. 

99 """ 

100 

101 def __init__(self): 

102 argparse.ArgumentParser.__init__( 

103 self, 

104 description='Executes the LSST DM AP pipeline and analyzes its performance using metrics.', 

105 epilog='', 

106 parents=[_InputOutputParser(), _ProcessingParser(), ApPipeParser(), MetricsParser()], 

107 add_help=True) 

108 

109 def parse_args(self, args=None, namespace=None): 

110 namespace = super().parse_args(args, namespace) 

111 # Code duplication; too hard to implement at shared _InputOutputParser level 

112 if not namespace.useGen3: 

113 warnings.warn("The --gen2 flag is deprecated; it will be removed after release 23.", 

114 category=FutureWarning) 

115 return namespace 

116 

117 

118class _IngestOnlyParser(argparse.ArgumentParser): 

119 """An argument parser for data needed by dataset ingestion. 

120 """ 

121 

122 def __init__(self): 

123 argparse.ArgumentParser.__init__( 

124 self, 

125 description='Ingests an ap_verify dataset into a pair of Butler repositories. ' 

126 'The program will create repository(ies) appropriate for --gen2 or --gen3 ' 

127 'in subdirectories of <OUTPUT>. ' 

128 'These repositories may be used directly by ap_verify.py by ' 

129 'passing the same --output argument, or by other programs that accept ' 

130 'Butler repositories as input.', 

131 epilog='', 

132 parents=[_InputOutputParser(), _ProcessingParser()], 

133 add_help=True) 

134 

135 def parse_args(self, args=None, namespace=None): 

136 namespace = super().parse_args(args, namespace) 

137 # Code duplication; too hard to implement at shared _InputOutputParser level 

138 if not namespace.useGen3: 

139 warnings.warn("The --gen2 flag is deprecated; it will be removed after release 23.", 

140 category=FutureWarning) 

141 return namespace 

142 

143 

144class _FormattedType: 

145 """An argparse type converter that requires strings in a particular format. 

146 

147 Leaves the input as a string if it matches, else raises `argparse.ArgumentTypeError`. 

148 

149 Parameters 

150 ---------- 

151 fmt : `str` 

152 A regular expression that values must satisfy to be accepted. The *entire* string must match the 

153 expression in order to pass. 

154 msg : `str` 

155 An error string to display for invalid values. The first "%s" shall be filled with the 

156 invalid argument. 

157 """ 

158 def __init__(self, fmt, msg='"%s" does not have the expected format.'): 

159 fullFormat = fmt 

160 if not fullFormat.startswith('^'): 

161 fullFormat = '^' + fullFormat 

162 if not fullFormat.endswith('$'): 

163 fullFormat += '$' 

164 self._format = re.compile(fullFormat) 

165 self._message = msg 

166 

167 def __call__(self, value): 

168 if self._format.match(value): 

169 return value 

170 else: 

171 raise argparse.ArgumentTypeError(self._message % value) 

172 

173 

174class _DatasetAction(argparse.Action): 

175 """A converter for dataset arguments. 

176 

177 Not an argparse type converter so that the ``choices`` parameter can be 

178 expressed using strings; ``choices`` checks happen after type conversion 

179 but before actions. 

180 """ 

181 def __call__(self, _parser, namespace, values, _option_string=None): 

182 setattr(namespace, self.dest, Dataset(values)) 

183 

184 

185def runApVerify(cmdLine=None): 

186 """Execute the AP pipeline while handling metrics. 

187 

188 This is the main function for ``ap_verify``, and handles logging, 

189 command-line argument parsing, pipeline execution, and metrics 

190 generation. 

191 

192 Parameters 

193 ---------- 

194 cmdLine : `list` of `str` 

195 an optional command line used to execute `runApVerify` from other 

196 Python code. If `None`, `sys.argv` will be used. 

197 

198 Returns 

199 ------- 

200 nFailed : `int` 

201 The number of data IDs that were not successfully processed, up to 127, 

202 or 127 if the task runner framework failed. 

203 """ 

204 _configure_logger() 

205 log = _LOG.getChild('main') 

206 # TODO: what is LSST's policy on exceptions escaping into main()? 

207 args = _ApVerifyParser().parse_args(args=cmdLine) 

208 log.debug('Command-line arguments: %s', args) 

209 

210 if args.useGen3: 

211 workspace = WorkspaceGen3(args.output) 

212 ingestDatasetGen3(args.dataset, workspace, processes=args.processes) 

213 log.info('Running pipeline...') 

214 # Gen 3 pipeline includes both AP and metrics 

215 return runApPipeGen3(workspace, args, processes=args.processes) 

216 else: 

217 workspace = WorkspaceGen2(args.output) 

218 ingestDataset(args.dataset, workspace) 

219 log.info('Running pipeline...') 

220 apPipeResults = runApPipeGen2(workspace, args, processes=args.processes) 

221 computeMetrics(workspace, apPipeResults.parsedCmd.id, args) 

222 return _getCmdLineExitStatus(apPipeResults.resultList) 

223 

224 

225def _getCmdLineExitStatus(resultList): 

226 """Return the exit status following the conventions of 

227 :ref:`running a CmdLineTask from the command line 

228 <command-line-task-argument-reference>`. 

229 

230 Parameters 

231 ---------- 

232 resultList : `list` [`Struct`] or `None` 

233 A list of `Struct`, as returned by `ApPipeTask.parseAndRun`. Each 

234 element must contain at least an ``exitStatus`` member. 

235 

236 Returns 

237 ------- 

238 exitStatus : `int` 

239 The number of failed runs in ``resultList``, up to 127, or 127 if 

240 ``resultList`` is `None`. 

241 """ 

242 if resultList: 

243 # ApPipeTaskRunner does not override default results handling, exitStatus always defined 

244 return min(127, sum(((res.exitStatus != 0) for res in resultList))) 

245 else: 

246 return 127 

247 

248 

249def runIngestion(cmdLine=None): 

250 """Ingest a dataset, but do not process it. 

251 

252 This is the main function for ``ingest_dataset``, and handles logging, 

253 command-line argument parsing, and ingestion. 

254 

255 Parameters 

256 ---------- 

257 cmdLine : `list` of `str` 

258 an optional command line used to execute `runIngestion` from other 

259 Python code. If `None`, `sys.argv` will be used. 

260 """ 

261 _configure_logger() 

262 log = _LOG.getChild('ingest') 

263 # TODO: what is LSST's policy on exceptions escaping into main()? 

264 args = _IngestOnlyParser().parse_args(args=cmdLine) 

265 log.debug('Command-line arguments: %s', args) 

266 

267 if args.useGen3: 

268 workspace = WorkspaceGen3(args.output) 

269 ingestDatasetGen3(args.dataset, workspace, processes=args.processes) 

270 else: 

271 workspace = WorkspaceGen2(args.output) 

272 ingestDataset(args.dataset, workspace)