Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Command-line program for running and analyzing AP pipeline. 

25 

26In addition to containing ap_verify's main function, this module manages 

27command-line argument parsing. 

28""" 

29 

30__all__ = ["runApVerify", "runIngestion"] 

31 

32import argparse 

33import re 

34 

35import lsst.log 

36 

37from .dataset import Dataset 

38from .ingestion import ingestDataset, ingestDatasetGen3 

39from .metrics import MetricsParser, computeMetrics 

40from .pipeline_driver import ApPipeParser, runApPipeGen2, runApPipeGen3 

41from .workspace import WorkspaceGen2, WorkspaceGen3 

42 

43 

44class _InputOutputParser(argparse.ArgumentParser): 

45 """An argument parser for program-wide input and output. 

46 

47 This parser is not complete, and is designed to be passed to another parser 

48 using the `parent` parameter. 

49 """ 

50 

51 def __init__(self): 

52 # Help and documentation will be handled by main program's parser 

53 argparse.ArgumentParser.__init__(self, add_help=False) 

54 self.add_argument('--dataset', action=_DatasetAction, choices=Dataset.getSupportedDatasets(), 

55 required=True, help='The source of data to pass through the pipeline.') 

56 self.add_argument('--output', required=True, 

57 help='The location of the workspace to use for pipeline repositories.') 

58 

59 gen23 = self.add_mutually_exclusive_group() 

60 # Because store_true and store_false use the same dest, add explicit 

61 # default to avoid ambiguity. 

62 gen23.add_argument('--gen2', dest='useGen3', action='store_false', default=False, 

63 help='Handle the ap_verify dataset using the Gen 2 framework (default).') 

64 gen23.add_argument('--gen3', dest='useGen3', action='store_true', default=False, 

65 help='Handle the ap_verify dataset using the Gen 3 framework.') 

66 

67 

68class _ApVerifyParser(argparse.ArgumentParser): 

69 """An argument parser for data needed by the main ap_verify program. 

70 """ 

71 

72 def __init__(self): 

73 argparse.ArgumentParser.__init__( 

74 self, 

75 description='Executes the LSST DM AP pipeline and analyzes its performance using metrics.', 

76 epilog='', 

77 parents=[_InputOutputParser(), ApPipeParser(), MetricsParser()], 

78 add_help=True) 

79 

80 

81class _IngestOnlyParser(argparse.ArgumentParser): 

82 """An argument parser for data needed by dataset ingestion. 

83 """ 

84 

85 def __init__(self): 

86 argparse.ArgumentParser.__init__( 

87 self, 

88 description='Ingests an ap_verify dataset into a pair of Butler repositories. ' 

89 'The program will create repository(ies) appropriate for --gen2 or --gen3 ' 

90 'in subdirectories of <OUTPUT>. ' 

91 'These repositories may be used directly by ap_verify.py by ' 

92 'passing the same --output argument, or by other programs that accept ' 

93 'Butler repositories as input.', 

94 epilog='', 

95 parents=[_InputOutputParser()], 

96 add_help=True) 

97 

98 

99class _FormattedType: 

100 """An argparse type converter that requires strings in a particular format. 

101 

102 Leaves the input as a string if it matches, else raises `argparse.ArgumentTypeError`. 

103 

104 Parameters 

105 ---------- 

106 fmt : `str` 

107 A regular expression that values must satisfy to be accepted. The *entire* string must match the 

108 expression in order to pass. 

109 msg : `str` 

110 An error string to display for invalid values. The first "%s" shall be filled with the 

111 invalid argument. 

112 """ 

113 def __init__(self, fmt, msg='"%s" does not have the expected format.'): 

114 fullFormat = fmt 

115 if not fullFormat.startswith('^'): 

116 fullFormat = '^' + fullFormat 

117 if not fullFormat.endswith('$'): 

118 fullFormat += '$' 

119 self._format = re.compile(fullFormat) 

120 self._message = msg 

121 

122 def __call__(self, value): 

123 if self._format.match(value): 

124 return value 

125 else: 

126 raise argparse.ArgumentTypeError(self._message % value) 

127 

128 

129class _DatasetAction(argparse.Action): 

130 """A converter for dataset arguments. 

131 

132 Not an argparse type converter so that the ``choices`` parameter can be 

133 expressed using strings; ``choices`` checks happen after type conversion 

134 but before actions. 

135 """ 

136 def __call__(self, _parser, namespace, values, _option_string=None): 

137 setattr(namespace, self.dest, Dataset(values)) 

138 

139 

140def runApVerify(cmdLine=None): 

141 """Execute the AP pipeline while handling metrics. 

142 

143 This is the main function for ``ap_verify``, and handles logging, 

144 command-line argument parsing, pipeline execution, and metrics 

145 generation. 

146 

147 Parameters 

148 ---------- 

149 cmdLine : `list` of `str` 

150 an optional command line used to execute `runApVerify` from other 

151 Python code. If `None`, `sys.argv` will be used. 

152 

153 Returns 

154 ------- 

155 nFailed : `int` 

156 The number of data IDs that were not successfully processed, up to 127, 

157 or 127 if the task runner framework failed. 

158 """ 

159 lsst.log.configure() 

160 log = lsst.log.Log.getLogger('ap.verify.ap_verify.main') 

161 # TODO: what is LSST's policy on exceptions escaping into main()? 

162 args = _ApVerifyParser().parse_args(args=cmdLine) 

163 log.debug('Command-line arguments: %s', args) 

164 

165 if args.useGen3: 

166 workspace = WorkspaceGen3(args.output) 

167 ingestDatasetGen3(args.dataset, workspace) 

168 log.info('Running pipeline...') 

169 # Gen 3 pipeline includes both AP and metrics 

170 return runApPipeGen3(workspace, args) 

171 else: 

172 workspace = WorkspaceGen2(args.output) 

173 ingestDataset(args.dataset, workspace) 

174 log.info('Running pipeline...') 

175 apPipeResults = runApPipeGen2(workspace, args) 

176 computeMetrics(workspace, apPipeResults.parsedCmd.id, args) 

177 return _getCmdLineExitStatus(apPipeResults.resultList) 

178 

179 

180def _getCmdLineExitStatus(resultList): 

181 """Return the exit status following the conventions of 

182 :ref:`running a CmdLineTask from the command line 

183 <command-line-task-argument-reference>`. 

184 

185 Parameters 

186 ---------- 

187 resultList : `list` [`Struct`] or `None` 

188 A list of `Struct`, as returned by `ApPipeTask.parseAndRun`. Each 

189 element must contain at least an ``exitStatus`` member. 

190 

191 Returns 

192 ------- 

193 exitStatus : `int` 

194 The number of failed runs in ``resultList``, up to 127, or 127 if 

195 ``resultList`` is `None`. 

196 """ 

197 if resultList: 

198 # ApPipeTaskRunner does not override default results handling, exitStatus always defined 

199 return min(127, sum(((res.exitStatus != 0) for res in resultList))) 

200 else: 

201 return 127 

202 

203 

204def runIngestion(cmdLine=None): 

205 """Ingest a dataset, but do not process it. 

206 

207 This is the main function for ``ingest_dataset``, and handles logging, 

208 command-line argument parsing, and ingestion. 

209 

210 Parameters 

211 ---------- 

212 cmdLine : `list` of `str` 

213 an optional command line used to execute `runIngestion` from other 

214 Python code. If `None`, `sys.argv` will be used. 

215 """ 

216 lsst.log.configure() 

217 log = lsst.log.Log.getLogger('ap.verify.ap_verify.ingest') 

218 # TODO: what is LSST's policy on exceptions escaping into main()? 

219 args = _IngestOnlyParser().parse_args(args=cmdLine) 

220 log.debug('Command-line arguments: %s', args) 

221 

222 if args.useGen3: 

223 workspace = WorkspaceGen3(args.output) 

224 ingestDatasetGen3(args.dataset, workspace) 

225 else: 

226 workspace = WorkspaceGen2(args.output) 

227 ingestDataset(args.dataset, workspace)