Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Interface between `ap_verify` and `ap_pipe`. 

25 

26This module handles calling `ap_pipe` and converting any information 

27as needed. 

28""" 

29 

30__all__ = ["ApPipeParser", "runApPipeGen2", "runApPipeGen3"] 

31 

32import argparse 

33import os 

34import re 

35 

36import click.testing 

37 

38import lsst.log 

39from lsst.utils import getPackageDir 

40import lsst.pipe.base as pipeBase 

41import lsst.ctrl.mpexec.cli.pipetask 

42import lsst.ap.pipe as apPipe 

43from lsst.ap.pipe.make_apdb import makeApdb 

44 

45 

46class ApPipeParser(argparse.ArgumentParser): 

47 """An argument parser for data needed by ``ap_pipe`` activities. 

48 

49 This parser is not complete, and is designed to be passed to another parser 

50 using the `parent` parameter. 

51 """ 

52 

53 def __init__(self): 

54 defaultPipeline = os.path.join(getPackageDir("ap_verify"), "pipelines", "ApVerify.yaml") 

55 

56 # Help and documentation will be handled by main program's parser 

57 argparse.ArgumentParser.__init__(self, add_help=False) 

58 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs. 

59 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'. 

60 self.add_argument('--id', '-d', '--data-query', dest='dataIds', 

61 action=self.AppendOptional, nargs='?', default=[], 

62 help='An identifier for the data to process.') 

63 self.add_argument("-p", "--pipeline", default=defaultPipeline, 

64 help="A custom version of the ap_verify pipeline (e.g., with different metrics).") 

65 self.add_argument("--db", "--db_url", default=None, 

66 help="A location for the AP database, formatted as if for ApdbConfig.db_url. " 

67 "Defaults to an SQLite file in the --output directory.") 

68 self.add_argument("--skip-pipeline", action="store_true", 

69 help="Do not run the AP pipeline itself. This argument is useful " 

70 "for testing metrics on a fixed data set.") 

71 self.add_argument("--clean-run", action="store_true", 

72 help="Run the pipeline with a new run collection, " 

73 "even if one already exists.") 

74 

75 class AppendOptional(argparse.Action): 

76 """A variant of the built-in "append" action that ignores None values 

77 instead of appending them. 

78 """ 

79 # This class can't safely inherit from the built-in "append" action 

80 # because there is no public class that implements it. 

81 def __call__(self, parser, namespace, values, option_string=None): 

82 if values is not None: 

83 try: 

84 allValues = getattr(namespace, self.dest) 

85 allValues.append(values) 

86 except AttributeError: 

87 setattr(namespace, self.dest, [values]) 

88 

89 

90def runApPipeGen2(workspace, parsedCmdLine, processes=1): 

91 """Run `ap_pipe` on this object's dataset. 

92 

93 Parameters 

94 ---------- 

95 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

96 The abstract location containing input and output repositories. 

97 parsedCmdLine : `argparse.Namespace` 

98 Command-line arguments, including all arguments supported by `ApPipeParser`. 

99 processes : `int` 

100 The number of processes with which to call the AP pipeline 

101 

102 Returns 

103 ------- 

104 apPipeReturn : `Struct` 

105 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with 

106 ``doReturnResults=False``. This object is valid even if 

107 `~lsst.ap.pipe.ApPipeTask` was never run. 

108 """ 

109 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen2') 

110 

111 makeApdb(_getApdbArguments(workspace, parsedCmdLine)) 

112 

113 pipelineArgs = [workspace.dataRepo, 

114 "--output", workspace.outputRepo, 

115 "--calib", workspace.calibRepo, 

116 "--template", workspace.templateRepo] 

117 pipelineArgs.extend(_getConfigArguments(workspace, parsedCmdLine)) 

118 if parsedCmdLine.dataIds: 

119 for singleId in parsedCmdLine.dataIds: 

120 pipelineArgs.extend(["--id", *singleId.split(" ")]) 

121 else: 

122 pipelineArgs.extend(["--id"]) 

123 pipelineArgs.extend(["--processes", str(processes)]) 

124 pipelineArgs.extend(["--noExit"]) 

125 

126 if not parsedCmdLine.skip_pipeline: 

127 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs) 

128 log.info('Pipeline complete') 

129 else: 

130 log.info('Skipping AP pipeline entirely.') 

131 apPipeParser = apPipe.ApPipeTask._makeArgumentParser() 

132 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs) 

133 results = pipeBase.Struct( 

134 argumentParser=apPipeParser, 

135 parsedCmd=apPipeParsed, 

136 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed), 

137 resultList=[], 

138 ) 

139 

140 return results 

141 

142 

143def runApPipeGen3(workspace, parsedCmdLine, processes=1): 

144 """Run `ap_pipe` on this object's dataset. 

145 

146 Parameters 

147 ---------- 

148 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

149 The abstract location containing input and output repositories. 

150 parsedCmdLine : `argparse.Namespace` 

151 Command-line arguments, including all arguments supported by `ApPipeParser`. 

152 processes : `int` 

153 The number of processes with which to call the AP pipeline 

154 """ 

155 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen3') 

156 

157 makeApdb(_getApdbArguments(workspace, parsedCmdLine)) 

158 

159 pipelineArgs = ["run", 

160 "--butler-config", workspace.repo, 

161 "--pipeline", parsedCmdLine.pipeline, 

162 ] 

163 # TODO: collections should be determined exclusively by Workspace.workButler, 

164 # but I can't find a way to hook that up to the graph builder. So use the CLI 

165 # for now and revisit once DM-26239 is done. 

166 pipelineArgs.extend(_getCollectionArguments(workspace, reuse=(not parsedCmdLine.clean_run))) 

167 pipelineArgs.extend(_getConfigArgumentsGen3(workspace, parsedCmdLine)) 

168 if parsedCmdLine.dataIds: 

169 for singleId in parsedCmdLine.dataIds: 

170 pipelineArgs.extend(["--data-query", singleId]) 

171 pipelineArgs.extend(["--processes", str(processes)]) 

172 pipelineArgs.extend(["--register-dataset-types"]) 

173 

174 if not parsedCmdLine.skip_pipeline: 

175 # CliRunner is an unsafe workaround for DM-26239 

176 runner = click.testing.CliRunner() 

177 # TODO: generalize this code in DM-26028 

178 # TODO: work off of workspace.workButler after DM-26239 

179 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs) 

180 if results.exception: 

181 raise RuntimeError("Pipeline failed.") from results.exception 

182 

183 log.info('Pipeline complete.') 

184 return results.exit_code 

185 else: 

186 log.info('Skipping AP pipeline entirely.') 

187 

188 

189def _getApdbArguments(workspace, parsed): 

190 """Return the config options for running make_apdb.py on this workspace, 

191 as command-line arguments. 

192 

193 Parameters 

194 ---------- 

195 workspace : `lsst.ap.verify.workspace.Workspace` 

196 A Workspace whose config directory may contain an 

197 `~lsst.ap.pipe.ApPipeTask` config. 

198 parsed : `argparse.Namespace` 

199 Command-line arguments, including all arguments supported by `ApPipeParser`. 

200 

201 Returns 

202 ------- 

203 args : `list` of `str` 

204 Command-line arguments calling ``--config`` or ``--config-file``, 

205 following the conventions of `sys.argv`. 

206 """ 

207 if not parsed.db: 

208 parsed.db = "sqlite:///" + workspace.dbLocation 

209 

210 args = ["--config", "db_url=" + parsed.db] 

211 # Same special-case check as ApdbConfig.validate() 

212 if parsed.db.startswith("sqlite"): 

213 args.extend(["--config", "isolation_level=READ_UNCOMMITTED"]) 

214 

215 return args 

216 

217 

218def _getConfigArguments(workspace, parsed): 

219 """Return the config options for running ApPipeTask on this workspace, as 

220 command-line arguments. 

221 

222 Parameters 

223 ---------- 

224 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

225 A Workspace whose config directory may contain an 

226 `~lsst.ap.pipe.ApPipeTask` config. 

227 parsed : `argparse.Namespace` 

228 Command-line arguments, including all arguments supported by `ApPipeParser`. 

229 

230 Returns 

231 ------- 

232 args : `list` of `str` 

233 Command-line arguments calling ``--config`` or ``--configfile``, 

234 following the conventions of `sys.argv`. 

235 """ 

236 overrideFile = apPipe.ApPipeTask._DefaultName + ".py" 

237 overridePath = os.path.join(workspace.configDir, overrideFile) 

238 

239 args = ["--configfile", overridePath] 

240 # Translate APDB-only arguments to work as a sub-config 

241 args.extend([("diaPipe.apdb." + arg if arg != "--config" else arg) 

242 for arg in _getApdbArguments(workspace, parsed)]) 

243 # Put output alerts into the workspace. 

244 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.alertLocation]) 

245 args.extend(["--config", "diaPipe.doPackageAlerts=True"]) 

246 

247 return args 

248 

249 

250def _getConfigArgumentsGen3(workspace, parsed): 

251 """Return the config options for running the Gen 3 AP Pipeline on this 

252 workspace, as command-line arguments. 

253 

254 Parameters 

255 ---------- 

256 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

257 A Workspace whose config directory may contain various configs. 

258 parsed : `argparse.Namespace` 

259 Command-line arguments, including all arguments supported by `ApPipeParser`. 

260 

261 Returns 

262 ------- 

263 args : `list` of `str` 

264 Command-line arguments calling ``--config`` or ``--config-file``, 

265 following the conventions of `sys.argv`. 

266 """ 

267 # Translate APDB-only arguments to work as a sub-config 

268 args = [("diaPipe:apdb." + arg if arg != "--config" else arg) 

269 for arg in _getApdbArguments(workspace, parsed)] 

270 args.extend([ 

271 # Put output alerts into the workspace. 

272 "--config", "diaPipe:alertPackager.alertWriteLocation=" + workspace.alertLocation, 

273 "--config", "diaPipe:doPackageAlerts=True", 

274 # TODO: the configs below should not be needed after DM-26140 

275 "--config-file", "calibrate:" + os.path.join(workspace.configDir, "calibrate.py"), 

276 "--config-file", "imageDifference:" + os.path.join(workspace.configDir, "imageDifference.py"), 

277 ]) 

278 # TODO: reverse-engineering the instrument should not be needed after DM-26140 

279 # pipetask will crash if there is more than one instrument 

280 for idRecord in workspace.workButler.registry.queryDataIds("instrument").expanded(): 

281 className = idRecord.records["instrument"].class_name 

282 args.extend(["--instrument", className]) 

283 

284 return args 

285 

286 

287def _getCollectionArguments(workspace, reuse): 

288 """Return the collections for running the Gen 3 AP Pipeline on this 

289 workspace, as command-line arguments. 

290 

291 Parameters 

292 ---------- 

293 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

294 A Workspace with a Gen 3 repository. 

295 reuse : `bool` 

296 If true, use the previous run collection if one exists. Otherwise, 

297 create a new run. 

298 

299 Returns 

300 ------- 

301 args : `list` of `str` 

302 Command-line arguments calling ``--input`` or ``--output``, 

303 following the conventions of `sys.argv`. 

304 """ 

305 # workspace.outputName is a chained collection containing all inputs 

306 args = ["--output", workspace.outputName, 

307 "--clobber-partial-outputs", 

308 ] 

309 

310 registry = workspace.workButler.registry 

311 oldRuns = list(registry.queryCollections(re.compile(workspace.outputName + r"/\d+T\d+Z"))) 

312 if reuse and oldRuns: 

313 args.extend(["--extend-run", "--skip-existing"]) 

314 return args