Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Interface between `ap_verify` and `ap_pipe`. 

25 

26This module handles calling `ap_pipe` and converting any information 

27as needed. 

28""" 

29 

30__all__ = ["ApPipeParser", "runApPipeGen2", "runApPipeGen3"] 

31 

32import argparse 

33import os 

34import re 

35 

36import click.testing 

37 

38import lsst.log 

39from lsst.utils import getPackageDir 

40import lsst.pipe.base as pipeBase 

41import lsst.ctrl.mpexec.execFixupDataId # not part of lsst.ctrl.mpexec 

42import lsst.ctrl.mpexec.cli.pipetask 

43import lsst.ap.pipe as apPipe 

44from lsst.ap.pipe.make_apdb import makeApdb 

45 

46 

47class ApPipeParser(argparse.ArgumentParser): 

48 """An argument parser for data needed by ``ap_pipe`` activities. 

49 

50 This parser is not complete, and is designed to be passed to another parser 

51 using the `parent` parameter. 

52 """ 

53 

54 def __init__(self): 

55 defaultPipeline = os.path.join(getPackageDir("ap_verify"), "pipelines", "ApVerify.yaml") 

56 

57 # Help and documentation will be handled by main program's parser 

58 argparse.ArgumentParser.__init__(self, add_help=False) 

59 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs. 

60 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'. 

61 self.add_argument('--id', '-d', '--data-query', dest='dataIds', 

62 action=self.AppendOptional, nargs='?', default=[], 

63 help='An identifier for the data to process.') 

64 self.add_argument("-p", "--pipeline", default=defaultPipeline, 

65 help="A custom version of the ap_verify pipeline (e.g., with different metrics).") 

66 self.add_argument("--db", "--db_url", default=None, 

67 help="A location for the AP database, formatted as if for ApdbConfig.db_url. " 

68 "Defaults to an SQLite file in the --output directory.") 

69 self.add_argument("--skip-pipeline", action="store_true", 

70 help="Do not run the AP pipeline itself. This argument is useful " 

71 "for testing metrics on a fixed data set.") 

72 self.add_argument("--clean-run", action="store_true", 

73 help="Run the pipeline with a new run collection, " 

74 "even if one already exists.") 

75 

76 class AppendOptional(argparse.Action): 

77 """A variant of the built-in "append" action that ignores None values 

78 instead of appending them. 

79 """ 

80 # This class can't safely inherit from the built-in "append" action 

81 # because there is no public class that implements it. 

82 def __call__(self, parser, namespace, values, option_string=None): 

83 if values is not None: 

84 try: 

85 allValues = getattr(namespace, self.dest) 

86 allValues.append(values) 

87 except AttributeError: 

88 setattr(namespace, self.dest, [values]) 

89 

90 

91def runApPipeGen2(workspace, parsedCmdLine, processes=1): 

92 """Run `ap_pipe` on this object's dataset. 

93 

94 Parameters 

95 ---------- 

96 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

97 The abstract location containing input and output repositories. 

98 parsedCmdLine : `argparse.Namespace` 

99 Command-line arguments, including all arguments supported by `ApPipeParser`. 

100 processes : `int` 

101 The number of processes with which to call the AP pipeline 

102 

103 Returns 

104 ------- 

105 apPipeReturn : `Struct` 

106 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with 

107 ``doReturnResults=False``. This object is valid even if 

108 `~lsst.ap.pipe.ApPipeTask` was never run. 

109 """ 

110 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen2') 

111 

112 makeApdb(_getApdbArguments(workspace, parsedCmdLine)) 

113 

114 pipelineArgs = [workspace.dataRepo, 

115 "--output", workspace.outputRepo, 

116 "--calib", workspace.calibRepo, 

117 "--template", workspace.templateRepo] 

118 pipelineArgs.extend(_getConfigArguments(workspace, parsedCmdLine)) 

119 if parsedCmdLine.dataIds: 

120 for singleId in parsedCmdLine.dataIds: 

121 pipelineArgs.extend(["--id", *singleId.split(" ")]) 

122 else: 

123 pipelineArgs.extend(["--id"]) 

124 pipelineArgs.extend(["--processes", str(processes)]) 

125 pipelineArgs.extend(["--noExit"]) 

126 

127 if not parsedCmdLine.skip_pipeline: 

128 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs) 

129 log.info('Pipeline complete') 

130 else: 

131 log.info('Skipping AP pipeline entirely.') 

132 apPipeParser = apPipe.ApPipeTask._makeArgumentParser() 

133 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs) 

134 results = pipeBase.Struct( 

135 argumentParser=apPipeParser, 

136 parsedCmd=apPipeParsed, 

137 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed), 

138 resultList=[], 

139 ) 

140 

141 return results 

142 

143 

144def runApPipeGen3(workspace, parsedCmdLine, processes=1): 

145 """Run `ap_pipe` on this object's dataset. 

146 

147 Parameters 

148 ---------- 

149 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

150 The abstract location containing input and output repositories. 

151 parsedCmdLine : `argparse.Namespace` 

152 Command-line arguments, including all arguments supported by `ApPipeParser`. 

153 processes : `int` 

154 The number of processes with which to call the AP pipeline 

155 """ 

156 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen3') 

157 

158 makeApdb(_getApdbArguments(workspace, parsedCmdLine)) 

159 

160 pipelineArgs = ["run", 

161 "--butler-config", workspace.repo, 

162 "--pipeline", parsedCmdLine.pipeline, 

163 ] 

164 # TODO: collections should be determined exclusively by Workspace.workButler, 

165 # but I can't find a way to hook that up to the graph builder. So use the CLI 

166 # for now and revisit once DM-26239 is done. 

167 pipelineArgs.extend(_getCollectionArguments(workspace, reuse=(not parsedCmdLine.clean_run))) 

168 pipelineArgs.extend(_getConfigArgumentsGen3(workspace, parsedCmdLine)) 

169 if parsedCmdLine.dataIds: 

170 for singleId in parsedCmdLine.dataIds: 

171 pipelineArgs.extend(["--data-query", singleId]) 

172 pipelineArgs.extend(["--processes", str(processes)]) 

173 pipelineArgs.extend(["--register-dataset-types"]) 

174 pipelineArgs.extend(["--graph-fixup", "lsst.ap.verify.pipeline_driver._getExecOrder"]) 

175 

176 if not parsedCmdLine.skip_pipeline: 

177 # CliRunner is an unsafe workaround for DM-26239 

178 runner = click.testing.CliRunner() 

179 # TODO: generalize this code in DM-26028 

180 # TODO: work off of workspace.workButler after DM-26239 

181 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs) 

182 if results.exception: 

183 raise RuntimeError("Pipeline failed.") from results.exception 

184 

185 log.info('Pipeline complete.') 

186 return results.exit_code 

187 else: 

188 log.info('Skipping AP pipeline entirely.') 

189 

190 

191def _getExecOrder(): 

192 """Return any constraints on the Gen 3 execution order. 

193 

194 The current constraints are that executions of DiaPipelineTask must be 

195 ordered by visit ID, but this is subject to change. 

196 

197 Returns 

198 ------- 

199 order : `lsst.ctrl.mpexec.ExecutionGraphFixup` 

200 An object encoding the desired execution order as an algorithm for 

201 modifying inter-quantum dependencies. 

202 

203 Notes 

204 ----- 

205 This function must be importable, but need not be public. 

206 """ 

207 # Source association algorithm is not time-symmetric. Force execution of 

208 # association (through DiaPipelineTask) in order of ascending visit number. 

209 return lsst.ctrl.mpexec.execFixupDataId.ExecFixupDataId( 

210 taskLabel="diaPipe", dimensions=["visit", ], reverse=False) 

211 

212 

213def _getApdbArguments(workspace, parsed): 

214 """Return the config options for running make_apdb.py on this workspace, 

215 as command-line arguments. 

216 

217 Parameters 

218 ---------- 

219 workspace : `lsst.ap.verify.workspace.Workspace` 

220 A Workspace whose config directory may contain an 

221 `~lsst.ap.pipe.ApPipeTask` config. 

222 parsed : `argparse.Namespace` 

223 Command-line arguments, including all arguments supported by `ApPipeParser`. 

224 

225 Returns 

226 ------- 

227 args : `list` of `str` 

228 Command-line arguments calling ``--config`` or ``--config-file``, 

229 following the conventions of `sys.argv`. 

230 """ 

231 if not parsed.db: 

232 parsed.db = "sqlite:///" + workspace.dbLocation 

233 

234 args = ["--config", "db_url=" + parsed.db] 

235 # Same special-case check as ApdbConfig.validate() 

236 if parsed.db.startswith("sqlite"): 

237 args.extend(["--config", "isolation_level=READ_UNCOMMITTED"]) 

238 

239 return args 

240 

241 

242def _getConfigArguments(workspace, parsed): 

243 """Return the config options for running ApPipeTask on this workspace, as 

244 command-line arguments. 

245 

246 Parameters 

247 ---------- 

248 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

249 A Workspace whose config directory may contain an 

250 `~lsst.ap.pipe.ApPipeTask` config. 

251 parsed : `argparse.Namespace` 

252 Command-line arguments, including all arguments supported by `ApPipeParser`. 

253 

254 Returns 

255 ------- 

256 args : `list` of `str` 

257 Command-line arguments calling ``--config`` or ``--configfile``, 

258 following the conventions of `sys.argv`. 

259 """ 

260 overrideFile = apPipe.ApPipeTask._DefaultName + ".py" 

261 overridePath = os.path.join(workspace.configDir, overrideFile) 

262 

263 args = ["--configfile", overridePath] 

264 # Translate APDB-only arguments to work as a sub-config 

265 args.extend([("diaPipe.apdb." + arg if arg != "--config" else arg) 

266 for arg in _getApdbArguments(workspace, parsed)]) 

267 # Put output alerts into the workspace. 

268 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.alertLocation]) 

269 args.extend(["--config", "diaPipe.doPackageAlerts=True"]) 

270 

271 return args 

272 

273 

274def _getConfigArgumentsGen3(workspace, parsed): 

275 """Return the config options for running the Gen 3 AP Pipeline on this 

276 workspace, as command-line arguments. 

277 

278 Parameters 

279 ---------- 

280 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

281 A Workspace whose config directory may contain various configs. 

282 parsed : `argparse.Namespace` 

283 Command-line arguments, including all arguments supported by `ApPipeParser`. 

284 

285 Returns 

286 ------- 

287 args : `list` of `str` 

288 Command-line arguments calling ``--config`` or ``--config-file``, 

289 following the conventions of `sys.argv`. 

290 """ 

291 # Translate APDB-only arguments to work as a sub-config 

292 args = [("diaPipe:apdb." + arg if arg != "--config" else arg) 

293 for arg in _getApdbArguments(workspace, parsed)] 

294 args.extend([ 

295 # Put output alerts into the workspace. 

296 "--config", "diaPipe:alertPackager.alertWriteLocation=" + workspace.alertLocation, 

297 "--config", "diaPipe:doPackageAlerts=True", 

298 # TODO: the configs below should not be needed after DM-26140 

299 "--config-file", "calibrate:" + os.path.join(workspace.configDir, "calibrate.py"), 

300 "--config-file", "imageDifference:" + os.path.join(workspace.configDir, "imageDifference.py"), 

301 ]) 

302 # TODO: reverse-engineering the instrument should not be needed after DM-26140 

303 # pipetask will crash if there is more than one instrument 

304 for idRecord in workspace.workButler.registry.queryDataIds("instrument").expanded(): 

305 className = idRecord.records["instrument"].class_name 

306 args.extend(["--instrument", className]) 

307 

308 return args 

309 

310 

311def _getCollectionArguments(workspace, reuse): 

312 """Return the collections for running the Gen 3 AP Pipeline on this 

313 workspace, as command-line arguments. 

314 

315 Parameters 

316 ---------- 

317 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

318 A Workspace with a Gen 3 repository. 

319 reuse : `bool` 

320 If true, use the previous run collection if one exists. Otherwise, 

321 create a new run. 

322 

323 Returns 

324 ------- 

325 args : `list` of `str` 

326 Command-line arguments calling ``--input`` or ``--output``, 

327 following the conventions of `sys.argv`. 

328 """ 

329 # workspace.outputName is a chained collection containing all inputs 

330 args = ["--output", workspace.outputName, 

331 "--clobber-partial-outputs", 

332 ] 

333 

334 registry = workspace.workButler.registry 

335 oldRuns = list(registry.queryCollections(re.compile(workspace.outputName + r"/\d+T\d+Z"))) 

336 if reuse and oldRuns: 

337 args.extend(["--extend-run", "--skip-existing"]) 

338 return args