Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Interface between `ap_verify` and `ap_pipe`. 

25 

26This module handles calling `ap_pipe` and converting any information 

27as needed. 

28""" 

29 

30__all__ = ["ApPipeParser", "runApPipeGen2", "runApPipeGen3"] 

31 

32import argparse 

33import os 

34import re 

35 

36import click.testing 

37 

38import lsst.log 

39from lsst.utils import getPackageDir 

40import lsst.pipe.base as pipeBase 

41import lsst.obs.base as obsBase 

42import lsst.ctrl.mpexec.cli.pipetask 

43import lsst.ap.pipe as apPipe 

44from lsst.ap.pipe.make_apdb import makeApdb 

45 

46 

47class ApPipeParser(argparse.ArgumentParser): 

48 """An argument parser for data needed by ``ap_pipe`` activities. 

49 

50 This parser is not complete, and is designed to be passed to another parser 

51 using the `parent` parameter. 

52 """ 

53 

54 def __init__(self): 

55 defaultPipeline = os.path.join(getPackageDir("ap_verify"), "pipelines", "ApVerify.yaml") 

56 

57 # Help and documentation will be handled by main program's parser 

58 argparse.ArgumentParser.__init__(self, add_help=False) 

59 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs. 

60 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'. 

61 self.add_argument('--id', '-d', '--data-query', dest='dataIds', 

62 action=self.AppendOptional, nargs='?', default=[], 

63 help='An identifier for the data to process.') 

64 self.add_argument("-p", "--pipeline", default=defaultPipeline, 

65 help="A custom version of the ap_verify pipeline (e.g., with different metrics).") 

66 self.add_argument("--db", "--db_url", default=None, 

67 help="A location for the AP database, formatted as if for ApdbConfig.db_url. " 

68 "Defaults to an SQLite file in the --output directory.") 

69 self.add_argument("--skip-pipeline", action="store_true", 

70 help="Do not run the AP pipeline itself. This argument is useful " 

71 "for testing metrics on a fixed data set.") 

72 

73 class AppendOptional(argparse.Action): 

74 """A variant of the built-in "append" action that ignores None values 

75 instead of appending them. 

76 """ 

77 # This class can't safely inherit from the built-in "append" action 

78 # because there is no public class that implements it. 

79 def __call__(self, parser, namespace, values, option_string=None): 

80 if values is not None: 

81 try: 

82 allValues = getattr(namespace, self.dest) 

83 allValues.append(values) 

84 except AttributeError: 

85 setattr(namespace, self.dest, [values]) 

86 

87 

88def runApPipeGen2(workspace, parsedCmdLine, processes=1): 

89 """Run `ap_pipe` on this object's dataset. 

90 

91 Parameters 

92 ---------- 

93 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

94 The abstract location containing input and output repositories. 

95 parsedCmdLine : `argparse.Namespace` 

96 Command-line arguments, including all arguments supported by `ApPipeParser`. 

97 processes : `int` 

98 The number of processes with which to call the AP pipeline 

99 

100 Returns 

101 ------- 

102 apPipeReturn : `Struct` 

103 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with 

104 ``doReturnResults=False``. This object is valid even if 

105 `~lsst.ap.pipe.ApPipeTask` was never run. 

106 """ 

107 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen2') 

108 

109 makeApdb(_getApdbArguments(workspace, parsedCmdLine)) 

110 

111 pipelineArgs = [workspace.dataRepo, 

112 "--output", workspace.outputRepo, 

113 "--calib", workspace.calibRepo, 

114 "--template", workspace.templateRepo] 

115 pipelineArgs.extend(_getConfigArguments(workspace, parsedCmdLine)) 

116 if parsedCmdLine.dataIds: 

117 for singleId in parsedCmdLine.dataIds: 

118 pipelineArgs.extend(["--id", *singleId.split(" ")]) 

119 else: 

120 pipelineArgs.extend(["--id"]) 

121 pipelineArgs.extend(["--processes", str(processes)]) 

122 pipelineArgs.extend(["--noExit"]) 

123 

124 if not parsedCmdLine.skip_pipeline: 

125 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs) 

126 log.info('Pipeline complete') 

127 else: 

128 log.info('Skipping AP pipeline entirely.') 

129 apPipeParser = apPipe.ApPipeTask._makeArgumentParser() 

130 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs) 

131 results = pipeBase.Struct( 

132 argumentParser=apPipeParser, 

133 parsedCmd=apPipeParsed, 

134 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed), 

135 resultList=[], 

136 ) 

137 

138 return results 

139 

140 

141def runApPipeGen3(workspace, parsedCmdLine, processes=1): 

142 """Run `ap_pipe` on this object's dataset. 

143 

144 Parameters 

145 ---------- 

146 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

147 The abstract location containing input and output repositories. 

148 parsedCmdLine : `argparse.Namespace` 

149 Command-line arguments, including all arguments supported by `ApPipeParser`. 

150 processes : `int` 

151 The number of processes with which to call the AP pipeline 

152 """ 

153 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen3') 

154 

155 makeApdb(_getApdbArguments(workspace, parsedCmdLine)) 

156 

157 pipelineArgs = ["run", 

158 "--butler-config", workspace.repo, 

159 "--pipeline", parsedCmdLine.pipeline, 

160 ] 

161 # TODO: collections should be determined exclusively by Workspace.workButler, 

162 # but I can't find a way to hook that up to the graph builder. So use the CLI 

163 # for now and revisit once DM-26239 is done. 

164 pipelineArgs.extend(_getCollectionArguments(workspace)) 

165 pipelineArgs.extend(_getConfigArgumentsGen3(workspace, parsedCmdLine)) 

166 if parsedCmdLine.dataIds: 

167 for singleId in parsedCmdLine.dataIds: 

168 pipelineArgs.extend(["--data-query", singleId]) 

169 pipelineArgs.extend(["--processes", str(processes)]) 

170 pipelineArgs.extend(["--register-dataset-types"]) 

171 

172 if not parsedCmdLine.skip_pipeline: 

173 # CliRunner is an unsafe workaround for DM-26239 

174 runner = click.testing.CliRunner() 

175 # TODO: generalize this code in DM-26028 

176 # TODO: work off of workspace.workButler after DM-26239 

177 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs) 

178 if results.exception: 

179 raise RuntimeError("Pipeline failed.") from results.exception 

180 

181 log.info('Pipeline complete.') 

182 return results.exit_code 

183 else: 

184 log.info('Skipping AP pipeline entirely.') 

185 

186 

187def _getApdbArguments(workspace, parsed): 

188 """Return the config options for running make_apdb.py on this workspace, 

189 as command-line arguments. 

190 

191 Parameters 

192 ---------- 

193 workspace : `lsst.ap.verify.workspace.Workspace` 

194 A Workspace whose config directory may contain an 

195 `~lsst.ap.pipe.ApPipeTask` config. 

196 parsed : `argparse.Namespace` 

197 Command-line arguments, including all arguments supported by `ApPipeParser`. 

198 

199 Returns 

200 ------- 

201 args : `list` of `str` 

202 Command-line arguments calling ``--config`` or ``--config-file``, 

203 following the conventions of `sys.argv`. 

204 """ 

205 if not parsed.db: 

206 parsed.db = "sqlite:///" + workspace.dbLocation 

207 

208 args = ["--config", "db_url=" + parsed.db] 

209 # Same special-case check as ApdbConfig.validate() 

210 if parsed.db.startswith("sqlite"): 

211 args.extend(["--config", "isolation_level=READ_UNCOMMITTED"]) 

212 

213 return args 

214 

215 

216def _getConfigArguments(workspace, parsed): 

217 """Return the config options for running ApPipeTask on this workspace, as 

218 command-line arguments. 

219 

220 Parameters 

221 ---------- 

222 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

223 A Workspace whose config directory may contain an 

224 `~lsst.ap.pipe.ApPipeTask` config. 

225 parsed : `argparse.Namespace` 

226 Command-line arguments, including all arguments supported by `ApPipeParser`. 

227 

228 Returns 

229 ------- 

230 args : `list` of `str` 

231 Command-line arguments calling ``--config`` or ``--configfile``, 

232 following the conventions of `sys.argv`. 

233 """ 

234 overrideFile = apPipe.ApPipeTask._DefaultName + ".py" 

235 overridePath = os.path.join(workspace.configDir, overrideFile) 

236 

237 args = ["--configfile", overridePath] 

238 # Translate APDB-only arguments to work as a sub-config 

239 args.extend([("diaPipe.apdb." + arg if arg != "--config" else arg) 

240 for arg in _getApdbArguments(workspace, parsed)]) 

241 # Put output alerts into the workspace. 

242 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.alertLocation]) 

243 args.extend(["--config", "diaPipe.doPackageAlerts=True"]) 

244 

245 return args 

246 

247 

248def _getConfigArgumentsGen3(workspace, parsed): 

249 """Return the config options for running the Gen 3 AP Pipeline on this 

250 workspace, as command-line arguments. 

251 

252 Parameters 

253 ---------- 

254 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

255 A Workspace whose config directory may contain various configs. 

256 parsed : `argparse.Namespace` 

257 Command-line arguments, including all arguments supported by `ApPipeParser`. 

258 

259 Returns 

260 ------- 

261 args : `list` of `str` 

262 Command-line arguments calling ``--config`` or ``--config-file``, 

263 following the conventions of `sys.argv`. 

264 """ 

265 # Translate APDB-only arguments to work as a sub-config 

266 args = [("diaPipe:apdb." + arg if arg != "--config" else arg) 

267 for arg in _getApdbArguments(workspace, parsed)] 

268 args.extend([ 

269 # Put output alerts into the workspace. 

270 "--config", "diaPipe:alertPackager.alertWriteLocation=" + workspace.alertLocation, 

271 "--config", "diaPipe:doPackageAlerts=True", 

272 # TODO: the configs below should not be needed after DM-26140 

273 "--config-file", "calibrate:" + os.path.join(workspace.configDir, "calibrate.py"), 

274 "--config-file", "imageDifference:" + os.path.join(workspace.configDir, "imageDifference.py"), 

275 ]) 

276 # TODO: reverse-engineering the instrument should not be needed after DM-26140 

277 # pipetask will crash if there is more than one instrument 

278 for idRecord in workspace.workButler.registry.queryDataIds("instrument").expanded(): 

279 className = idRecord.records["instrument"].class_name 

280 args.extend(["--instrument", className]) 

281 

282 return args 

283 

284 

285def _getCollectionArguments(workspace): 

286 """Return the collections for running the Gen 3 AP Pipeline on this 

287 workspace, as command-line arguments. 

288 

289 Parameters 

290 ---------- 

291 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

292 A Workspace with a Gen 3 repository. 

293 

294 Returns 

295 ------- 

296 args : `list` of `str` 

297 Command-line arguments calling ``--input`` or ``--output``, 

298 following the conventions of `sys.argv`. 

299 """ 

300 butler = workspace.workButler 

301 # Hard-code the collection names because it's hard to infer the inputs from the Butler 

302 inputs = {"skymaps", "refcats"} 

303 for dimension in butler.registry.queryDataIds('instrument'): 

304 instrument = obsBase.Instrument.fromName(dimension["instrument"], butler.registry) 

305 inputs.add(instrument.makeDefaultRawIngestRunName()) 

306 inputs.add(instrument.makeCalibrationCollectionName()) 

307 inputs.update(butler.registry.queryCollections(re.compile(r"templates/\w+"))) 

308 

309 return ["--input", ",".join(inputs), 

310 "--output-run", workspace.runName, 

311 ]