Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Interface between `ap_verify` and `ap_pipe`. 

25 

26This module handles calling `ap_pipe` and converting any information 

27as needed. 

28""" 

29 

30__all__ = ["ApPipeParser", "runApPipeGen2", "runApPipeGen3"] 

31 

32import argparse 

33import os 

34import re 

35 

36import click.testing 

37 

38import lsst.log 

39from lsst.utils import getPackageDir 

40import lsst.pipe.base as pipeBase 

41import lsst.obs.base as obsBase 

42import lsst.ctrl.mpexec.cli.pipetask 

43import lsst.ap.pipe as apPipe 

44from lsst.ap.pipe.make_apdb import makeApdb 

45 

46 

47class ApPipeParser(argparse.ArgumentParser): 

48 """An argument parser for data needed by ``ap_pipe`` activities. 

49 

50 This parser is not complete, and is designed to be passed to another parser 

51 using the `parent` parameter. 

52 """ 

53 

54 def __init__(self): 

55 defaultPipeline = os.path.join(getPackageDir("ap_verify"), "pipelines", "ApVerify.yaml") 

56 

57 # Help and documentation will be handled by main program's parser 

58 argparse.ArgumentParser.__init__(self, add_help=False) 

59 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs. 

60 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'. 

61 self.add_argument('--id', '-d', '--data-query', dest='dataIds', 

62 action=self.AppendOptional, nargs='?', default=[], 

63 help='An identifier for the data to process.') 

64 self.add_argument("-p", "--pipeline", default=defaultPipeline, 

65 help="A custom version of the ap_verify pipeline (e.g., with different metrics).") 

66 self.add_argument("--skip-pipeline", action="store_true", 

67 help="Do not run the AP pipeline itself. This argument is useful " 

68 "for testing metrics on a fixed data set.") 

69 

70 class AppendOptional(argparse.Action): 

71 """A variant of the built-in "append" action that ignores None values 

72 instead of appending them. 

73 """ 

74 # This class can't safely inherit from the built-in "append" action 

75 # because there is no public class that implements it. 

76 def __call__(self, parser, namespace, values, option_string=None): 

77 if values is not None: 

78 try: 

79 allValues = getattr(namespace, self.dest) 

80 allValues.append(values) 

81 except AttributeError: 

82 setattr(namespace, self.dest, [values]) 

83 

84 

85def runApPipeGen2(workspace, parsedCmdLine, processes=1): 

86 """Run `ap_pipe` on this object's dataset. 

87 

88 Parameters 

89 ---------- 

90 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

91 The abstract location containing input and output repositories. 

92 parsedCmdLine : `argparse.Namespace` 

93 Command-line arguments, including all arguments supported by `ApPipeParser`. 

94 processes : `int` 

95 The number of processes with which to call the AP pipeline 

96 

97 Returns 

98 ------- 

99 apPipeReturn : `Struct` 

100 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with 

101 ``doReturnResults=False``. This object is valid even if 

102 `~lsst.ap.pipe.ApPipeTask` was never run. 

103 """ 

104 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen2') 

105 

106 configArgs = _getConfigArguments(workspace) 

107 makeApdb(configArgs) 

108 

109 pipelineArgs = [workspace.dataRepo, 

110 "--output", workspace.outputRepo, 

111 "--calib", workspace.calibRepo, 

112 "--template", workspace.templateRepo] 

113 pipelineArgs.extend(configArgs) 

114 if parsedCmdLine.dataIds: 

115 for singleId in parsedCmdLine.dataIds: 

116 pipelineArgs.extend(["--id", *singleId.split(" ")]) 

117 else: 

118 pipelineArgs.extend(["--id"]) 

119 pipelineArgs.extend(["--processes", str(processes)]) 

120 pipelineArgs.extend(["--noExit"]) 

121 

122 if not parsedCmdLine.skip_pipeline: 

123 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs) 

124 log.info('Pipeline complete') 

125 else: 

126 log.info('Skipping AP pipeline entirely.') 

127 apPipeParser = apPipe.ApPipeTask._makeArgumentParser() 

128 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs) 

129 results = pipeBase.Struct( 

130 argumentParser=apPipeParser, 

131 parsedCmd=apPipeParsed, 

132 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed), 

133 resultList=[], 

134 ) 

135 

136 return results 

137 

138 

139def runApPipeGen3(workspace, parsedCmdLine, processes=1): 

140 """Run `ap_pipe` on this object's dataset. 

141 

142 Parameters 

143 ---------- 

144 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

145 The abstract location containing input and output repositories. 

146 parsedCmdLine : `argparse.Namespace` 

147 Command-line arguments, including all arguments supported by `ApPipeParser`. 

148 processes : `int` 

149 The number of processes with which to call the AP pipeline 

150 """ 

151 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen3') 

152 

153 # Currently makeApdb has different argument conventions from Gen 3; see DM-22663 

154 makeApdb(_getConfigArguments(workspace)) 

155 

156 pipelineArgs = ["run", 

157 "--butler-config", workspace.repo, 

158 "--pipeline", parsedCmdLine.pipeline, 

159 ] 

160 # TODO: collections should be determined exclusively by Workspace.workButler, 

161 # but I can't find a way to hook that up to the graph builder. So use the CLI 

162 # for now and revisit once DM-26239 is done. 

163 pipelineArgs.extend(_getCollectionArguments(workspace)) 

164 pipelineArgs.extend(_getConfigArgumentsGen3(workspace)) 

165 if parsedCmdLine.dataIds: 

166 for singleId in parsedCmdLine.dataIds: 

167 pipelineArgs.extend(["--data-query", singleId]) 

168 pipelineArgs.extend(["--processes", str(processes)]) 

169 pipelineArgs.extend(["--register-dataset-types"]) 

170 

171 if not parsedCmdLine.skip_pipeline: 

172 # CliRunner is an unsafe workaround for DM-26239 

173 runner = click.testing.CliRunner() 

174 # TODO: generalize this code in DM-26028 

175 # TODO: work off of workspace.workButler after DM-26239 

176 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs) 

177 if results.exception: 

178 raise RuntimeError("Pipeline failed.") from results.exception 

179 

180 log.info('Pipeline complete.') 

181 return results.exit_code 

182 else: 

183 log.info('Skipping AP pipeline entirely.') 

184 

185 

186def _getConfigArguments(workspace): 

187 """Return the config options for running ApPipeTask on this workspace, as 

188 command-line arguments. 

189 

190 Parameters 

191 ---------- 

192 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

193 A Workspace whose config directory may contain an 

194 `~lsst.ap.pipe.ApPipeTask` config. 

195 

196 Returns 

197 ------- 

198 args : `list` of `str` 

199 Command-line arguments calling ``--config`` or ``--configFile``, 

200 following the conventions of `sys.argv`. 

201 """ 

202 overrideFile = apPipe.ApPipeTask._DefaultName + ".py" 

203 overridePath = os.path.join(workspace.configDir, overrideFile) 

204 

205 args = ["--configfile", overridePath] 

206 # ApVerify will use the sqlite hooks for the Apdb. 

207 args.extend(["--config", "diaPipe.apdb.db_url=sqlite:///" + workspace.dbLocation]) 

208 args.extend(["--config", "diaPipe.apdb.isolation_level=READ_UNCOMMITTED"]) 

209 # Put output alerts into the workspace. 

210 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.alertLocation]) 

211 args.extend(["--config", "diaPipe.doPackageAlerts=True"]) 

212 

213 return args 

214 

215 

216def _getConfigArgumentsGen3(workspace): 

217 """Return the config options for running the Gen 3 AP Pipeline on this 

218 workspace, as command-line arguments. 

219 

220 Parameters 

221 ---------- 

222 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

223 A Workspace whose config directory may contain various configs. 

224 

225 Returns 

226 ------- 

227 args : `list` of `str` 

228 Command-line arguments calling ``--config`` or ``--config-file``, 

229 following the conventions of `sys.argv`. 

230 """ 

231 args = [ 

232 # ApVerify will use the sqlite hooks for the Apdb. 

233 "--config", "diaPipe:apdb.db_url=sqlite:///" + workspace.dbLocation, 

234 "--config", "diaPipe:apdb.isolation_level=READ_UNCOMMITTED", 

235 # Put output alerts into the workspace. 

236 "--config", "diaPipe:alertPackager.alertWriteLocation=" + workspace.alertLocation, 

237 "--config", "diaPipe:doPackageAlerts=True", 

238 # TODO: the configs below should not be needed after DM-26140 

239 "--config-file", "calibrate:" + os.path.join(workspace.configDir, "calibrate.py"), 

240 "--config-file", "imageDifference:" + os.path.join(workspace.configDir, "imageDifference.py"), 

241 ] 

242 # TODO: reverse-engineering the instrument should not be needed after DM-26140 

243 # pipetask will crash if there is more than one instrument 

244 for idRecord in workspace.workButler.registry.queryDataIds("instrument").expanded(): 

245 className = idRecord.records["instrument"].class_name 

246 args.extend(["--instrument", className]) 

247 

248 return args 

249 

250 

251def _getCollectionArguments(workspace): 

252 """Return the collections for running the Gen 3 AP Pipeline on this 

253 workspace, as command-line arguments. 

254 

255 Parameters 

256 ---------- 

257 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

258 A Workspace with a Gen 3 repository. 

259 

260 Returns 

261 ------- 

262 args : `list` of `str` 

263 Command-line arguments calling ``--input`` or ``--output``, 

264 following the conventions of `sys.argv`. 

265 """ 

266 butler = workspace.workButler 

267 # Hard-code the collection names because it's hard to infer the inputs from the Butler 

268 inputs = {"skymaps", "refcats"} 

269 for dimension in butler.registry.queryDataIds('instrument'): 

270 instrument = obsBase.Instrument.fromName(dimension["instrument"], butler.registry) 

271 inputs.add(instrument.makeDefaultRawIngestRunName()) 

272 inputs.add(instrument.makeCalibrationCollectionName()) 

273 inputs.update(butler.registry.queryCollections(re.compile(r"templates/\w+"))) 

274 

275 return ["--input", ",".join(inputs), 

276 "--output-run", workspace.runName, 

277 ]