Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Interface between `ap_verify` and `ap_pipe`. 

25 

26This module handles calling `ap_pipe` and converting any information 

27as needed. 

28""" 

29 

30__all__ = ["ApPipeParser", "runApPipeGen2", "runApPipeGen3"] 

31 

32import argparse 

33import os 

34 

35import lsst.log 

36from lsst.utils import getPackageDir 

37import lsst.daf.butler as dafButler 

38import lsst.pipe.base as pipeBase 

39import lsst.ctrl.mpexec as ctrlMpexec 

40import lsst.ap.pipe as apPipe 

41from lsst.ap.pipe.make_apdb import makeApdb 

42 

43 

44class ApPipeParser(argparse.ArgumentParser): 

45 """An argument parser for data needed by ``ap_pipe`` activities. 

46 

47 This parser is not complete, and is designed to be passed to another parser 

48 using the `parent` parameter. 

49 """ 

50 

51 def __init__(self): 

52 defaultPipeline = os.path.join(getPackageDir("ap_verify"), "pipelines", "ApVerify.yaml") 

53 

54 # Help and documentation will be handled by main program's parser 

55 argparse.ArgumentParser.__init__(self, add_help=False) 

56 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs. 

57 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'. 

58 self.add_argument('--id', dest='dataIds', action=self.AppendOptional, nargs='?', default=[], 

59 help='An identifier for the data to process.') 

60 self.add_argument("-p", "--pipeline", default=defaultPipeline, 

61 help="A custom version of the ap_verify pipeline (e.g., with different metrics).") 

62 self.add_argument("--skip-pipeline", action="store_true", 

63 help="Do not run the AP pipeline itself. This argument is useful " 

64 "for testing metrics on a fixed data set.") 

65 

66 class AppendOptional(argparse.Action): 

67 """A variant of the built-in "append" action that ignores None values 

68 instead of appending them. 

69 """ 

70 # This class can't safely inherit from the built-in "append" action 

71 # because there is no public class that implements it. 

72 def __call__(self, parser, namespace, values, option_string=None): 

73 if values is not None: 

74 try: 

75 allValues = getattr(namespace, self.dest) 

76 allValues.append(values) 

77 except AttributeError: 

78 setattr(namespace, self.dest, [values]) 

79 

80 

81def runApPipeGen2(workspace, parsedCmdLine, processes=1): 

82 """Run `ap_pipe` on this object's dataset. 

83 

84 Parameters 

85 ---------- 

86 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

87 The abstract location containing input and output repositories. 

88 parsedCmdLine : `argparse.Namespace` 

89 Command-line arguments, including all arguments supported by `ApPipeParser`. 

90 processes : `int` 

91 The number of processes with which to call the AP pipeline 

92 

93 Returns 

94 ------- 

95 apPipeReturn : `Struct` 

96 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with 

97 ``doReturnResults=False``. This object is valid even if 

98 `~lsst.ap.pipe.ApPipeTask` was never run. 

99 """ 

100 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen2') 

101 

102 configArgs = _getConfigArguments(workspace) 

103 makeApdb(configArgs) 

104 

105 pipelineArgs = [workspace.dataRepo, 

106 "--output", workspace.outputRepo, 

107 "--calib", workspace.calibRepo, 

108 "--template", workspace.templateRepo] 

109 pipelineArgs.extend(configArgs) 

110 if parsedCmdLine.dataIds: 

111 for singleId in parsedCmdLine.dataIds: 

112 pipelineArgs.extend(["--id", *singleId.split(" ")]) 

113 else: 

114 pipelineArgs.extend(["--id"]) 

115 pipelineArgs.extend(["--processes", str(processes)]) 

116 pipelineArgs.extend(["--noExit"]) 

117 

118 if not parsedCmdLine.skip_pipeline: 

119 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs) 

120 log.info('Pipeline complete') 

121 else: 

122 log.info('Skipping AP pipeline entirely.') 

123 apPipeParser = apPipe.ApPipeTask._makeArgumentParser() 

124 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs) 

125 results = pipeBase.Struct( 

126 argumentParser=apPipeParser, 

127 parsedCmd=apPipeParsed, 

128 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed), 

129 resultList=[], 

130 ) 

131 

132 return results 

133 

134 

135def runApPipeGen3(workspace, parsedCmdLine, processes=1): 

136 """Run `ap_pipe` on this object's dataset. 

137 

138 Parameters 

139 ---------- 

140 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

141 The abstract location containing input and output repositories. 

142 parsedCmdLine : `argparse.Namespace` 

143 Command-line arguments, including all arguments supported by `ApPipeParser`. 

144 processes : `int` 

145 The number of processes with which to call the AP pipeline 

146 """ 

147 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen3') 

148 

149 # Currently makeApdb has different argument conventions from Gen 3; see DM-22663 

150 makeApdb(_getConfigArguments(workspace)) 

151 

152 pipelineArgs = ["run", 

153 "--butler-config", workspace.repo, 

154 "--pipeline", parsedCmdLine.pipeline, 

155 ] 

156 # TODO: collections should be determined exclusively by Workspace.workButler, 

157 # but I can't find a way to hook that up to the graph builder. So use the CLI 

158 # for now and revisit once DM-26239 is done. 

159 pipelineArgs.extend(_getCollectionArguments(workspace)) 

160 pipelineArgs.extend(_getConfigArgumentsGen3(workspace)) 

161 if parsedCmdLine.dataIds: 

162 for singleId in parsedCmdLine.dataIds: 

163 pipelineArgs.extend(["--data-query", singleId]) 

164 pipelineArgs.extend(["--processes", str(processes)]) 

165 pipelineArgs.extend(["--register-dataset-types"]) 

166 

167 if not parsedCmdLine.skip_pipeline: 

168 # TODO: generalize this code in DM-26028 

169 activator = ctrlMpexec.CmdLineFwk() 

170 # TODO: work off of workspace.workButler after DM-26239 

171 results = activator.parseAndRun(pipelineArgs) 

172 

173 log.info('Pipeline complete.') 

174 return results 

175 else: 

176 log.info('Skipping AP pipeline entirely.') 

177 

178 

179def _getConfigArguments(workspace): 

180 """Return the config options for running ApPipeTask on this workspace, as 

181 command-line arguments. 

182 

183 Parameters 

184 ---------- 

185 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

186 A Workspace whose config directory may contain an 

187 `~lsst.ap.pipe.ApPipeTask` config. 

188 

189 Returns 

190 ------- 

191 args : `list` of `str` 

192 Command-line arguments calling ``--config`` or ``--configFile``, 

193 following the conventions of `sys.argv`. 

194 """ 

195 overrideFile = apPipe.ApPipeTask._DefaultName + ".py" 

196 overridePath = os.path.join(workspace.configDir, overrideFile) 

197 

198 args = ["--configfile", overridePath] 

199 # ApVerify will use the sqlite hooks for the Apdb. 

200 args.extend(["--config", "diaPipe.apdb.db_url=sqlite:///" + workspace.dbLocation]) 

201 args.extend(["--config", "diaPipe.apdb.isolation_level=READ_UNCOMMITTED"]) 

202 # Put output alerts into the workspace. 

203 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.alertLocation]) 

204 args.extend(["--config", "diaPipe.doPackageAlerts=True"]) 

205 

206 return args 

207 

208 

209def _getConfigArgumentsGen3(workspace): 

210 """Return the config options for running the Gen 3 AP Pipeline on this 

211 workspace, as command-line arguments. 

212 

213 Parameters 

214 ---------- 

215 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

216 A Workspace whose config directory may contain various configs. 

217 

218 Returns 

219 ------- 

220 args : `list` of `str` 

221 Command-line arguments calling ``--config`` or ``--configFile``, 

222 following the conventions of `sys.argv`. 

223 """ 

224 args = [ 

225 # ApVerify will use the sqlite hooks for the Apdb. 

226 "--config", "diaPipe:apdb.db_url=sqlite:///" + workspace.dbLocation, 

227 "--config", "diaPipe:apdb.isolation_level=READ_UNCOMMITTED", 

228 # Put output alerts into the workspace. 

229 "--config", "diaPipe:alertPackager.alertWriteLocation=" + workspace.alertLocation, 

230 "--config", "diaPipe:doPackageAlerts=True", 

231 # TODO: the configs below should not be needed after DM-26140 

232 "--configfile", "calibrate:" + os.path.join(workspace.configDir, "calibrate.py"), 

233 "--configfile", "imageDifference:" + os.path.join(workspace.configDir, "imageDifference.py"), 

234 ] 

235 # TODO: reverse-engineering the instrument should not be needed after DM-26140 

236 # pipetask will crash if there is more than one instrument 

237 for idRecord in workspace.workButler.registry.queryDataIds("instrument").expanded(): 

238 className = idRecord.records["instrument"].class_name 

239 args.extend(["--instrument", className]) 

240 

241 return args 

242 

243 

244def _getCollectionArguments(workspace): 

245 """Return the collections for running the Gen 3 AP Pipeline on this 

246 workspace, as command-line arguments. 

247 

248 Parameters 

249 ---------- 

250 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

251 A Workspace with a Gen 3 repository. 

252 

253 Returns 

254 ------- 

255 args : `list` of `str` 

256 Command-line arguments calling ``--input`` or ``--output``, 

257 following the conventions of `sys.argv`. 

258 """ 

259 butler = workspace.workButler 

260 inputs = set(butler.registry.queryCollections(collectionTypes={dafButler.CollectionType.RUN})) 

261 inputs.discard(workspace.runName) 

262 return ["--input", ",".join(inputs), 

263 "--output-run", workspace.runName, 

264 ]