Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Interface between `ap_verify` and `ap_pipe`. 

25 

26This module handles calling `ap_pipe` and converting any information 

27as needed. 

28""" 

29 

30__all__ = ["ApPipeParser", "runApPipeGen2", "runApPipeGen3"] 

31 

32import argparse 

33import os 

34import re 

35 

36import lsst.log 

37from lsst.utils import getPackageDir 

38import lsst.pipe.base as pipeBase 

39import lsst.obs.base as obsBase 

40import lsst.ctrl.mpexec as ctrlMpexec 

41import lsst.ap.pipe as apPipe 

42from lsst.ap.pipe.make_apdb import makeApdb 

43 

44 

45class ApPipeParser(argparse.ArgumentParser): 

46 """An argument parser for data needed by ``ap_pipe`` activities. 

47 

48 This parser is not complete, and is designed to be passed to another parser 

49 using the `parent` parameter. 

50 """ 

51 

52 def __init__(self): 

53 defaultPipeline = os.path.join(getPackageDir("ap_verify"), "pipelines", "ApVerify.yaml") 

54 

55 # Help and documentation will be handled by main program's parser 

56 argparse.ArgumentParser.__init__(self, add_help=False) 

57 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs. 

58 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'. 

59 self.add_argument('--id', dest='dataIds', action=self.AppendOptional, nargs='?', default=[], 

60 help='An identifier for the data to process.') 

61 self.add_argument("-p", "--pipeline", default=defaultPipeline, 

62 help="A custom version of the ap_verify pipeline (e.g., with different metrics).") 

63 self.add_argument("--skip-pipeline", action="store_true", 

64 help="Do not run the AP pipeline itself. This argument is useful " 

65 "for testing metrics on a fixed data set.") 

66 

67 class AppendOptional(argparse.Action): 

68 """A variant of the built-in "append" action that ignores None values 

69 instead of appending them. 

70 """ 

71 # This class can't safely inherit from the built-in "append" action 

72 # because there is no public class that implements it. 

73 def __call__(self, parser, namespace, values, option_string=None): 

74 if values is not None: 

75 try: 

76 allValues = getattr(namespace, self.dest) 

77 allValues.append(values) 

78 except AttributeError: 

79 setattr(namespace, self.dest, [values]) 

80 

81 

82def runApPipeGen2(workspace, parsedCmdLine, processes=1): 

83 """Run `ap_pipe` on this object's dataset. 

84 

85 Parameters 

86 ---------- 

87 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

88 The abstract location containing input and output repositories. 

89 parsedCmdLine : `argparse.Namespace` 

90 Command-line arguments, including all arguments supported by `ApPipeParser`. 

91 processes : `int` 

92 The number of processes with which to call the AP pipeline 

93 

94 Returns 

95 ------- 

96 apPipeReturn : `Struct` 

97 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with 

98 ``doReturnResults=False``. This object is valid even if 

99 `~lsst.ap.pipe.ApPipeTask` was never run. 

100 """ 

101 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen2') 

102 

103 configArgs = _getConfigArguments(workspace) 

104 makeApdb(configArgs) 

105 

106 pipelineArgs = [workspace.dataRepo, 

107 "--output", workspace.outputRepo, 

108 "--calib", workspace.calibRepo, 

109 "--template", workspace.templateRepo] 

110 pipelineArgs.extend(configArgs) 

111 if parsedCmdLine.dataIds: 

112 for singleId in parsedCmdLine.dataIds: 

113 pipelineArgs.extend(["--id", *singleId.split(" ")]) 

114 else: 

115 pipelineArgs.extend(["--id"]) 

116 pipelineArgs.extend(["--processes", str(processes)]) 

117 pipelineArgs.extend(["--noExit"]) 

118 

119 if not parsedCmdLine.skip_pipeline: 

120 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs) 

121 log.info('Pipeline complete') 

122 else: 

123 log.info('Skipping AP pipeline entirely.') 

124 apPipeParser = apPipe.ApPipeTask._makeArgumentParser() 

125 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs) 

126 results = pipeBase.Struct( 

127 argumentParser=apPipeParser, 

128 parsedCmd=apPipeParsed, 

129 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed), 

130 resultList=[], 

131 ) 

132 

133 return results 

134 

135 

136def runApPipeGen3(workspace, parsedCmdLine, processes=1): 

137 """Run `ap_pipe` on this object's dataset. 

138 

139 Parameters 

140 ---------- 

141 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

142 The abstract location containing input and output repositories. 

143 parsedCmdLine : `argparse.Namespace` 

144 Command-line arguments, including all arguments supported by `ApPipeParser`. 

145 processes : `int` 

146 The number of processes with which to call the AP pipeline 

147 """ 

148 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen3') 

149 

150 # Currently makeApdb has different argument conventions from Gen 3; see DM-22663 

151 makeApdb(_getConfigArguments(workspace)) 

152 

153 pipelineArgs = ["run", 

154 "--butler-config", workspace.repo, 

155 "--pipeline", parsedCmdLine.pipeline, 

156 ] 

157 # TODO: collections should be determined exclusively by Workspace.workButler, 

158 # but I can't find a way to hook that up to the graph builder. So use the CLI 

159 # for now and revisit once DM-26239 is done. 

160 pipelineArgs.extend(_getCollectionArguments(workspace)) 

161 pipelineArgs.extend(_getConfigArgumentsGen3(workspace)) 

162 if parsedCmdLine.dataIds: 

163 for singleId in parsedCmdLine.dataIds: 

164 pipelineArgs.extend(["--data-query", singleId]) 

165 pipelineArgs.extend(["--processes", str(processes)]) 

166 pipelineArgs.extend(["--register-dataset-types"]) 

167 

168 if not parsedCmdLine.skip_pipeline: 

169 # TODO: generalize this code in DM-26028 

170 activator = ctrlMpexec.CmdLineFwk() 

171 # TODO: work off of workspace.workButler after DM-26239 

172 results = activator.parseAndRun(pipelineArgs) 

173 

174 log.info('Pipeline complete.') 

175 return results 

176 else: 

177 log.info('Skipping AP pipeline entirely.') 

178 

179 

180def _getConfigArguments(workspace): 

181 """Return the config options for running ApPipeTask on this workspace, as 

182 command-line arguments. 

183 

184 Parameters 

185 ---------- 

186 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

187 A Workspace whose config directory may contain an 

188 `~lsst.ap.pipe.ApPipeTask` config. 

189 

190 Returns 

191 ------- 

192 args : `list` of `str` 

193 Command-line arguments calling ``--config`` or ``--configFile``, 

194 following the conventions of `sys.argv`. 

195 """ 

196 overrideFile = apPipe.ApPipeTask._DefaultName + ".py" 

197 overridePath = os.path.join(workspace.configDir, overrideFile) 

198 

199 args = ["--configfile", overridePath] 

200 # ApVerify will use the sqlite hooks for the Apdb. 

201 args.extend(["--config", "diaPipe.apdb.db_url=sqlite:///" + workspace.dbLocation]) 

202 args.extend(["--config", "diaPipe.apdb.isolation_level=READ_UNCOMMITTED"]) 

203 # Put output alerts into the workspace. 

204 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.alertLocation]) 

205 args.extend(["--config", "diaPipe.doPackageAlerts=True"]) 

206 

207 return args 

208 

209 

210def _getConfigArgumentsGen3(workspace): 

211 """Return the config options for running the Gen 3 AP Pipeline on this 

212 workspace, as command-line arguments. 

213 

214 Parameters 

215 ---------- 

216 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

217 A Workspace whose config directory may contain various configs. 

218 

219 Returns 

220 ------- 

221 args : `list` of `str` 

222 Command-line arguments calling ``--config`` or ``--configFile``, 

223 following the conventions of `sys.argv`. 

224 """ 

225 args = [ 

226 # ApVerify will use the sqlite hooks for the Apdb. 

227 "--config", "diaPipe:apdb.db_url=sqlite:///" + workspace.dbLocation, 

228 "--config", "diaPipe:apdb.isolation_level=READ_UNCOMMITTED", 

229 # Put output alerts into the workspace. 

230 "--config", "diaPipe:alertPackager.alertWriteLocation=" + workspace.alertLocation, 

231 "--config", "diaPipe:doPackageAlerts=True", 

232 # TODO: the configs below should not be needed after DM-26140 

233 "--configfile", "calibrate:" + os.path.join(workspace.configDir, "calibrate.py"), 

234 "--configfile", "imageDifference:" + os.path.join(workspace.configDir, "imageDifference.py"), 

235 ] 

236 # TODO: reverse-engineering the instrument should not be needed after DM-26140 

237 # pipetask will crash if there is more than one instrument 

238 for idRecord in workspace.workButler.registry.queryDataIds("instrument").expanded(): 

239 className = idRecord.records["instrument"].class_name 

240 args.extend(["--instrument", className]) 

241 

242 return args 

243 

244 

245def _getCollectionArguments(workspace): 

246 """Return the collections for running the Gen 3 AP Pipeline on this 

247 workspace, as command-line arguments. 

248 

249 Parameters 

250 ---------- 

251 workspace : `lsst.ap.verify.workspace.WorkspaceGen3` 

252 A Workspace with a Gen 3 repository. 

253 

254 Returns 

255 ------- 

256 args : `list` of `str` 

257 Command-line arguments calling ``--input`` or ``--output``, 

258 following the conventions of `sys.argv`. 

259 """ 

260 butler = workspace.workButler 

261 # Hard-code the collection names because it's hard to infer the inputs from the Butler 

262 inputs = {"skymaps", "refcats"} 

263 for dimension in butler.registry.queryDataIds('instrument'): 

264 instrument = obsBase.Instrument.fromName(dimension["instrument"], butler.registry) 

265 inputs.add(instrument.makeDefaultRawIngestRunName()) 

266 inputs.add(instrument.makeCalibrationCollectionName()) 

267 inputs.update(butler.registry.queryCollections(re.compile(r"templates/\w+"))) 

268 

269 return ["--input", ",".join(inputs), 

270 "--output-run", workspace.runName, 

271 ]