Coverage for python/lsst/ap/verify/pipeline_driver.py : 18%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Interface between `ap_verify` and `ap_pipe`.
26This module handles calling `ap_pipe` and converting any information
27as needed.
28"""
30__all__ = ["ApPipeParser", "runApPipeGen2", "runApPipeGen3"]
32import argparse
33import os
34import re
36import click.testing
38import lsst.log
39from lsst.utils import getPackageDir
40import lsst.pipe.base as pipeBase
41import lsst.ctrl.mpexec.cli.pipetask
42import lsst.ap.pipe as apPipe
43from lsst.ap.pipe.make_apdb import makeApdb
46class ApPipeParser(argparse.ArgumentParser):
47 """An argument parser for data needed by ``ap_pipe`` activities.
49 This parser is not complete, and is designed to be passed to another parser
50 using the `parent` parameter.
51 """
53 def __init__(self):
54 defaultPipeline = os.path.join(getPackageDir("ap_verify"), "pipelines", "ApVerify.yaml")
56 # Help and documentation will be handled by main program's parser
57 argparse.ArgumentParser.__init__(self, add_help=False)
58 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs.
59 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'.
60 self.add_argument('--id', '-d', '--data-query', dest='dataIds',
61 action=self.AppendOptional, nargs='?', default=[],
62 help='An identifier for the data to process.')
63 self.add_argument("-p", "--pipeline", default=defaultPipeline,
64 help="A custom version of the ap_verify pipeline (e.g., with different metrics).")
65 self.add_argument("--db", "--db_url", default=None,
66 help="A location for the AP database, formatted as if for ApdbConfig.db_url. "
67 "Defaults to an SQLite file in the --output directory.")
68 self.add_argument("--skip-pipeline", action="store_true",
69 help="Do not run the AP pipeline itself. This argument is useful "
70 "for testing metrics on a fixed data set.")
71 self.add_argument("--clean-run", action="store_true",
72 help="Run the pipeline with a new run collection, "
73 "even if one already exists.")
75 class AppendOptional(argparse.Action):
76 """A variant of the built-in "append" action that ignores None values
77 instead of appending them.
78 """
79 # This class can't safely inherit from the built-in "append" action
80 # because there is no public class that implements it.
81 def __call__(self, parser, namespace, values, option_string=None):
82 if values is not None:
83 try:
84 allValues = getattr(namespace, self.dest)
85 allValues.append(values)
86 except AttributeError:
87 setattr(namespace, self.dest, [values])
90def runApPipeGen2(workspace, parsedCmdLine, processes=1):
91 """Run `ap_pipe` on this object's dataset.
93 Parameters
94 ----------
95 workspace : `lsst.ap.verify.workspace.WorkspaceGen2`
96 The abstract location containing input and output repositories.
97 parsedCmdLine : `argparse.Namespace`
98 Command-line arguments, including all arguments supported by `ApPipeParser`.
99 processes : `int`
100 The number of processes with which to call the AP pipeline
102 Returns
103 -------
104 apPipeReturn : `Struct`
105 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with
106 ``doReturnResults=False``. This object is valid even if
107 `~lsst.ap.pipe.ApPipeTask` was never run.
108 """
109 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen2')
111 makeApdb(_getApdbArguments(workspace, parsedCmdLine))
113 pipelineArgs = [workspace.dataRepo,
114 "--output", workspace.outputRepo,
115 "--calib", workspace.calibRepo,
116 "--template", workspace.templateRepo]
117 pipelineArgs.extend(_getConfigArguments(workspace, parsedCmdLine))
118 if parsedCmdLine.dataIds:
119 for singleId in parsedCmdLine.dataIds:
120 pipelineArgs.extend(["--id", *singleId.split(" ")])
121 else:
122 pipelineArgs.extend(["--id"])
123 pipelineArgs.extend(["--processes", str(processes)])
124 pipelineArgs.extend(["--noExit"])
126 if not parsedCmdLine.skip_pipeline:
127 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs)
128 log.info('Pipeline complete')
129 else:
130 log.info('Skipping AP pipeline entirely.')
131 apPipeParser = apPipe.ApPipeTask._makeArgumentParser()
132 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs)
133 results = pipeBase.Struct(
134 argumentParser=apPipeParser,
135 parsedCmd=apPipeParsed,
136 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed),
137 resultList=[],
138 )
140 return results
143def runApPipeGen3(workspace, parsedCmdLine, processes=1):
144 """Run `ap_pipe` on this object's dataset.
146 Parameters
147 ----------
148 workspace : `lsst.ap.verify.workspace.WorkspaceGen3`
149 The abstract location containing input and output repositories.
150 parsedCmdLine : `argparse.Namespace`
151 Command-line arguments, including all arguments supported by `ApPipeParser`.
152 processes : `int`
153 The number of processes with which to call the AP pipeline
154 """
155 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen3')
157 makeApdb(_getApdbArguments(workspace, parsedCmdLine))
159 pipelineArgs = ["run",
160 "--butler-config", workspace.repo,
161 "--pipeline", parsedCmdLine.pipeline,
162 ]
163 # TODO: collections should be determined exclusively by Workspace.workButler,
164 # but I can't find a way to hook that up to the graph builder. So use the CLI
165 # for now and revisit once DM-26239 is done.
166 pipelineArgs.extend(_getCollectionArguments(workspace, reuse=(not parsedCmdLine.clean_run)))
167 pipelineArgs.extend(_getConfigArgumentsGen3(workspace, parsedCmdLine))
168 if parsedCmdLine.dataIds:
169 for singleId in parsedCmdLine.dataIds:
170 pipelineArgs.extend(["--data-query", singleId])
171 pipelineArgs.extend(["--processes", str(processes)])
172 pipelineArgs.extend(["--register-dataset-types"])
174 if not parsedCmdLine.skip_pipeline:
175 # CliRunner is an unsafe workaround for DM-26239
176 runner = click.testing.CliRunner()
177 # TODO: generalize this code in DM-26028
178 # TODO: work off of workspace.workButler after DM-26239
179 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs)
180 if results.exception:
181 raise RuntimeError("Pipeline failed.") from results.exception
183 log.info('Pipeline complete.')
184 return results.exit_code
185 else:
186 log.info('Skipping AP pipeline entirely.')
189def _getApdbArguments(workspace, parsed):
190 """Return the config options for running make_apdb.py on this workspace,
191 as command-line arguments.
193 Parameters
194 ----------
195 workspace : `lsst.ap.verify.workspace.Workspace`
196 A Workspace whose config directory may contain an
197 `~lsst.ap.pipe.ApPipeTask` config.
198 parsed : `argparse.Namespace`
199 Command-line arguments, including all arguments supported by `ApPipeParser`.
201 Returns
202 -------
203 args : `list` of `str`
204 Command-line arguments calling ``--config`` or ``--config-file``,
205 following the conventions of `sys.argv`.
206 """
207 if not parsed.db:
208 parsed.db = "sqlite:///" + workspace.dbLocation
210 args = ["--config", "db_url=" + parsed.db]
211 # Same special-case check as ApdbConfig.validate()
212 if parsed.db.startswith("sqlite"):
213 args.extend(["--config", "isolation_level=READ_UNCOMMITTED"])
215 return args
218def _getConfigArguments(workspace, parsed):
219 """Return the config options for running ApPipeTask on this workspace, as
220 command-line arguments.
222 Parameters
223 ----------
224 workspace : `lsst.ap.verify.workspace.WorkspaceGen2`
225 A Workspace whose config directory may contain an
226 `~lsst.ap.pipe.ApPipeTask` config.
227 parsed : `argparse.Namespace`
228 Command-line arguments, including all arguments supported by `ApPipeParser`.
230 Returns
231 -------
232 args : `list` of `str`
233 Command-line arguments calling ``--config`` or ``--configfile``,
234 following the conventions of `sys.argv`.
235 """
236 overrideFile = apPipe.ApPipeTask._DefaultName + ".py"
237 overridePath = os.path.join(workspace.configDir, overrideFile)
239 args = ["--configfile", overridePath]
240 # Translate APDB-only arguments to work as a sub-config
241 args.extend([("diaPipe.apdb." + arg if arg != "--config" else arg)
242 for arg in _getApdbArguments(workspace, parsed)])
243 # Put output alerts into the workspace.
244 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.alertLocation])
245 args.extend(["--config", "diaPipe.doPackageAlerts=True"])
247 return args
250def _getConfigArgumentsGen3(workspace, parsed):
251 """Return the config options for running the Gen 3 AP Pipeline on this
252 workspace, as command-line arguments.
254 Parameters
255 ----------
256 workspace : `lsst.ap.verify.workspace.WorkspaceGen3`
257 A Workspace whose config directory may contain various configs.
258 parsed : `argparse.Namespace`
259 Command-line arguments, including all arguments supported by `ApPipeParser`.
261 Returns
262 -------
263 args : `list` of `str`
264 Command-line arguments calling ``--config`` or ``--config-file``,
265 following the conventions of `sys.argv`.
266 """
267 # Translate APDB-only arguments to work as a sub-config
268 args = [("diaPipe:apdb." + arg if arg != "--config" else arg)
269 for arg in _getApdbArguments(workspace, parsed)]
270 args.extend([
271 # Put output alerts into the workspace.
272 "--config", "diaPipe:alertPackager.alertWriteLocation=" + workspace.alertLocation,
273 "--config", "diaPipe:doPackageAlerts=True",
274 # TODO: the configs below should not be needed after DM-26140
275 "--config-file", "calibrate:" + os.path.join(workspace.configDir, "calibrate.py"),
276 "--config-file", "imageDifference:" + os.path.join(workspace.configDir, "imageDifference.py"),
277 ])
278 # TODO: reverse-engineering the instrument should not be needed after DM-26140
279 # pipetask will crash if there is more than one instrument
280 for idRecord in workspace.workButler.registry.queryDataIds("instrument").expanded():
281 className = idRecord.records["instrument"].class_name
282 args.extend(["--instrument", className])
284 return args
287def _getCollectionArguments(workspace, reuse):
288 """Return the collections for running the Gen 3 AP Pipeline on this
289 workspace, as command-line arguments.
291 Parameters
292 ----------
293 workspace : `lsst.ap.verify.workspace.WorkspaceGen3`
294 A Workspace with a Gen 3 repository.
295 reuse : `bool`
296 If true, use the previous run collection if one exists. Otherwise,
297 create a new run.
299 Returns
300 -------
301 args : `list` of `str`
302 Command-line arguments calling ``--input`` or ``--output``,
303 following the conventions of `sys.argv`.
304 """
305 # workspace.outputName is a chained collection containing all inputs
306 args = ["--output", workspace.outputName,
307 "--clobber-partial-outputs",
308 ]
310 registry = workspace.workButler.registry
311 oldRuns = list(registry.queryCollections(re.compile(workspace.outputName + r"/\d+T\d+Z")))
312 if reuse and oldRuns:
313 args.extend(["--extend-run", "--skip-existing"])
314 return args