Coverage for python/lsst/ap/verify/pipeline_driver.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Interface between `ap_verify` and `ap_pipe`.
26This module handles calling `ap_pipe` and converting any information
27as needed.
28"""
30__all__ = ["ApPipeParser", "runApPipeGen2", "runApPipeGen3"]
32import argparse
33import os
35import lsst.log
36from lsst.utils import getPackageDir
37import lsst.daf.butler as dafButler
38import lsst.pipe.base as pipeBase
39import lsst.ctrl.mpexec as ctrlMpexec
40import lsst.ap.pipe as apPipe
41from lsst.ap.pipe.make_apdb import makeApdb
44class ApPipeParser(argparse.ArgumentParser):
45 """An argument parser for data needed by ``ap_pipe`` activities.
47 This parser is not complete, and is designed to be passed to another parser
48 using the `parent` parameter.
49 """
51 def __init__(self):
52 defaultPipeline = os.path.join(getPackageDir("ap_verify"), "pipelines", "ApVerify.yaml")
54 # Help and documentation will be handled by main program's parser
55 argparse.ArgumentParser.__init__(self, add_help=False)
56 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs.
57 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'.
58 self.add_argument('--id', dest='dataIds', action=self.AppendOptional, nargs='?', default=[],
59 help='An identifier for the data to process.')
60 self.add_argument("-p", "--pipeline", default=defaultPipeline,
61 help="A custom version of the ap_verify pipeline (e.g., with different metrics).")
62 self.add_argument("--skip-pipeline", action="store_true",
63 help="Do not run the AP pipeline itself. This argument is useful "
64 "for testing metrics on a fixed data set.")
66 class AppendOptional(argparse.Action):
67 """A variant of the built-in "append" action that ignores None values
68 instead of appending them.
69 """
70 # This class can't safely inherit from the built-in "append" action
71 # because there is no public class that implements it.
72 def __call__(self, parser, namespace, values, option_string=None):
73 if values is not None:
74 try:
75 allValues = getattr(namespace, self.dest)
76 allValues.append(values)
77 except AttributeError:
78 setattr(namespace, self.dest, [values])
81def runApPipeGen2(workspace, parsedCmdLine, processes=1):
82 """Run `ap_pipe` on this object's dataset.
84 Parameters
85 ----------
86 workspace : `lsst.ap.verify.workspace.WorkspaceGen2`
87 The abstract location containing input and output repositories.
88 parsedCmdLine : `argparse.Namespace`
89 Command-line arguments, including all arguments supported by `ApPipeParser`.
90 processes : `int`
91 The number of processes with which to call the AP pipeline
93 Returns
94 -------
95 apPipeReturn : `Struct`
96 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with
97 ``doReturnResults=False``. This object is valid even if
98 `~lsst.ap.pipe.ApPipeTask` was never run.
99 """
100 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen2')
102 configArgs = _getConfigArguments(workspace)
103 makeApdb(configArgs)
105 pipelineArgs = [workspace.dataRepo,
106 "--output", workspace.outputRepo,
107 "--calib", workspace.calibRepo,
108 "--template", workspace.templateRepo]
109 pipelineArgs.extend(configArgs)
110 if parsedCmdLine.dataIds:
111 for singleId in parsedCmdLine.dataIds:
112 pipelineArgs.extend(["--id", *singleId.split(" ")])
113 else:
114 pipelineArgs.extend(["--id"])
115 pipelineArgs.extend(["--processes", str(processes)])
116 pipelineArgs.extend(["--noExit"])
118 if not parsedCmdLine.skip_pipeline:
119 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs)
120 log.info('Pipeline complete')
121 else:
122 log.info('Skipping AP pipeline entirely.')
123 apPipeParser = apPipe.ApPipeTask._makeArgumentParser()
124 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs)
125 results = pipeBase.Struct(
126 argumentParser=apPipeParser,
127 parsedCmd=apPipeParsed,
128 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed),
129 resultList=[],
130 )
132 return results
135def runApPipeGen3(workspace, parsedCmdLine, processes=1):
136 """Run `ap_pipe` on this object's dataset.
138 Parameters
139 ----------
140 workspace : `lsst.ap.verify.workspace.WorkspaceGen3`
141 The abstract location containing input and output repositories.
142 parsedCmdLine : `argparse.Namespace`
143 Command-line arguments, including all arguments supported by `ApPipeParser`.
144 processes : `int`
145 The number of processes with which to call the AP pipeline
146 """
147 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipeGen3')
149 # Currently makeApdb has different argument conventions from Gen 3; see DM-22663
150 makeApdb(_getConfigArguments(workspace))
152 pipelineArgs = ["run",
153 "--butler-config", workspace.repo,
154 "--pipeline", parsedCmdLine.pipeline,
155 ]
156 # TODO: collections should be determined exclusively by Workspace.workButler,
157 # but I can't find a way to hook that up to the graph builder. So use the CLI
158 # for now and revisit once DM-26239 is done.
159 pipelineArgs.extend(_getCollectionArguments(workspace))
160 pipelineArgs.extend(_getConfigArgumentsGen3(workspace))
161 if parsedCmdLine.dataIds:
162 for singleId in parsedCmdLine.dataIds:
163 pipelineArgs.extend(["--data-query", singleId])
164 pipelineArgs.extend(["--processes", str(processes)])
165 pipelineArgs.extend(["--register-dataset-types"])
167 if not parsedCmdLine.skip_pipeline:
168 # TODO: generalize this code in DM-26028
169 activator = ctrlMpexec.CmdLineFwk()
170 # TODO: work off of workspace.workButler after DM-26239
171 results = activator.parseAndRun(pipelineArgs)
173 log.info('Pipeline complete.')
174 return results
175 else:
176 log.info('Skipping AP pipeline entirely.')
179def _getConfigArguments(workspace):
180 """Return the config options for running ApPipeTask on this workspace, as
181 command-line arguments.
183 Parameters
184 ----------
185 workspace : `lsst.ap.verify.workspace.WorkspaceGen2`
186 A Workspace whose config directory may contain an
187 `~lsst.ap.pipe.ApPipeTask` config.
189 Returns
190 -------
191 args : `list` of `str`
192 Command-line arguments calling ``--config`` or ``--configFile``,
193 following the conventions of `sys.argv`.
194 """
195 overrideFile = apPipe.ApPipeTask._DefaultName + ".py"
196 overridePath = os.path.join(workspace.configDir, overrideFile)
198 args = ["--configfile", overridePath]
199 # ApVerify will use the sqlite hooks for the Apdb.
200 args.extend(["--config", "diaPipe.apdb.db_url=sqlite:///" + workspace.dbLocation])
201 args.extend(["--config", "diaPipe.apdb.isolation_level=READ_UNCOMMITTED"])
202 # Put output alerts into the workspace.
203 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.alertLocation])
204 args.extend(["--config", "diaPipe.doPackageAlerts=True"])
206 return args
209def _getConfigArgumentsGen3(workspace):
210 """Return the config options for running the Gen 3 AP Pipeline on this
211 workspace, as command-line arguments.
213 Parameters
214 ----------
215 workspace : `lsst.ap.verify.workspace.WorkspaceGen3`
216 A Workspace whose config directory may contain various configs.
218 Returns
219 -------
220 args : `list` of `str`
221 Command-line arguments calling ``--config`` or ``--configFile``,
222 following the conventions of `sys.argv`.
223 """
224 args = [
225 # ApVerify will use the sqlite hooks for the Apdb.
226 "--config", "diaPipe:apdb.db_url=sqlite:///" + workspace.dbLocation,
227 "--config", "diaPipe:apdb.isolation_level=READ_UNCOMMITTED",
228 # Put output alerts into the workspace.
229 "--config", "diaPipe:alertPackager.alertWriteLocation=" + workspace.alertLocation,
230 "--config", "diaPipe:doPackageAlerts=True",
231 # TODO: the configs below should not be needed after DM-26140
232 "--configfile", "calibrate:" + os.path.join(workspace.configDir, "calibrate.py"),
233 "--configfile", "imageDifference:" + os.path.join(workspace.configDir, "imageDifference.py"),
234 ]
235 # TODO: reverse-engineering the instrument should not be needed after DM-26140
236 # pipetask will crash if there is more than one instrument
237 for idRecord in workspace.workButler.registry.queryDataIds("instrument").expanded():
238 className = idRecord.records["instrument"].class_name
239 args.extend(["--instrument", className])
241 return args
244def _getCollectionArguments(workspace):
245 """Return the collections for running the Gen 3 AP Pipeline on this
246 workspace, as command-line arguments.
248 Parameters
249 ----------
250 workspace : `lsst.ap.verify.workspace.WorkspaceGen3`
251 A Workspace with a Gen 3 repository.
253 Returns
254 -------
255 args : `list` of `str`
256 Command-line arguments calling ``--input`` or ``--output``,
257 following the conventions of `sys.argv`.
258 """
259 butler = workspace.workButler
260 inputs = set(butler.registry.queryCollections(collectionTypes={dafButler.CollectionType.RUN}))
261 inputs.discard(workspace.runName)
262 return ["--input", ",".join(inputs),
263 "--output-run", workspace.runName,
264 ]