Coverage for python/lsst/ap/verify/pipeline_driver.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24"""Interface between `ap_verify` and `ap_pipe`.
26This module handles calling `ap_pipe` and converting any information
27as needed.
28"""
30__all__ = ["ApPipeParser", "runApPipe"]
32import argparse
33import os
35import lsst.log
36import lsst.pipe.base as pipeBase
37import lsst.ap.pipe as apPipe
38from lsst.ap.pipe.make_apdb import makeApdb
41class ApPipeParser(argparse.ArgumentParser):
42 """An argument parser for data needed by ``ap_pipe`` activities.
44 This parser is not complete, and is designed to be passed to another parser
45 using the `parent` parameter.
46 """
48 def __init__(self):
49 # Help and documentation will be handled by main program's parser
50 argparse.ArgumentParser.__init__(self, add_help=False)
51 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs.
52 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'.
53 self.add_argument('--id', dest='dataIds', action=self.AppendOptional, nargs='?', default=[],
54 help='An identifier for the data to process.')
55 self.add_argument("-j", "--processes", default=1, type=int,
56 help="Number of processes to use.")
57 self.add_argument("--skip-pipeline", action="store_true",
58 help="Do not run the AP pipeline itself. This argument is useful "
59 "for testing metrics on a fixed data set.")
61 class AppendOptional(argparse.Action):
62 """A variant of the built-in "append" action that ignores None values
63 instead of appending them.
64 """
65 # This class can't safely inherit from the built-in "append" action
66 # because there is no public class that implements it.
67 def __call__(self, parser, namespace, values, option_string=None):
68 if values is not None:
69 try:
70 allValues = getattr(namespace, self.dest)
71 allValues.append(values)
72 except AttributeError:
73 setattr(namespace, self.dest, [values])
76def runApPipe(workspace, parsedCmdLine):
77 """Run `ap_pipe` on this object's dataset.
79 Parameters
80 ----------
81 workspace : `lsst.ap.verify.workspace.WorkspaceGen2`
82 The abstract location containing input and output repositories.
83 parsedCmdLine : `argparse.Namespace`
84 Command-line arguments, including all arguments supported by `ApPipeParser`.
86 Returns
87 -------
88 apPipeReturn : `Struct`
89 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with
90 ``doReturnResults=False``. This object is valid even if
91 `~lsst.ap.pipe.ApPipeTask` was never run.
92 """
93 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipe')
95 configArgs = _getConfigArguments(workspace)
96 makeApdb(configArgs)
98 pipelineArgs = [workspace.dataRepo,
99 "--output", workspace.outputRepo,
100 "--calib", workspace.calibRepo,
101 "--template", workspace.templateRepo]
102 pipelineArgs.extend(configArgs)
103 if parsedCmdLine.dataIds:
104 for singleId in parsedCmdLine.dataIds:
105 pipelineArgs.extend(["--id", *singleId.split(" ")])
106 else:
107 pipelineArgs.extend(["--id"])
108 pipelineArgs.extend(["--processes", str(parsedCmdLine.processes)])
109 pipelineArgs.extend(["--noExit"])
111 if not parsedCmdLine.skip_pipeline:
112 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs)
113 log.info('Pipeline complete')
114 else:
115 log.info('Skipping AP pipeline entirely.')
116 apPipeParser = apPipe.ApPipeTask._makeArgumentParser()
117 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs)
118 results = pipeBase.Struct(
119 argumentParser=apPipeParser,
120 parsedCmd=apPipeParsed,
121 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed),
122 resultList=[],
123 )
125 return results
128def _getConfigArguments(workspace):
129 """Return the config options for running ApPipeTask on this workspace, as
130 command-line arguments.
132 Parameters
133 ----------
134 workspace : `lsst.ap.verify.workspace.Workspace`
135 A Workspace whose config directory may contain an
136 `~lsst.ap.pipe.ApPipeTask` config.
138 Returns
139 -------
140 args : `list` of `str`
141 Command-line arguments calling ``--config`` or ``--configFile``,
142 following the conventions of `sys.argv`.
143 """
144 overrideFile = apPipe.ApPipeTask._DefaultName + ".py"
145 overridePath = os.path.join(workspace.configDir, overrideFile)
147 args = ["--configfile", overridePath]
148 # ApVerify will use the sqlite hooks for the Apdb.
149 args.extend(["--config", "diaPipe.apdb.db_url=sqlite:///" + workspace.dbLocation])
150 args.extend(["--config", "diaPipe.apdb.isolation_level=READ_UNCOMMITTED"])
151 # Put output alerts into the workspace.
152 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.workDir + "/alerts"])
153 args.extend(["--config", "diaPipe.doPackageAlerts=True"])
155 return args