lsst.pipe.base  16.0-25-g2c6bf4a
pipelineBuilder.py
Go to the documentation of this file.
1 # This file is part of pipe_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Module defining PipelineBuilder class and related methods.
23 """
24 
25 __all__ = ["PipelineBuilder"]
26 
27 # -------------------------------
28 # Imports of standard modules --
29 # -------------------------------
30 import ast
31 import logging
32 
33 # -----------------------------
34 # Imports for other modules --
35 # -----------------------------
36 from .configOverrides import ConfigOverrides
37 from .pipeline import Pipeline, TaskDef
38 from . import pipeTools
39 import lsst.pex.exceptions as pexExceptions
40 
41 # ----------------------------------
42 # Local non-exported definitions --
43 # ----------------------------------
44 
45 _LOG = logging.getLogger(__name__.partition(".")[2])
46 
47 # ------------------------
48 # Exported definitions --
49 # ------------------------
50 
51 
53  """PipelineBuilder class is responsible for building task pipeline.
54 
55  The class provides a set of methods to manipulate pipeline by adding,
56  deleting, re-ordering tasks in pipeline and changing their labels or
57  configuration.
58 
59  Parameters
60  ----------
61  taskFactory : `TaskFactory`
62  Factory object used to load/instantiate PipelineTasks
63  pipeline : `Pipeline`, optional
64  Initial pipeline to be modified, if `None` then new empty pipeline
65  will be created.
66  """
67  def __init__(self, taskFactory, pipeline=None):
68  if pipeline is None:
69  pipeline = Pipeline()
70  self._taskFactory = taskFactory
71  self._pipeline = pipeline
72 
73  def pipeline(self, ordered=False):
74  """Return updated pipeline instance.
75 
76  Pipeline will be checked for possible inconsistencies before
77  returning.
78 
79  Parameters
80  ----------
81  ordered : `bool`, optional
82  If `True` then order resulting pipeline according to Task data
83  dependencies.
84 
85  Returns
86  -------
87  pipeline : `Pipeline`
88 
89  Raises
90  ------
91  Exception
92  Raised if any inconsistencies are detected in pipeline definition,
93  see `pipeTools.orderPipeline` for list of exception types.
94  """
95  # conditionally re-order pipeline if requested, but unconditionally
96  # check for possible errors
97  orderedPipeline = pipeTools.orderPipeline(self._pipeline, self._taskFactory)
98  if ordered:
99  return orderedPipeline
100  else:
101  return self._pipeline
102 
103  def addTask(self, taskName, label=None):
104  """Append new task to a pipeline.
105 
106  Parameters
107  ----------
108  taskName : `str`
109  Name of the new task, can be either full class name including
110  package and module, or just a class name to be searched in
111  known packages and modules.
112  label : `str`, optional
113  Label for new task, if `None` then task class name is used as
114  label.
115  """
116  # load task class, will throw on errors
117  taskClass, taskName = self._taskFactory.loadTaskClass(taskName)
118 
119  # get label and check that it is unique
120  if not label:
121  label = taskName.rpartition('.')[2]
122  if self._pipeline.labelIndex(label) >= 0:
123  raise LookupError("Task label (or name) is not unique: " + label)
124 
125  # make config instance with defaults
126  config = taskClass.ConfigClass()
127 
128  self._pipeline.append(TaskDef(taskName=taskName, config=config,
129  taskClass=taskClass, label=label))
130 
131  def deleteTask(self, label):
132  """Remove task from a pipeline.
133 
134  Parameters
135  ----------
136  label : `str`
137  Label of the task to remove.
138  """
139  idx = self._pipeline.labelIndex(label)
140  if idx < 0:
141  raise LookupError("Task label is not found: " + label)
142  del self._pipeline[idx]
143 
144  def moveTask(self, label, newIndex):
145  """Move task to a new position in a pipeline.
146 
147  Parameters
148  ----------
149  label : `str`
150  Label of the task to move.
151  newIndex : `int`
152  New position.
153  """
154  idx = self._pipeline.labelIndex(label)
155  if idx < 0:
156  raise LookupError("Task label is not found: " + label)
157  self._pipeline.insert(newIndex, self._pipeline.pop(idx))
158 
159  def labelTask(self, label, newLabel):
160  """Change task label.
161 
162  Parameters
163  ----------
164  label : `str`
165  Existing label of the task.
166  newLabel : `str`
167  New label of the task.
168  """
169  idx = self._pipeline.labelIndex(label)
170  if idx < 0:
171  raise LookupError("Task label is not found: " + label)
172  # check that new one is unique
173  if newLabel != label and self._pipeline.labelIndex(newLabel) >= 0:
174  raise LookupError("New task label is not unique: " + label)
175  self._pipeline[idx].label = newLabel
176 
177  def configOverride(self, label, value):
178  """Apply single config override.
179 
180  Parameters
181  ----------
182  label : `str`
183  Label of the task.
184  value : `str`
185  String in the form ``"param=value"`` or ``"parm.subpar=value"``,
186  ``value`` can be a Python constant or a list of constants.
187  """
188  idx = self._pipeline.labelIndex(label)
189  if idx < 0:
190  raise LookupError("Task label is not found: " + label)
191  key, sep, val = value.partition('=')
192  overrides = ConfigOverrides()
193  overrides.addValueOverride(key, val)
194  overrides.applyTo(self._pipeline[idx].config)
195 
196  def configOverrideFile(self, label, path):
197  """Apply overrides from file.
198 
199  Parameters
200  ----------
201  label : `str`
202  Label of the task.
203  path : `str`
204  Path to file with overrides.
205  """
206  idx = self._pipeline.labelIndex(label)
207  if idx < 0:
208  raise LookupError("Task label is not found: " + label)
209  overrides = ConfigOverrides()
210  overrides.addFileOverride(path)
211  overrides.applyTo(self._pipeline[idx].config)
212 
213  def substituteDatatypeNames(self, label, value):
214  """Apply name string formatting to config file
215  Parameters
216  ----------
217  label : `str`
218  Label of the task.
219  value : `str`
220  String of the form of a dictionary of template keyword to value
221  """
222  idx = self._pipeline.labelIndex(label)
223  if idx < 0:
224  raise LookupError("Task label is not found: " + label)
225 
226  try:
227  parsedNamesDict = ast.literal_eval(value)
228  if not isinstance(parsedNamesDict, dict):
229  raise ValueError()
230  except ValueError:
231  raise pexExceptions.RuntimeError(f"Unable parse --dataset-name-substitution {value} "
232  "into a valid dict")
233 
234  overrides = ConfigOverrides()
235  overrides.addDatasetNameSubstitution(parsedNamesDict)
236  overrides.applyTo(self._pipeline[idx].config)
def __init__(self, taskFactory, pipeline=None)