lsst.pipe.base  16.0-26-g8e79609
graph.py
Go to the documentation of this file.
1 # This file is part of pipe_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Module defining quantum graph classes and related methods.
23 
24 There could be different representations of the quantum graph depending
25 on the client needs. Presently this module contains graph implementation
26 which is based on requirements of command-line environment. In the future
27 we could add other implementations and methods to convert between those
28 representations.
29 """
30 
31 # "exported" names
32 __all__ = ["QuantumGraphNodes", "QuantumGraph"]
33 
34 # -------------------------------
35 # Imports of standard modules --
36 # -------------------------------
37 
38 # -----------------------------
39 # Imports for other modules --
40 # -----------------------------
41 
42 # ----------------------------------
43 # Local non-exported definitions --
44 # ----------------------------------
45 
46 # ------------------------
47 # Exported definitions --
48 # ------------------------
49 
50 
52  """QuantumGraphNodes represents a bunch of nodes in an quantum graph.
53 
54  The node in quantum graph is represented by the `PipelineTask` and a
55  single `Quantum` instance. One possible representation of the graph is
56  just a list of nodes without edges (edges can be deduced from nodes'
57  quantum inputs and outputs if needed). That representation can be reduced
58  to the list of PipelineTasks and the corresponding list of Quanta.
59  This class defines this reduced representation.
60 
61  Different frameworks may use different graph representation, this
62  representation was based mostly on requirements of command-line
63  executor which does not need explicit edges information.
64 
65  Attributes
66  ----------
67  taskDef : :py:class:`TaskDef`
68  Task defintion for this set of nodes.
69  quanta : `list` of :py:class:`lsst.daf.butler.Quantum`
70  List of quanta corresponding to the task.
71  """
72  def __init__(self, taskDef, quanta):
73  self.taskDef = taskDef
74  self.quanta = quanta
75 
76 
77 class QuantumGraph(list):
78  """QuantumGraph is a sequence of QuantumGraphNodes objects.
79 
80  Typically the order of the tasks in the list will be the same as the
81  order of tasks in a pipeline (obviously depends on the code which
82  constructs graph).
83 
84  Parameters
85  ----------
86  iterable : iterable of :py:class:`QuantumGraphNodes` instances, optional
87  Initial sequence of per-task nodes.
88  """
89  def __init__(self, iterable=None):
90  list.__init__(self, iterable or [])
91  self.initInputs = []
92  self.initOutputs = []
93  self._inputDatasetTypes = set()
94  self._outputDatasetTypes = set()
95 
96  def quanta(self):
97  """Iterator over quanta in a graph.
98 
99  Yields
100  ------
101  taskDef : `TaskDef`
102  Task definition for a Quantum.
103  quantum : `Quantum`
104  Single quantum.
105  """
106  for taskNodes in self:
107  taskDef = taskNodes.taskDef
108  for quantum in taskNodes.quanta:
109  yield taskDef, quantum
110 
111  def getDatasetTypes(self, initInputs=True, initOutputs=True, inputs=True, outputs=True):
112  total = set()
113  if initInputs:
114  for dsRef in self.initInputs:
115  total.add(dsRef.datasetType)
116  if initOutputs:
117  for dsRef in self.initOutputs:
118  total.add(dsRef.datasetType)
119  if inputs:
120  total |= self._inputDatasetTypes
121  if outputs:
122  total |= self._outputDatasetTypes
123  return total
def __init__(self, taskDef, quanta)
Definition: graph.py:72
def getDatasetTypes(self, initInputs=True, initOutputs=True, inputs=True, outputs=True)
Definition: graph.py:111
def __init__(self, iterable=None)
Definition: graph.py:89