Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

# This file is part of pipe_base. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

 

"""Module defining quantum graph classes and related methods. 

 

There could be different representations of the quantum graph depending 

on the client needs. Presently this module contains graph implementation 

which is based on requirements of command-line environment. In the future 

we could add other implementations and methods to convert between those 

representations. 

""" 

 

# "exported" names 

__all__ = ["QuantumGraphNodes", "QuantumGraph"] 

 

# ------------------------------- 

# Imports of standard modules -- 

# ------------------------------- 

 

# ----------------------------- 

# Imports for other modules -- 

# ----------------------------- 

 

# ---------------------------------- 

# Local non-exported definitions -- 

# ---------------------------------- 

 

# ------------------------ 

# Exported definitions -- 

# ------------------------ 

 

 

class QuantumGraphNodes: 

"""QuantumGraphNodes represents a bunch of nodes in an quantum graph. 

 

The node in quantum graph is represented by the `PipelineTask` and a 

single `Quantum` instance. One possible representation of the graph is 

just a list of nodes without edges (edges can be deduced from nodes' 

quantum inputs and outputs if needed). That representation can be reduced 

to the list of PipelineTasks and the corresponding list of Quanta. 

This class defines this reduced representation. 

 

Different frameworks may use different graph representation, this 

representation was based mostly on requirements of command-line 

executor which does not need explicit edges information. 

 

Attributes 

---------- 

taskDef : :py:class:`TaskDef` 

Task defintion for this set of nodes. 

quanta : `list` of :py:class:`lsst.daf.butler.Quantum` 

List of quanta corresponding to the task. 

""" 

def __init__(self, taskDef, quanta): 

self.taskDef = taskDef 

self.quanta = quanta 

 

 

class QuantumGraph(list): 

"""QuantumGraph is a sequence of QuantumGraphNodes objects. 

 

Typically the order of the tasks in the list will be the same as the 

order of tasks in a pipeline (obviously depends on the code which 

constructs graph). 

 

Parameters 

---------- 

iterable : iterable of :py:class:`QuantumGraphNodes` instances, optional 

Initial sequence of per-task nodes. 

""" 

def __init__(self, iterable=None): 

list.__init__(self, iterable or []) 

self.initInputs = [] 

self.initOutputs = [] 

self._inputDatasetTypes = set() 

self._outputDatasetTypes = set() 

 

def quanta(self): 

"""Iterator over quanta in a graph. 

 

Yields 

------ 

taskDef : `TaskDef` 

Task definition for a Quantum. 

quantum : `Quantum` 

Single quantum. 

""" 

for taskNodes in self: 

taskDef = taskNodes.taskDef 

for quantum in taskNodes.quanta: 

yield taskDef, quantum 

 

def getDatasetTypes(self, initInputs=True, initOutputs=True, inputs=True, outputs=True): 

total = set() 

if initInputs: 

for dsRef in self.initInputs: 

total.add(dsRef.datasetType) 

if initOutputs: 

for dsRef in self.initOutputs: 

total.add(dsRef.datasetType) 

if inputs: 

total |= self._inputDatasetTypes 

if outputs: 

total |= self._outputDatasetTypes 

return total