Coverage for python/lsst/ctrl/mpexec/execFixupDataId.py: 21%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

39 statements  

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ['ExecutionGraphFixup'] 

23 

24from collections import defaultdict 

25import networkx as nx 

26from typing import Sequence, Union, Tuple, Any 

27 

28from lsst.pipe.base import QuantumGraph, QuantumNode 

29from .executionGraphFixup import ExecutionGraphFixup 

30 

31 

32class ExecFixupDataId(ExecutionGraphFixup): 

33 """Implementation of ExecutionGraphFixup for ordering of tasks based 

34 on DataId values. 

35 

36 This class is a trivial implementation mostly useful as an example, 

37 though it can be used to make actual fixup instances by defining 

38 a method that instantiates it, e.g.:: 

39 

40 # lsst/ap/verify/ci_fixup.py 

41 

42 from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

43 

44 def assoc_fixup(): 

45 return ExecFixupDataId(taskLabel="ap_assoc", 

46 dimensions=("visit", "detector")) 

47 

48 

49 and then executing pipetask:: 

50 

51 pipetask run --graph-fixup=lsst.ap.verify.ci_fixup.assoc_fixup ... 

52 

53 This will add new dependencies between quanta executed by the task with 

54 label "ap_assoc". Quanta with higher visit number will depend on quanta 

55 with lower visit number and their execution will wait until lower visit 

56 number finishes. 

57 

58 Parameters 

59 ---------- 

60 taskLabel : `str` 

61 The label of the task for which to add dependencies. 

62 dimensions : `str` or sequence [`str`] 

63 One or more dimension names, quanta execution will be ordered 

64 according to values of these dimensions. 

65 reverse : `bool`, optional 

66 If `False` (default) then quanta with higher values of dimensions 

67 will be executed after quanta with lower values, otherwise the order 

68 is reversed. 

69 """ 

70 

71 def __init__(self, taskLabel: str, dimensions: Union[str, Sequence[str]], reverse: bool = False): 

72 self.taskLabel = taskLabel 

73 self.dimensions = dimensions 

74 self.reverse = reverse 

75 if isinstance(self.dimensions, str): 

76 self.dimensions = (self.dimensions, ) 

77 else: 

78 self.dimensions = tuple(self.dimensions) 

79 

80 def _key(self, qnode: QuantumNode) -> Tuple[Any, ...]: 

81 """Produce comparison key for quantum data. 

82 Parameters 

83 ---------- 

84 qnode : `QuantumNode` 

85 An individual node in a `~lsst.pipe.base.QuantumGraph` 

86 

87 Returns 

88 ------- 

89 key : `tuple` 

90 """ 

91 dataId = qnode.quantum.dataId 

92 key = tuple(dataId[dim] for dim in self.dimensions) 

93 return key 

94 

95 def fixupQuanta(self, graph: QuantumGraph) -> QuantumGraph: 

96 taskDef = graph.findTaskDefByLabel(self.taskLabel) 

97 if taskDef is None: 

98 raise ValueError(f"Cannot find task with label {self.taskLabel}") 

99 quanta = list(graph.getNodesForTask(taskDef)) 

100 keyQuanta = defaultdict(list) 

101 for q in quanta: 

102 key = self._key(q) 

103 keyQuanta[key].append(q) 

104 keys = sorted(keyQuanta.keys(), reverse=self.reverse) 

105 networkGraph = graph.graph 

106 

107 for prev_key, key in zip(keys, keys[1:]): 

108 for prev_node in keyQuanta[prev_key]: 

109 for node in keyQuanta[key]: 

110 # remove any existing edges between the two nodes, but 

111 # don't fail if there are not any. Both directions need 

112 # tried because in a directed graph, order maters 

113 for edge in ((node, prev_node), (prev_node, node)): 

114 try: 

115 networkGraph.remove_edge(*edge) 

116 except nx.NetworkXException: 

117 pass 

118 networkGraph.add_edge(prev_node, node) 

119 return graph