Coverage for python/lsst/ctrl/mpexec/execFixupDataId.py: 21%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

39 statements  

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ["ExecutionGraphFixup"] 

23 

24from collections import defaultdict 

25from typing import Any, Sequence, Tuple, Union 

26 

27import networkx as nx 

28from lsst.pipe.base import QuantumGraph, QuantumNode 

29 

30from .executionGraphFixup import ExecutionGraphFixup 

31 

32 

33class ExecFixupDataId(ExecutionGraphFixup): 

34 """Implementation of ExecutionGraphFixup for ordering of tasks based 

35 on DataId values. 

36 

37 This class is a trivial implementation mostly useful as an example, 

38 though it can be used to make actual fixup instances by defining 

39 a method that instantiates it, e.g.:: 

40 

41 # lsst/ap/verify/ci_fixup.py 

42 

43 from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

44 

45 def assoc_fixup(): 

46 return ExecFixupDataId(taskLabel="ap_assoc", 

47 dimensions=("visit", "detector")) 

48 

49 

50 and then executing pipetask:: 

51 

52 pipetask run --graph-fixup=lsst.ap.verify.ci_fixup.assoc_fixup ... 

53 

54 This will add new dependencies between quanta executed by the task with 

55 label "ap_assoc". Quanta with higher visit number will depend on quanta 

56 with lower visit number and their execution will wait until lower visit 

57 number finishes. 

58 

59 Parameters 

60 ---------- 

61 taskLabel : `str` 

62 The label of the task for which to add dependencies. 

63 dimensions : `str` or sequence [`str`] 

64 One or more dimension names, quanta execution will be ordered 

65 according to values of these dimensions. 

66 reverse : `bool`, optional 

67 If `False` (default) then quanta with higher values of dimensions 

68 will be executed after quanta with lower values, otherwise the order 

69 is reversed. 

70 """ 

71 

72 def __init__(self, taskLabel: str, dimensions: Union[str, Sequence[str]], reverse: bool = False): 

73 self.taskLabel = taskLabel 

74 self.dimensions = dimensions 

75 self.reverse = reverse 

76 if isinstance(self.dimensions, str): 

77 self.dimensions = (self.dimensions,) 

78 else: 

79 self.dimensions = tuple(self.dimensions) 

80 

81 def _key(self, qnode: QuantumNode) -> Tuple[Any, ...]: 

82 """Produce comparison key for quantum data. 

83 Parameters 

84 ---------- 

85 qnode : `QuantumNode` 

86 An individual node in a `~lsst.pipe.base.QuantumGraph` 

87 

88 Returns 

89 ------- 

90 key : `tuple` 

91 """ 

92 dataId = qnode.quantum.dataId 

93 key = tuple(dataId[dim] for dim in self.dimensions) 

94 return key 

95 

96 def fixupQuanta(self, graph: QuantumGraph) -> QuantumGraph: 

97 taskDef = graph.findTaskDefByLabel(self.taskLabel) 

98 if taskDef is None: 

99 raise ValueError(f"Cannot find task with label {self.taskLabel}") 

100 quanta = list(graph.getNodesForTask(taskDef)) 

101 keyQuanta = defaultdict(list) 

102 for q in quanta: 

103 key = self._key(q) 

104 keyQuanta[key].append(q) 

105 keys = sorted(keyQuanta.keys(), reverse=self.reverse) 

106 networkGraph = graph.graph 

107 

108 for prev_key, key in zip(keys, keys[1:]): 

109 for prev_node in keyQuanta[prev_key]: 

110 for node in keyQuanta[key]: 

111 # remove any existing edges between the two nodes, but 

112 # don't fail if there are not any. Both directions need 

113 # tried because in a directed graph, order maters 

114 for edge in ((node, prev_node), (prev_node, node)): 

115 try: 

116 networkGraph.remove_edge(*edge) 

117 except nx.NetworkXException: 

118 pass 

119 networkGraph.add_edge(prev_node, node) 

120 return graph