Coverage for python/lsst/ctrl/mpexec/execFixupDataId.py: 18%

40 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-30 02:51 -0700

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ["ExecutionGraphFixup"] 

23 

24from collections import defaultdict 

25from typing import Any, Sequence, Tuple, Union 

26 

27import networkx as nx 

28from lsst.pipe.base import QuantumGraph, QuantumNode 

29 

30from .executionGraphFixup import ExecutionGraphFixup 

31 

32 

33class ExecFixupDataId(ExecutionGraphFixup): 

34 """Implementation of ExecutionGraphFixup for ordering of tasks based 

35 on DataId values. 

36 

37 This class is a trivial implementation mostly useful as an example, 

38 though it can be used to make actual fixup instances by defining 

39 a method that instantiates it, e.g.:: 

40 

41 # lsst/ap/verify/ci_fixup.py 

42 

43 from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

44 

45 def assoc_fixup(): 

46 return ExecFixupDataId(taskLabel="ap_assoc", 

47 dimensions=("visit", "detector")) 

48 

49 

50 and then executing pipetask:: 

51 

52 pipetask run --graph-fixup=lsst.ap.verify.ci_fixup.assoc_fixup ... 

53 

54 This will add new dependencies between quanta executed by the task with 

55 label "ap_assoc". Quanta with higher visit number will depend on quanta 

56 with lower visit number and their execution will wait until lower visit 

57 number finishes. 

58 

59 Parameters 

60 ---------- 

61 taskLabel : `str` 

62 The label of the task for which to add dependencies. 

63 dimensions : `str` or sequence [`str`] 

64 One or more dimension names, quanta execution will be ordered 

65 according to values of these dimensions. 

66 reverse : `bool`, optional 

67 If `False` (default) then quanta with higher values of dimensions 

68 will be executed after quanta with lower values, otherwise the order 

69 is reversed. 

70 """ 

71 

72 def __init__(self, taskLabel: str, dimensions: Union[str, Sequence[str]], reverse: bool = False): 

73 self.taskLabel = taskLabel 

74 self.dimensions = dimensions 

75 self.reverse = reverse 

76 if isinstance(self.dimensions, str): 

77 self.dimensions = (self.dimensions,) 

78 else: 

79 self.dimensions = tuple(self.dimensions) 

80 

81 def _key(self, qnode: QuantumNode) -> Tuple[Any, ...]: 

82 """Produce comparison key for quantum data. 

83 Parameters 

84 ---------- 

85 qnode : `QuantumNode` 

86 An individual node in a `~lsst.pipe.base.QuantumGraph` 

87 

88 Returns 

89 ------- 

90 key : `tuple` 

91 """ 

92 dataId = qnode.quantum.dataId 

93 assert dataId is not None, "Quantum DataId cannot be None" 

94 key = tuple(dataId[dim] for dim in self.dimensions) 

95 return key 

96 

97 def fixupQuanta(self, graph: QuantumGraph) -> QuantumGraph: 

98 taskDef = graph.findTaskDefByLabel(self.taskLabel) 

99 if taskDef is None: 

100 raise ValueError(f"Cannot find task with label {self.taskLabel}") 

101 quanta = list(graph.getNodesForTask(taskDef)) 

102 keyQuanta = defaultdict(list) 

103 for q in quanta: 

104 key = self._key(q) 

105 keyQuanta[key].append(q) 

106 keys = sorted(keyQuanta.keys(), reverse=self.reverse) 

107 networkGraph = graph.graph 

108 

109 for prev_key, key in zip(keys, keys[1:]): 

110 for prev_node in keyQuanta[prev_key]: 

111 for node in keyQuanta[key]: 

112 # remove any existing edges between the two nodes, but 

113 # don't fail if there are not any. Both directions need 

114 # tried because in a directed graph, order maters 

115 for edge in ((node, prev_node), (prev_node, node)): 

116 try: 

117 networkGraph.remove_edge(*edge) 

118 except nx.NetworkXException: 

119 pass 

120 networkGraph.add_edge(prev_node, node) 

121 return graph