Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ['ExecutionGraphFixup'] 

23 

24from typing import Any, Iterable, Sequence, Tuple, Union 

25 

26from lsst.pipe.base import QuantumIterData 

27from .executionGraphFixup import ExecutionGraphFixup 

28 

29 

30class ExecFixupDataId(ExecutionGraphFixup): 

31 """Implementation of ExecutionGraphFixup for ordering of tasks based 

32 on DataId values. 

33 

34 This class is a trivial implementation mostly useful as an example, 

35 though it can be used to make actual fixup instances by defining 

36 a method that instantiates it, e.g.:: 

37 

38 # lsst/ap/verify/ci_fixup.py 

39 

40 from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

41 

42 def assoc_fixup(): 

43 return ExecFixupDataId(taskLabel="ap_assoc", 

44 dimensions=("visit", "detector")) 

45 

46 

47 and then executing pipetask:: 

48 

49 pipetask run --graph-fixup=lsst.ap.verify.ci_fixup.assoc_fixup ... 

50 

51 This will add new dependencies between quanta executed by the task with 

52 label "ap_assoc". Quanta with higher visit number will depend on quanta 

53 with lower visit number and their execution will wait until lower visit 

54 number finishes. 

55 

56 Parameters 

57 ---------- 

58 taskLabel : `str` 

59 The label of the task for which to add dependencies. 

60 dimensions : `str` or sequence [`str`] 

61 One or more dimension names, quanta execution will be ordered 

62 according to values of these dimensions. 

63 reverse : `bool`, optional 

64 If `False` (default) then quanta with higher values of dimensions 

65 will be executed after quanta with lower values, otherwise the order 

66 is reversed. 

67 """ 

68 

69 def __init__(self, taskLabel: str, dimensions: Union[str, Sequence[str]], reverse: bool = False): 

70 self.taskLabel = taskLabel 

71 self.dimensions = dimensions 

72 self.reverse = reverse 

73 if isinstance(self.dimensions, str): 

74 self.dimensions = (self.dimensions, ) 

75 else: 

76 self.dimensions = tuple(self.dimensions) 

77 

78 def _key(self, qdata: QuantumIterData) -> Tuple[Any, ...]: 

79 """Produce comparison key for quantum data. 

80 

81 Parameters 

82 ---------- 

83 qdata : `QuantumIterData` 

84 

85 Returns 

86 ------- 

87 key : `tuple` 

88 """ 

89 dataId = qdata.quantum.dataId 

90 key = tuple(dataId[dim] for dim in self.dimensions) 

91 return key 

92 

93 def fixupQuanta(self, quanta: Iterable[QuantumIterData]) -> Iterable[QuantumIterData]: 

94 # Docstring inherited from ExecutionGraphFixup.fixupQuanta 

95 quanta = list(quanta) 

96 # Index task quanta by the key 

97 keyQuanta = {} 

98 for qdata in quanta: 

99 if qdata.taskDef.label == self.taskLabel: 

100 key = self._key(qdata) 

101 keyQuanta.setdefault(key, []).append(qdata) 

102 if not keyQuanta: 

103 raise ValueError(f"Cannot find task with label {self.taskLabel}") 

104 # order keys 

105 keys = sorted(keyQuanta.keys(), reverse=self.reverse) 

106 # for each quanta in a key add dependency to all quanta in a preceding key 

107 for prev_key, key in zip(keys, keys[1:]): 

108 prev_indices = frozenset(qdata.index for qdata in keyQuanta[prev_key]) 

109 for qdata in keyQuanta[key]: 

110 qdata.dependencies |= prev_indices 

111 return quanta