Coverage for python/lsst/ctrl/mpexec/execFixupDataId.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ['ExecutionGraphFixup']
24from typing import Any, Iterable, Sequence, Tuple, Union
26from lsst.pipe.base import QuantumIterData
27from .executionGraphFixup import ExecutionGraphFixup
30class ExecFixupDataId(ExecutionGraphFixup):
31 """Implementation of ExecutionGraphFixup for ordering of tasks based
32 on DataId values.
34 This class is a trivial implementation mostly useful as an example,
35 though it can be used to make actual fixup instances by defining
36 a method that instantiates it, e.g.::
38 # lsst/ap/verify/ci_fixup.py
40 from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
42 def assoc_fixup():
43 return ExecFixupDataId(taskLabel="ap_assoc",
44 dimensions=("visit", "detector"))
47 and then executing pipetask::
49 pipetask run --graph-fixup=lsst.ap.verify.ci_fixup.assoc_fixup ...
51 This will add new dependencies between quanta executed by the task with
52 label "ap_assoc". Quanta with higher visit number will depend on quanta
53 with lower visit number and their execution will wait until lower visit
54 number finishes.
56 Parameters
57 ----------
58 taskLabel : `str`
59 The label of the task for which to add dependencies.
60 dimensions : `str` or sequence [`str`]
61 One or more dimension names, quanta execution will be ordered
62 according to values of these dimensions.
63 reverse : `bool`, optional
64 If `False` (default) then quanta with higher values of dimensions
65 will be executed after quanta with lower values, otherwise the order
66 is reversed.
67 """
69 def __init__(self, taskLabel: str, dimensions: Union[str, Sequence[str]], reverse: bool = False):
70 self.taskLabel = taskLabel
71 self.dimensions = dimensions
72 self.reverse = reverse
73 if isinstance(self.dimensions, str):
74 self.dimensions = (self.dimensions, )
75 else:
76 self.dimensions = tuple(self.dimensions)
78 def _key(self, qdata: QuantumIterData) -> Tuple[Any, ...]:
79 """Produce comparison key for quantum data.
81 Parameters
82 ----------
83 qdata : `QuantumIterData`
85 Returns
86 -------
87 key : `tuple`
88 """
89 dataId = qdata.quantum.dataId
90 key = tuple(dataId[dim] for dim in self.dimensions)
91 return key
93 def fixupQuanta(self, quanta: Iterable[QuantumIterData]) -> Iterable[QuantumIterData]:
94 # Docstring inherited from ExecutionGraphFixup.fixupQuanta
95 quanta = list(quanta)
96 # Index task quanta by the key
97 keyQuanta = {}
98 for qdata in quanta:
99 if qdata.taskDef.label == self.taskLabel:
100 key = self._key(qdata)
101 keyQuanta.setdefault(key, []).append(qdata)
102 if not keyQuanta:
103 raise ValueError(f"Cannot find task with label {self.taskLabel}")
104 # order keys
105 keys = sorted(keyQuanta.keys(), reverse=self.reverse)
106 # for each quanta in a key add dependency to all quanta in a preceding key
107 for prev_key, key in zip(keys, keys[1:]):
108 prev_indices = frozenset(qdata.index for qdata in keyQuanta[prev_key])
109 for qdata in keyQuanta[key]:
110 qdata.dependencies |= prev_indices
111 return quanta