21 """High-level interface to the Gen2 repository-walking functionality defined
24 from __future__
import annotations
26 __all__ = [
"RepoWalker"]
28 from collections
import defaultdict
41 from lsst.log
import Log
42 from lsst.daf.butler
import (
47 from .builders
import BuilderTargetInput, BuilderSkipInput, BuilderTree
48 from .scanner
import DirectoryScanner
52 """An object that recursively walks a Gen2 data repository tree, extracting
53 Gen3 `FileDataset` objects and warning about unrecognized or unconvertable
58 inputs : `~collections.abc.Iterable` of `Target` or `Skip`
59 Structs that indicate dataset types to be extracted (`Target`) or
60 explicitly skipped (`Skip`). Skips may include a warning message to
61 log when matching entries are encountered.
62 fileIgnoreRegEx : `re.Pattern`, optional
63 A regular expression pattern that identifies non-dataset files that
64 can be ignored, to be applied at all levels of the directory tree.
65 dirIgnoreRegEx : `re.Pattern`, optional
66 A regular expression pattern that identifies non-dataset subdirectories
67 that can be ignored, to be applied at all levels of the directory tree.
69 def __init__(self, inputs: Iterable[Union[Target, Skip]], *,
70 fileIgnoreRegEx: Optional[re.Pattern] =
None, dirIgnoreRegEx: Optional[re.Pattern] =
None):
73 allKeys: Dict[str, type] = {}
76 for key, dtype
in leaf.keys.items():
77 if allKeys.setdefault(key, dtype) != dtype:
78 raise ValueError(f
"Multiple types for key '{key}': {dtype} "
79 f
"(from {leaf.template}) vs. {allKeys[key]}.")
80 tree, messages, pruned = tree.prune()
83 tree.fill(self.
_scanner, allKeys, {}, fileIgnoreRegEx=fileIgnoreRegEx,
84 dirIgnoreRegEx=dirIgnoreRegEx)
90 Target: ClassVar[type] = BuilderTargetInput
91 """An input struct type whose instances represent a dataset type to be
95 Skip: ClassVar[type] = BuilderSkipInput
96 """An input struct type whose instances represent a dataset type to be
100 def walk(self, root: str, *, log: Log, predicate: Optional[Callable[[DataCoordinate], bool]]
101 ) -> Mapping[DatasetType, List[FileDataset]]:
102 """Walk a Gen2 repository root to extract Gen3 `FileDataset` instances
108 Absolute path to the repository root.
110 Logger for warnings and diagnostic information.
111 predicate : `~collections.abc.Callable`, optional
112 If not `None`, a callable that returns `True` if a `DataCoordinate`
113 is consistent with what we want to extract. If ``predicate``
114 returns `False`, the file or directory that data ID was extracted
115 from will not be processed, even if it includes target dataset
120 datasets : `defaultdict` [`DatasetType`, `list`[`FileDataset`]]
121 Extracted datasets, grouped by Gen3 `DatasetType`.
123 if predicate
is None:
124 def predicate(dataId: DataCoordinate) -> bool:
126 datasets = defaultdict(list)
128 self.
_scanner.scan(root, datasets, log=log, predicate=predicate)