21 """Classes used in `RepoWalker` construction.
23 The objects here form a temporary tree that is pruned and then transformed
24 into a similar tree of `PathElementHandler` instances. See `BuilderNode`
25 method documentation for more information.
27 from __future__
import annotations
29 __all__ = [
"BuilderSkipInput",
"BuilderTargetInput",
"BuilderTree"]
31 from abc
import ABC, abstractmethod
42 from lsst.daf.butler
import DatasetType, DimensionUniverse, StorageClass, FormatterParameter
43 from ..translators
import Translator
44 from .parser
import PathElementParser
45 from .scanner
import PathElementHandler, DirectoryScanner
46 from .handlers
import (IgnoreHandler, SubdirectoryHandler, SkipHandler,
51 """Abstract interface for nodes in the temporary tree that is used to
52 construct a `RepoWalker`.
56 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
57 """Attempt to prune this node and its children from the tree.
61 replacement : `BuilderNode`
62 The result of recursively pruning child nodes; often just ``self``.
63 messages : `list` [`str`]
64 Warning messages that should be logged by a parent node when a
65 matching path element is encountered, if this node is pruned.
67 If `True`, this node may be pruned from the tree (but will not
68 necessarily be - it may correspond to a path element that should
69 be skipped with siblings that should not be).
71 raise NotImplementedError()
74 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
75 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
76 ) -> PathElementHandler:
77 """Transform this node in the build tree into a corresponding
78 `PathElementHandler`, recursing to any children.
80 Must be called after `prune`.
84 parser : `PathElementParser`
85 An object that matches the path element the new handler is
86 responsible for and extracts a (partial) Gen2 data ID from it.
87 allKeys : `dict` [`str`, `type`]
88 A mapping from Gen2 data ID key to the type of its value. Will
89 contain all keys that may be extracted by the given parser, and
91 cumulativeKeys : `dict` [`str`, `type`], optional
92 A dictionary containing key strings and types for Gen2 data ID keys
93 that have been extracted from previous path elements for this
94 template, including those extracted by ``parser``.
98 handler : `PathElementHandler`
101 raise NotImplementedError()
105 """An intermediate base for `BuilderNode` classes that are provided as
106 direct inputs to a `RepoWalker`, and generally correspond to exactly one
112 The complete Gen2 template to be matched (not just the template for
114 keys : `dict` [`str`, `type`]
115 A mapping from Gen2 data ID key to the type of its value.
117 def __init__(self, template: str, keys: Dict[str, type]):
123 """The complete Gen2 template to be matched (`str`).
126 keys: Dict[str, type]
127 """A mapping from Gen2 data ID key to the type of its value
128 (`dict` [`str`, `type`]).
132 """The path elements (file or directory levels) of `template`
138 """An input to a `RepoWalker` that indicates that matched files should be
139 skipped, possibly with a warning message.
141 BuilderSkipInputs can be pruned. When they are not pruned, they build
142 `SkipHandler` instances.
147 The complete Gen2 template to be matched (not just the template for
149 keys : `dict` [`str`, `type`]
150 A mapping from Gen2 data ID key to the type of its value.
151 message : `str`, optional
152 If not `None`, a warning message that should be printed either when a
153 matching file is enountered or a directory that may contain such files
155 isForFiles : `bool`, optional
156 If `True` (default), this handler should be run on files. Otherwise it
157 should be run on directories.
159 def __init__(self, template: str, keys: Dict[str, type], message: Optional[str] =
None, *,
160 isForFiles: bool =
True):
161 super().
__init__(template=template, keys=keys)
165 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
166 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
167 ) -> PathElementHandler:
171 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
177 """An input to a `RepoWalker` that matches files that correspond to
178 datasets that we want to extract.
180 BuilderTargetInputs can never be pruned, and always build
181 `TargetFileHandler` instances.
185 datasetTypeName : `str`
186 Name of the dataset type.
188 Full Gen2 filename template.
189 keys : `dict` [`str`, `type`]
190 Dictionary that maps Gen2 data ID key to the type of its value.
191 storageClass : `StorageClass`
192 `StorageClass` for the Gen3 dataset type.
193 universe : `DimensionUniverse`
194 All candidate dimensions for the Gen3 dataset type.
195 formatter : `lsst.daf.butler.Formatter` or `str`, optional
196 A Gen 3 formatter class or fully-qualified name.
197 targetHandler : `PathElementHandler`, optional
198 Override target handler for this dataset type.
200 Additional keyword arguments are passed to `Translator.makeMatching`,
201 in along with ``datasetTypeName`` and ``keys``.
203 def __init__(self, *, datasetTypeName: str, template: str, keys: Dict[str, type],
204 storageClass: StorageClass, universe: DimensionUniverse,
205 formatter: FormatterParameter,
206 targetHandler: Optional[PathElementHandler] =
None, **kwargs: Any):
208 template = template.split(
'[%(')[0]
209 super().
__init__(template=template, keys=keys)
210 self.
_translator = Translator.makeMatching(datasetTypeName, keys, **kwargs)
212 storageClass=storageClass, universe=universe)
214 if targetHandler
is None:
215 targetHandler = TargetFileHandler
218 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
219 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
220 ) -> PathElementHandler:
225 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
227 return self, [],
False
229 datasetType: DatasetType
230 """The Gen3 dataset type extracted by the handler this object builds
231 (`lsst.daf.butler.DatasetType`).
236 """A `BuilderNode` that represents a subdirectory to be skipped,
237 created by pruning `BuilderTree` that contained only `BuilderSkipInput`
240 BuilderPrunedTrees can be pruned. When they are not pruned, they
241 build `SkipHandler` instances.
245 messages : `list` [`str`]
246 A list of warning messages to be printed when the handler produced by
247 this builder matches a subdirectory.
253 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
254 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
255 ) -> PathElementHandler:
258 return SkipHandler(parser=parser, isForFiles=
False, message=message)
260 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
266 """A `BuilderNode` that represents a collection of `BuilderInput` instances
267 that all have the same template.
269 def __init__(self, old: BuilderInput, new: BuilderInput):
271 if isinstance(old, BuilderDuplicateInputs):
278 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
279 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
280 ) -> PathElementHandler:
283 return SkipHandler(parser=parser, isForFiles=
False, message=message)
285 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
290 newChild, childMessages, toPruneChild = child.prune()
294 unprunable.append(newChild)
295 newChildren.append(newChildren)
297 if len(unprunable) == 0:
301 elif len(unprunable) == 1
and not self.
_messages:
306 return unprunable[0], [],
False
312 nested = [f
"{c.datasetType.name} (target)" for c
in unprunable]
314 self.
_messages = [f
"ambiguous match: [{', '.join(nested)}]"]
319 """A `BuilderNode` that represents a directory.
321 This is the only `BuilderNode` class that is not a leaf node. If all
322 of its children can be pruned, it is replaced by a `BuilderPrunedTree`
323 (which can then be pruned itself). It builds `SubdirectoryHandler`
324 instances when not pruned.
329 def insert(self, level: int, leaf: BuilderInput):
330 """Insert an input leaf node into the tree, recursively constructing
331 intermediate parents in order to put it at the right level.
336 The level ``self``is at in the larger tree, with zero the
337 repository root. The right level for the leaf is given by the
338 length of ``leaf.elements``.
339 leaf : `BuilderInput`
340 The leaf node to insert.
342 nextLevel = level + 1
343 element = leaf.elements[level]
344 if nextLevel == len(leaf.elements):
346 if conflict
is not None:
353 child.insert(nextLevel, leaf)
355 def fill(self, scanner: DirectoryScanner, allKeys: Dict[str, type], previousKeys: Dict[str, type], *,
356 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]):
357 """Fill a `DirectoryScanner` instance by recursively building all
362 scanner : `DirectoryScanner`
364 allKeys : `dict` [`str`, `type`]
365 Mapping from Gen2 data ID key to its value type, covering all keys
366 that could be used in any child template.
367 previousKeys : `dict` [`str`, `type`], optional
368 A dictionary containing key strings and types for Gen2 data ID keys
369 that have been extracted from previous path elements of the same
372 if fileIgnoreRegEx
is not None:
374 if dirIgnoreRegEx
is not None:
376 for template, child
in self.
_children.items():
378 cumulativeKeys = previousKeys.copy()
379 cumulativeKeys.update(parser.keys)
380 scanner.add(child.build(parser, allKeys, cumulativeKeys, fileIgnoreRegEx=fileIgnoreRegEx,
381 dirIgnoreRegEx=dirIgnoreRegEx))
383 def prune(self) -> Tuple[BuilderNode, List[str], bool]:
389 for template, child
in list(self.
_children.items()):
390 newChild, childMessages, toPruneChild = child.prune()
391 newChildren[template] = newChild
392 messages.extend(childMessages)
399 return self, [],
False
401 def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
402 fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
403 ) -> PathElementHandler:
406 self.
fill(built.scanner, allKeys, cumulativeKeys, fileIgnoreRegEx=fileIgnoreRegEx,
407 dirIgnoreRegEx=dirIgnoreRegEx)