Coverage for python/lsst/obs/base/gen2to3/repoWalker/parser.py : 31%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21"""Classes that transform (part of) a Gen2 filename template into a regular
22expression that we can use to extract Gen2 data IDs from files.
23"""
24from __future__ import annotations
26__all__ = ["PathElementParser"]
29from abc import ABC, abstractmethod
30import re
31from typing import ClassVar, Dict, Optional
33from lsst.log import Log
36class FormattableRegEx(ABC):
37 """An interface that generates a regular expression from a template and
38 a data ID.
40 This is used by `PathElementParser` to abstract over whether a path
41 element's regex needs to include values from a data ID extracted from
42 parent path elements or not.
43 """
45 @abstractmethod
46 def format(self, dataId: dict) -> re.Pattern:
47 """Substitute values from the given data ID and return a regular
48 expression.
50 Parameters
51 ----------
52 dataId : `dict`
53 A dictionary whose entries may be used to format the regular
54 expression. May include unused entries.
55 """
56 raise NotImplementedError()
59class FixedRegEx(FormattableRegEx):
60 """A trivial implementation of `FormattableRegEx` that does no formatting.
62 Parameters
63 ----------
64 regex : `re.Pattern`
65 The fixed regular expression to return.
66 """
67 def __init__(self, regex: re.Pattern):
68 self.regex = regex
70 __slots__ = ("regex",)
72 def format(self, dataId: dict) -> re.Pattern:
73 # Docstring inherited from FormattableRegEx.
74 return self.regex
76 def __str__(self):
77 return f"{type(self).__name__}({self.regex})"
80class SubstitutableRegEx:
81 """An implementation of `FormattableRegEx` formed from a concatenation of
82 actual regular terms and %-style format strings.
83 """
84 def __init__(self):
85 self._terms = []
87 __slots__ = ("_terms",)
89 def addRegexTerm(self, regex: str):
90 """Add a regular expression term.
91 """
92 self._terms.append((regex, False))
94 def addSubstitutionTerm(self, template: str):
95 """Add a %-style format template term.
96 """
97 self._terms.append((template, True))
99 def format(self, dataId: dict) -> re.Pattern:
100 # Docstring inherited from FormattableRegEx.
101 return re.compile("".join(re.escape(s % dataId) if isSub else s
102 for s, isSub in self._terms))
104 def simplify(self) -> FormattableRegEx:
105 """Return a possibly-simplified version of this object.
107 If `addSubstitionTerm` was never called, this returns a simple
108 `FixedRegEx`.
109 """
110 if not any(isSub for _, isSub in self._terms):
111 return FixedRegEx(re.compile("".join(s for s, _ in self._terms)))
112 else:
113 return self
116class PathElementParser:
117 """An object that matches Gen2 file names and extracts Gen2 data IDs.
119 Parameters
120 ----------
121 target : `str`
122 Either a full Gen2 path template or the part of one the corresponds to
123 a single path element (a subdirectory or file name).
124 allKeys : `dict` [`str`, `type`]
125 A dictionary that provides types for all Gen2 data ID keys that are
126 substituted into the given template. Additional key-value pairs may
127 be present and will be ignored.
128 previousKeys : `dict` [`str`, `type`], optional
129 A dictionary containing key strings and types for Gen2 data ID keys
130 that have been extracted from previous path elements of the same
131 template. Values for these keys must be provided via the
132 ``lastDataId`` argument when calling `parse`.
133 """
134 def __init__(self, template: str, allKeys: Dict[str, type], *,
135 previousKeys: Optional[Dict[str, type]] = None):
136 self.template = template
137 self.keys = {}
138 # For each template path element, we iterate over each %-tagged
139 # substitution string.
140 last = 0
141 self.regex = SubstitutableRegEx()
142 for match in self.TEMPLATE_RE.finditer(self.template):
143 # Copy the (escaped) regular string between the last substitution
144 # and this one, escaping it appropriately.
145 self.regex.addRegexTerm(re.escape(self.template[last:match.start()]))
146 # Pull out the data ID key from the name used in the
147 # substitution string. Use that and the substition
148 # type to come up with the pattern to use in the regex.
149 name = match.group("name")
150 if name == "patch":
151 pattern = r"\d+,\d+"
152 elif match.group("type") in "id": # integers
153 pattern = r"0*\d+"
154 else:
155 pattern = ".+"
156 # Create a new named groups for the first occurence of a key
157 # within an element.
158 if name not in self.keys:
159 if previousKeys and name in previousKeys:
160 # Key is new to this part of the template, but it appeared
161 # in some previous part of the template. We'll format the
162 # original template with the data ID from that previous
163 # step later.
164 start, stop = match.span()
165 self.regex.addSubstitutionTerm(self.template[start:stop])
166 else:
167 # Key is new; expect to extract a data ID value from it.
168 self.regex.addRegexTerm(r"(?P<%s>%s)" % (name, pattern))
169 self.keys[name] = allKeys[name]
170 else:
171 # Require a match with the last group for a second
172 # occurrence.
173 self.regex.addRegexTerm(r"(?P=<%s>)" % name)
174 # Remember the end of this match
175 last = match.end()
176 # Append anything remaining after the last substitution string.
177 self.regex.addRegexTerm(re.escape(self.template[last:]))
178 # If there are no substitutions, join and compile into a single regex
179 # now.
180 self.regex = self.regex.simplify()
182 __slots__ = ("keys", "template", "regex")
184 TEMPLATE_RE: ClassVar[re.Pattern] = re.compile(r"\%\((?P<name>\w+)\)[^\%]*?(?P<type>[idrs])")
185 """Regular expression that matches a single substitution in
186 Gen2 CameraMapper template, such as "%(tract)04d".
187 """
189 def __str__(self):
190 return f"{type(self).__name__}({self.regex})"
192 def parse(self, name: str, lastDataId: dict, *, log: Optional[Log] = None) -> Optional[dict]:
193 """Parse the path element.
195 Parameters
196 ----------
197 name : `str`
198 The path name to parse.
199 lastDataId : `dict`
200 The cumulative Gen2 data ID obtaining by calling `parse` on parsers
201 for parent directories of the same path.
202 log : `Log`, optional
203 Log to use to report warnings and debug information.
205 Returns
206 -------
207 dataId : `dict` or `None`
208 Gen2 data ID that combines key-value pairs obtained from this path
209 with those from ``lastDataId``. `None` if ``name`` is not matched
210 by this parser. If the keys extracted are inconsistent with those
211 in ``lastDataID``, a warning is sent to ``log`` and `None` is
212 returned.
213 """
214 m = self.regex.format(lastDataId).fullmatch(name)
215 if m is None:
216 return None
217 newDataId = {k: v(m.group(k)) for k, v in self.keys.items()}
218 for commonKey in lastDataId.keys() & newDataId.keys():
219 if newDataId[commonKey] != lastDataId[commonKey]:
220 if log is not None:
221 log.warning("Inconsistent value %s=%r when parsing %r with %r.",
222 commonKey, newDataId[commonKey], name, lastDataId)
223 return None
224 newDataId.update(lastDataId)
225 return newDataId
227 keys: Dict[str, type]
228 """Dictionary mapping Gen2 data ID key to the type of its associated
229 value, covering only those keys that can be extracted from this path
230 element.
231 """
233 template: str
234 """The portion of the original Gen2 filename template that this parser was
235 constructed with.
236 """
238 regex: re.Pattern
239 """A regular expression that can be used to match the path element and
240 populate the Gen2 data ID items whose keys are in ``keys``.
241 """