Coverage for python/lsst/obs/base/gen2to3/repoWalker/parser.py : 29%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
# This file is part of obs_base. # # Developed for the LSST Data Management System. # This product includes software developed by the LSST Project # (http://www.lsst.org). # See the COPYRIGHT file at the top-level directory of this distribution # for details of code ownership. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. expression that we can use to extract Gen2 data IDs from files. """
"""An interface that generates a regular expression from a template and a data ID.
This is used by `PathElementParser` to abstract over whether a path element's regex needs to include values from a data ID extracted from parent path elements or not. """
def format(self, dataId: dict) -> re.Pattern: """Substitute values from the given data ID and return a regular expression.
Parameters ---------- dataId : `dict` A dictionary whose entries may be used to format the regular expression. May include unused entries. """ raise NotImplementedError()
"""A trivial implementation of `FormattableRegEx` that does no formatting.
Parameters ---------- regex : `re.Pattern` The fixed regular expression to return. """ self.regex = regex
# Docstring inherited from FormattableRegEx. return self.regex
"""An implementation of `FormattableRegEx` formed from a concatenation of actual regular terms and %-style format strings. """ self._terms = []
"""Add a regular expression term. """ self._terms.append((regex, False))
"""Add a %-style format template term. """ self._terms.append((template, True))
# Docstring inherited from FormattableRegEx. return re.compile("".join(re.escape(s % dataId) if isSub else s for s, isSub in self._terms))
"""Return a possibly-simplified version of this object.
If `addSubstitionTerm` was never called, this returns a simple `FixedRegEx`. """ if not any(isSub for _, isSub in self._terms): return FixedRegEx(re.compile("".join(s for s, _ in self._terms))) else: return self
"""An object that matches Gen2 file names and extracts Gen2 data IDs.
Parameters ---------- target : `str` Either a full Gen2 path template or the part of one the corresponds to a single path element (a subdirectory or file name). allKeys : `dict` [`str`, `type`] A dictionary that provides types for all Gen2 data ID keys that are substituted into the given template. Additional key-value pairs may be present and will be ignored. previousKeys : `dict` [`str`, `type`], optional A dictionary containing key strings and types for Gen2 data ID keys that have been extracted from previous path elements of the same template. Values for these keys must be provided via the ``lastDataId`` argument when calling `parse`. """ previousKeys: Optional[Dict[str, type]] = None): self.template = template self.keys = {} # For each template path element, we iterate over each %-tagged # substitution string. last = 0 self.regex = SubstitutableRegEx() for match in self.TEMPLATE_RE.finditer(self.template): # Copy the (escaped) regular string between the last substitution # and this one, escaping it appropriately. self.regex.addRegexTerm(re.escape(self.template[last:match.start()])) # Pull out the data ID key from the name used in the # substitution string. Use that and the substition # type to come up with the pattern to use in the regex. name = match.group("name") if name == "patch": pattern = r"\d+,\d+" elif match.group("type") in "id": # integers pattern = r"0*\d+" else: pattern = ".+" # Create a new named groups for the first occurence of a key # within an element. if name not in self.keys: if previousKeys and name in previousKeys: # Key is new to this part of the template, but it appeared # in some previous part of the template. We'll format the # original template with the data ID from that previous # step later. start, stop = match.span() self.regex.addSubstitutionTerm(self.template[start:stop]) else: # Key is new; expect to extract a data ID value from it. self.regex.addRegexTerm(r"(?P<%s>%s)" % (name, pattern)) self.keys[name] = allKeys[name] else: # Require a match with the last group for a second # occurrence. self.regex.addRegexTerm(r"(?P=<%s>)" % name) # Remember the end of this match last = match.end() # Append anything remaining after the last substitution string. self.regex.addRegexTerm(re.escape(self.template[last:])) # If there are no substitutions, join and compile into a single regex # now. self.regex = self.regex.simplify()
"""Regular expression that matches a single substitution in Gen2 CameraMapper template, such as "%(tract)04d". """
"""Parse the path element.
Parameters ---------- name : `str` The path name to parse. lastDataId : `dict` The cumulative Gen2 data ID obtaining by calling `parse` on parsers for parent directories of the same path. log : `Log`, optional Log to use to report warnings and debug information.
Returns ------- dataId : `dict` or `None` Gen2 data ID that combines key-value pairs obtained from this path with those from ``lastDataId``. `None` if ``name`` is not matched by this parser. If the keys extracted are inconsistent with those in ``lastDataID``, a warning is sent to ``log`` and `None` is returned. """ m = self.regex.format(lastDataId).fullmatch(name) if m is None: return None newDataId = {k: v(m.group(k)) for k, v in self.keys.items()} for commonKey in lastDataId.keys() & newDataId.keys(): if newDataId[commonKey] != lastDataId[commonKey]: if log is not None: log.warn("Inconsistent value %s=%r when parsing %r with %r.", commonKey, newDataId[commonKey], name, lastDataId) return None newDataId.update(lastDataId) return newDataId
"""Dictionary mapping Gen2 data ID key to the type of its associated value, covering only those keys that can be extracted from this path element. """
"""The portion of the original Gen2 filename template that this parser was constructed with. """
populate the Gen2 data ID items whose keys are in ``keys``. """ |