Coverage for python/lsst/obs/base/gen2to3/translators.py : 51%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
# This file is part of obs_base. # # Developed for the LSST Data Management System. # This product includes software developed by the LSST Project # (https://www.lsst.org). # See the COPYRIGHT file at the top-level directory of this distribution # for details of code ownership. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>.
"makeCalibrationLabel")
filter: Optional[str] = None) -> str: """Make a Gen3 calibration_label string corresponding to a Gen2 data ID.
Parameters ---------- datasetTypeName : `str` Name of the dataset type this calibration label identifies. calibDate : `str` Date string used in the Gen2 template. ccd : `int`, optional Detector ID used in the Gen2 template. filter : `str`, optional Filter used in the Gen2 template.
Returns ------- label : `str` Calibration label string. """ # TODO: this function is probably HSC-specific, but I don't know how other # obs calib registries behave so I don't know (yet) how to generalize it. elements = [datasetTypeName, calibDate] if ccd is not None: elements.append(f"{ccd:03d}") if filter is not None: elements.append(filter) return "gen2/{}".format("_".join(elements))
"""Base class for Translator helpers that each handle just one Gen3 Data ID key.
Parameters ---------- dimension : `str` Name of the Gen3 dimension (data ID key) populated by this handler (e.g. "visit" or "abstract_filter"). """
skyMap: Optional[BaseSkyMap], skyMapName: Optional[str], datasetTypeName: str): """Update a Gen3 data ID dict with a single key-value pair from a Gen2 data ID.
This method is implemented by the base class and is not expected to be re-implemented by subclasses.
Parameters ---------- gen2id: `dict` Gen2 data ID from which to draw key-value pairs from. gen3id: `dict` Gen3 data ID to update in-place. skyMap: `BaseSkyMap`, optional SkyMap that defines the tracts and patches used in the Gen2 data ID, if any. skyMapName: `str` Name of the Gen3 skymap dimension that defines the tracts and patches used in the Gen3 data ID. datasetTypeName: `str` Name of the dataset type. """ gen3id[self.dimension] = self.extract(gen2id, skyMap=skyMap, skyMapName=skyMapName, datasetTypeName=datasetTypeName)
datasetTypeName: str) -> Any: """Extract a Gen3 data ID value from a Gen2 data ID.
Parameters ---------- gen2id: `dict` Gen2 data ID from which to draw key-value pairs from. skyMap: `BaseSkyMap`, optional SkyMap that defines the tracts and patches used in the Gen2 data ID, if any. skyMapName: `str` Name of the Gen3 skymap dimension that defines the tracts and patches used in the Gen3 data ID. datasetTypeName: `str` Name of the dataset type. """ raise NotImplementedError()
"""A KeyHandler that adds a constant key-value pair to the Gen3 data ID.
Parameters ---------- dimension : `str` Name of the Gen3 dimension (data ID key) populated by this handler (e.g. "visit" or "abstract_filter"). value : `object` Data ID value. """
datasetTypeName: str) -> Any: # Docstring inherited from KeyHandler.extract. return self.value
"""A KeyHandler that simply copies a value from a Gen3 data ID.
Parameters ---------- dimension : `str` Name of the Gen3 dimension produced by this handler. dtype : `type`, optional If not `None`, the type that values for this key must be an instance of. """ dtype: Optional[type] = None):
datasetTypeName: str) -> Any: # Docstring inherited from KeyHandler.extract. r = gen2id[self.gen2key] if self.dtype is not None: try: r = self.dtype(r) except ValueError as err: raise TypeError( f"'{r}' is not a valid value for {self.dimension}; " f"expected {self.dtype.__name__}, got {type(r).__name__}." ) from err return r
"""A KeyHandler for skymap patches. """
datasetTypeName: str) -> Any: # Docstring inherited from KeyHandler.extract. tract = gen2id["tract"] tractInfo = skyMap[tract] x, y = gen2id["patch"].split(",") patchInfo = tractInfo[int(x), int(y)] return tractInfo.getSequentialPatchIndex(patchInfo)
"""A KeyHandler for skymaps."""
datasetTypeName: str) -> Any: # Docstring inherited from KeyHandler.extract. return skyMapName
"""A KeyHandler for master calibration datasets. """
datasetTypeName: str) -> Any: # Docstring inherited from KeyHandler.extract. return makeCalibrationLabel(datasetTypeName, gen2id["calibDate"], ccd=gen2id.get("ccd"), filter=gen2id.get("filter"))
"""Callable object that translates Gen2 Data IDs to Gen3 Data IDs for a particular DatasetType.
Translators should usually be constructed via the `makeMatching` method.
Parameters ---------- handlers : `list` A list of KeyHandlers this Translator should use. skyMap : `BaseSkyMap`, optional SkyMap instance used to define any tract or patch Dimensions. skyMapName : `str` Gen3 SkyMap Dimension name to be associated with any tract or patch Dimensions. datasetTypeName : `str` Name of the dataset type whose data IDs this translator handles. """ datasetTypeName: str): self.handlers = handlers self.skyMap = skyMap self.skyMapName = skyMapName self.datasetTypeName = datasetTypeName
# Rules used to match Handlers when constring a Translator. # outer key is instrument name, or None for any # inner key is DatasetType name, or None for any # values are 3-tuples of (frozenset(gen2keys), handler, consume) Optional[str], Dict[ Optional[str], Tuple[FrozenSet[str], KeyHandler, bool] ] ] = { None: { None: [] } }
datasetTypeName: Optional[str] = None, gen2keys: Iterable[str] = (), consume: bool = True): """Add a KeyHandler and an associated matching rule.
Parameters ---------- handler : `KeyHandler` A KeyHandler instance to add to a Translator when this rule matches. instrument : `str` Gen3 instrument name the Gen2 repository must be associated with for this rule to match, or None to match any instrument. datasetTypeName : `str` Name of the DatasetType this rule matches, or None to match any DatasetType. gen2Keys : sequence Sequence of Gen2 data ID keys that must all be present for this rule to match. consume : `bool` or `tuple` If True (default), remove all entries in gen2keys from the set of keys being matched to in order to prevent less-specific handlers from matching them. May also be a `tuple` listing only the keys to consume. """ # Ensure consume is always a frozenset, so we can process it uniformly # from here on. consume = frozenset(consume) else: # find the rules for this instrument, or if we haven't seen it before, # add a nested dictionary that matches any DatasetType name and then # append this rule.
skyMap: Optional[BaseSkyMap] = None, skyMapName: Optional[str] = None): """Construct a Translator appropriate for instances of the given dataset.
Parameters ---------- datasetTypeName : `str` Name of the dataset type. gen2keys: `dict` Keys of a Gen2 data ID for this dataset. instrument: `str`, optional Name of the Gen3 instrument dimension for translated data IDs. skyMap: `~lsst.skymap.BaseSkyMap`, optional The skymap instance that defines any tract/patch data IDs. `~lsst.skymap.BaseSkyMap` instances. skyMapName : `str`, optional Gen3 SkyMap Dimension name to be associated with any tract or patch Dimensions.
Returns ------- translator : `Translator` A translator whose translate() method can be used to transform Gen2 data IDs to Gen3 dataIds. """ if instrument is not None: rulesForInstrument = cls._rules.get(instrument, {None: []}) else: rulesForInstrument = {None: []} rulesForAnyInstrument = cls._rules[None] candidateRules = itertools.chain( rulesForInstrument.get(datasetTypeName, []), # this instrument, this DatasetType rulesForInstrument[None], # this instrument, any DatasetType rulesForAnyInstrument.get(datasetTypeName, []), # any instrument, this DatasetType rulesForAnyInstrument[None], # any instrument, any DatasetType ) matchedHandlers = [] targetKeys = set(gen2keys) for ruleKeys, ruleHandlers, consume in candidateRules: if ruleKeys.issubset(targetKeys): matchedHandlers.append(ruleHandlers) targetKeys -= consume return Translator(matchedHandlers, skyMap=skyMap, skyMapName=skyMapName, datasetTypeName=datasetTypeName)
"""Return a Gen3 data ID that corresponds to the given Gen2 data ID. """ gen3id = {} for handler in self.handlers: try: handler.translate(gen2id, gen3id, skyMap=self.skyMap, skyMapName=self.skyMapName, datasetTypeName=self.datasetTypeName) except KeyError: if partial: if log is not None: log.debug("Failed to translate %s from %s.", handler.dimension, gen2id) continue else: raise return gen3id
def dimensionNames(self): """The names of the dimensions populated by this Translator (`frozenset`). """ return frozenset(h.dimension for h in self.handlers)
# Add "skymap" to Gen3 ID if Gen2 ID has a "tract" key.
# Add "skymap" to Gen3 ID if DatasetType is one of a few specific ones
# Translate Gen2 str patch IDs to Gen3 sequential integers.
# Copy Gen2 "tract" to Gen3 "tract".
# Add valid_first, valid_last to instrument-level transmission/ datasets; # these are considered calibration products in Gen3. datasetTypeName=datasetTypeName)
# Translate Gen2 pixel_id to Gen3 skypix. # TODO: For now, we just assume that the refcat indexer uses htm7, since that's # what the ps1 refcat in testdata_ci_hsc uses.
# Translate Gen2 calibDate and datasetType to Gen3 calibration_label. |