Coverage for python/lsst/obs/base/gen2to3/translators.py : 47%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ("Translator", "KeyHandler", "CopyKeyHandler", "ConstantKeyHandler",
23 "CalibKeyHandler", "AbstractToPhysicalFilterKeyHandler", "PhysicalToAbstractFilterKeyHandler",
24 "makeCalibrationLabel")
26import itertools
27from typing import Optional, Any, Dict, Tuple, FrozenSet, Iterable, List
28from abc import ABCMeta, abstractmethod
30from lsst.log import Log
31from lsst.skymap import BaseSkyMap
34def makeCalibrationLabel(datasetTypeName: str, calibDate: str, ccd: Optional[int] = None,
35 filter: Optional[str] = None) -> str:
36 """Make a Gen3 calibration_label string corresponding to a Gen2 data ID.
38 Parameters
39 ----------
40 datasetTypeName : `str`
41 Name of the dataset type this calibration label identifies.
42 calibDate : `str`
43 Date string used in the Gen2 template.
44 ccd : `int`, optional
45 Detector ID used in the Gen2 template.
46 filter : `str`, optional
47 Filter used in the Gen2 template.
49 Returns
50 -------
51 label : `str`
52 Calibration label string.
53 """
54 # TODO: this function is probably HSC-specific, but I don't know how other
55 # obs calib registries behave so I don't know (yet) how to generalize it.
56 elements = [datasetTypeName, calibDate]
57 if ccd is not None:
58 elements.append(f"{ccd:03d}")
59 if filter is not None:
60 elements.append(filter)
61 return "gen2/{}".format("_".join(elements))
64class KeyHandler(metaclass=ABCMeta):
65 """Base class for Translator helpers that each handle just one Gen3 Data
66 ID key.
68 Parameters
69 ----------
70 dimension : `str`
71 Name of the Gen3 dimension (data ID key) populated by
72 this handler (e.g. "visit" or "abstract_filter").
73 """
74 def __init__(self, dimension: str):
75 self.dimension = dimension
77 __slots__ = ("dimension",)
79 def __str__(self):
80 return f"{type(self).__name__}({self.dimension})"
82 def translate(self, gen2id: dict, gen3id: dict,
83 skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
84 datasetTypeName: str):
85 """Update a Gen3 data ID dict with a single key-value pair from a Gen2
86 data ID.
88 This method is implemented by the base class and is not expected to
89 be re-implemented by subclasses.
91 Parameters
92 ----------
93 gen2id: `dict`
94 Gen2 data ID from which to draw key-value pairs from.
95 gen3id: `dict`
96 Gen3 data ID to update in-place.
97 skyMap: `BaseSkyMap`, optional
98 SkyMap that defines the tracts and patches used in the Gen2 data
99 ID, if any.
100 skyMapName: `str`
101 Name of the Gen3 skymap dimension that defines the tracts and
102 patches used in the Gen3 data ID.
103 datasetTypeName: `str`
104 Name of the dataset type.
105 """
106 gen3id[self.dimension] = self.extract(gen2id, skyMap=skyMap, skyMapName=skyMapName,
107 datasetTypeName=datasetTypeName)
109 @abstractmethod
110 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
111 datasetTypeName: str) -> Any:
112 """Extract a Gen3 data ID value from a Gen2 data ID.
114 Parameters
115 ----------
116 gen2id: `dict`
117 Gen2 data ID from which to draw key-value pairs from.
118 skyMap: `BaseSkyMap`, optional
119 SkyMap that defines the tracts and patches used in the Gen2 data
120 ID, if any.
121 skyMapName: `str`
122 Name of the Gen3 skymap dimension that defines the tracts and
123 patches used in the Gen3 data ID.
124 datasetTypeName: `str`
125 Name of the dataset type.
126 """
127 raise NotImplementedError()
130class ConstantKeyHandler(KeyHandler):
131 """A KeyHandler that adds a constant key-value pair to the Gen3 data ID.
133 Parameters
134 ----------
135 dimension : `str`
136 Name of the Gen3 dimension (data ID key) populated by
137 this handler (e.g. "visit" or "abstract_filter").
138 value : `object`
139 Data ID value.
140 """
141 def __init__(self, dimension: str, value: Any):
142 super().__init__(dimension)
143 self.value = value
145 __slots__ = ("value",)
147 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
148 datasetTypeName: str) -> Any:
149 # Docstring inherited from KeyHandler.extract.
150 return self.value
153class CopyKeyHandler(KeyHandler):
154 """A KeyHandler that simply copies a value from a Gen3 data ID.
156 Parameters
157 ----------
158 dimension : `str`
159 Name of the Gen3 dimension produced by this handler.
160 dtype : `type`, optional
161 If not `None`, the type that values for this key must be an
162 instance of.
163 """
164 def __init__(self, dimension: str, gen2key: Optional[str] = None,
165 dtype: Optional[type] = None):
166 super().__init__(dimension)
167 self.gen2key = gen2key if gen2key is not None else dimension
168 self.dtype = dtype
170 __slots__ = ("gen2key", "dtype")
172 def __str__(self):
173 return f"{type(self).__name__}({self.gen2key}, {self.dtype})"
175 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
176 datasetTypeName: str) -> Any:
177 # Docstring inherited from KeyHandler.extract.
178 r = gen2id[self.gen2key]
179 if self.dtype is not None:
180 try:
181 r = self.dtype(r)
182 except ValueError as err:
183 raise TypeError(
184 f"'{r}' is not a valid value for {self.dimension}; "
185 f"expected {self.dtype.__name__}, got {type(r).__name__}."
186 ) from err
187 return r
190class PatchKeyHandler(KeyHandler):
191 """A KeyHandler for skymap patches.
192 """
193 def __init__(self):
194 super().__init__("patch")
196 __slots__ = ()
198 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
199 datasetTypeName: str) -> Any:
200 # Docstring inherited from KeyHandler.extract.
201 tract = gen2id["tract"]
202 tractInfo = skyMap[tract]
203 x, y = gen2id["patch"].split(",")
204 patchInfo = tractInfo[int(x), int(y)]
205 return tractInfo.getSequentialPatchIndex(patchInfo)
208class SkyMapKeyHandler(KeyHandler):
209 """A KeyHandler for skymaps."""
210 def __init__(self):
211 super().__init__("skymap")
213 __slots__ = ()
215 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
216 datasetTypeName: str) -> Any:
217 # Docstring inherited from KeyHandler.extract.
218 return skyMapName
221class CalibKeyHandler(KeyHandler):
222 """A KeyHandler for master calibration datasets.
223 """
224 __slots__ = ("ccdKey",)
226 def __init__(self, ccdKey="ccd"):
227 self.ccdKey = ccdKey
228 super().__init__("calibration_label")
230 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
231 datasetTypeName: str) -> Any:
232 # Docstring inherited from KeyHandler.extract.
233 return makeCalibrationLabel(datasetTypeName, gen2id["calibDate"],
234 ccd=gen2id.get(self.ccdKey), filter=gen2id.get("filter"))
237class PhysicalToAbstractFilterKeyHandler(KeyHandler):
238 """KeyHandler for gen2 ``filter`` keys that match ``physical_filter``
239 keys in gen3 but should be mapped to ``abstract_filter``.
241 Note that multiple physical filter can potentially map to one abstract
242 filter, so be careful to only use this translator on obs packages where
243 there is a one-to-one mapping.
244 """
246 __slots__ = ("_map",)
248 def __init__(self, filterDefinitions):
249 super().__init__("abstract_filter")
250 self._map = {d.physical_filter: d.abstract_filter for d in filterDefinitions
251 if d.physical_filter is not None}
253 def extract(self, gen2id, *args, **kwargs):
254 physical = gen2id["filter"]
255 return self._map.get(physical, physical)
258class AbstractToPhysicalFilterKeyHandler(KeyHandler):
259 """KeyHandler for gen2 ``filter`` keys that match ``abstract_filter``
260 keys in gen3 but should be mapped to ``physical_filter``.
262 Note that one abstract filter can potentially map to multiple physical
263 filters, so be careful to only use this translator on obs packages where
264 there is a one-to-one mapping.
265 """
267 __slots__ = ("_map",)
269 def __init__(self, filterDefinitions):
270 super().__init__("physical_filter")
271 self._map = {d.abstract_filter: d.physical_filter for d in filterDefinitions
272 if d.abstract_filter is not None}
274 def extract(self, gen2id, *args, **kwargs):
275 abstract = gen2id["filter"]
276 return self._map.get(abstract, abstract)
279class Translator:
280 """Callable object that translates Gen2 Data IDs to Gen3 Data IDs for a
281 particular DatasetType.
283 Translators should usually be constructed via the `makeMatching` method.
285 Parameters
286 ----------
287 handlers : `list`
288 A list of KeyHandlers this Translator should use.
289 skyMap : `BaseSkyMap`, optional
290 SkyMap instance used to define any tract or patch Dimensions.
291 skyMapName : `str`
292 Gen3 SkyMap Dimension name to be associated with any tract or patch
293 Dimensions.
294 datasetTypeName : `str`
295 Name of the dataset type whose data IDs this translator handles.
296 """
297 def __init__(self, handlers: List[KeyHandler], skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
298 datasetTypeName: str):
299 self.handlers = handlers
300 self.skyMap = skyMap
301 self.skyMapName = skyMapName
302 self.datasetTypeName = datasetTypeName
304 __slots__ = ("handlers", "skyMap", "skyMapName", "datasetTypeName")
306 # Rules used to match Handlers when constring a Translator.
307 # outer key is instrument name, or None for any
308 # inner key is DatasetType name, or None for any
309 # values are 3-tuples of (frozenset(gen2keys), handler, consume)
310 _rules: Dict[
311 Optional[str],
312 Dict[
313 Optional[str],
314 Tuple[FrozenSet[str], KeyHandler, bool]
315 ]
316 ] = {
317 None: {
318 None: []
319 }
320 }
322 def __str__(self):
323 hstr = ",".join(str(h) for h in self.handlers)
324 return f"{type(self).__name__}(dtype={self.datasetTypeName}, handlers=[{hstr}])"
326 @classmethod
327 def addRule(cls, handler: KeyHandler, instrument: Optional[str] = None,
328 datasetTypeName: Optional[str] = None, gen2keys: Iterable[str] = (),
329 consume: bool = True):
330 """Add a KeyHandler and an associated matching rule.
332 Parameters
333 ----------
334 handler : `KeyHandler`
335 A KeyHandler instance to add to a Translator when this rule
336 matches.
337 instrument : `str`
338 Gen3 instrument name the Gen2 repository must be associated with
339 for this rule to match, or None to match any instrument.
340 datasetTypeName : `str`
341 Name of the DatasetType this rule matches, or None to match any
342 DatasetType.
343 gen2Keys : sequence
344 Sequence of Gen2 data ID keys that must all be present for this
345 rule to match.
346 consume : `bool` or `tuple`
347 If True (default), remove all entries in gen2keys from the set of
348 keys being matched to in order to prevent less-specific handlers
349 from matching them.
350 May also be a `tuple` listing only the keys to consume.
351 """
352 # Ensure consume is always a frozenset, so we can process it uniformly
353 # from here on.
354 if consume is True:
355 consume = frozenset(gen2keys)
356 elif consume: 356 ↛ 357line 356 didn't jump to line 357, because the condition on line 356 was never true
357 consume = frozenset(consume)
358 else:
359 consume = frozenset()
360 # find the rules for this instrument, or if we haven't seen it before,
361 # add a nested dictionary that matches any DatasetType name and then
362 # append this rule.
363 rulesForInstrument = cls._rules.setdefault(instrument, {None: []})
364 rulesForInstrumentAndDatasetType = rulesForInstrument.setdefault(datasetTypeName, [])
365 rulesForInstrumentAndDatasetType.append((frozenset(gen2keys), handler, consume))
367 @classmethod
368 def makeMatching(cls, datasetTypeName: str, gen2keys: Dict[str, type], instrument: Optional[str] = None,
369 skyMap: Optional[BaseSkyMap] = None, skyMapName: Optional[str] = None):
370 """Construct a Translator appropriate for instances of the given
371 dataset.
373 Parameters
374 ----------
375 datasetTypeName : `str`
376 Name of the dataset type.
377 gen2keys: `dict`
378 Keys of a Gen2 data ID for this dataset.
379 instrument: `str`, optional
380 Name of the Gen3 instrument dimension for translated data IDs.
381 skyMap: `~lsst.skymap.BaseSkyMap`, optional
382 The skymap instance that defines any tract/patch data IDs.
383 `~lsst.skymap.BaseSkyMap` instances.
384 skyMapName : `str`, optional
385 Gen3 SkyMap Dimension name to be associated with any tract or patch
386 Dimensions.
388 Returns
389 -------
390 translator : `Translator`
391 A translator whose translate() method can be used to transform Gen2
392 data IDs to Gen3 dataIds.
393 """
394 if instrument is not None:
395 rulesForInstrument = cls._rules.get(instrument, {None: []})
396 else:
397 rulesForInstrument = {None: []}
398 rulesForAnyInstrument = cls._rules[None]
399 candidateRules = itertools.chain(
400 rulesForInstrument.get(datasetTypeName, []), # this instrument, this DatasetType
401 rulesForInstrument[None], # this instrument, any DatasetType
402 rulesForAnyInstrument.get(datasetTypeName, []), # any instrument, this DatasetType
403 rulesForAnyInstrument[None], # any instrument, any DatasetType
404 )
405 matchedHandlers = []
406 targetKeys = set(gen2keys)
407 for ruleKeys, ruleHandlers, consume in candidateRules:
408 if ruleKeys.issubset(targetKeys):
409 matchedHandlers.append(ruleHandlers)
410 targetKeys -= consume
411 return Translator(matchedHandlers, skyMap=skyMap, skyMapName=skyMapName,
412 datasetTypeName=datasetTypeName)
414 def __call__(self, gen2id: Dict[str, Any], *, partial: bool = False, log: Optional[Log] = None):
415 """Return a Gen3 data ID that corresponds to the given Gen2 data ID.
416 """
417 gen3id = {}
418 for handler in self.handlers:
419 try:
420 handler.translate(gen2id, gen3id, skyMap=self.skyMap, skyMapName=self.skyMapName,
421 datasetTypeName=self.datasetTypeName)
422 except KeyError:
423 if partial:
424 if log is not None:
425 log.debug("Failed to translate %s from %s.", handler.dimension, gen2id)
426 continue
427 else:
428 raise
429 return gen3id
431 @property
432 def dimensionNames(self):
433 """The names of the dimensions populated by this Translator
434 (`frozenset`).
435 """
436 return frozenset(h.dimension for h in self.handlers)
439# Add "skymap" to Gen3 ID if Gen2 ID has a "tract" key.
440Translator.addRule(SkyMapKeyHandler(), gen2keys=("tract",), consume=False)
442# Add "skymap" to Gen3 ID if DatasetType is one of a few specific ones
443for coaddName in ("deep", "goodSeeing", "psfMatched", "dcr"):
444 Translator.addRule(SkyMapKeyHandler(), datasetTypeName=f"{coaddName}Coadd_skyMap")
446# Translate Gen2 str patch IDs to Gen3 sequential integers.
447Translator.addRule(PatchKeyHandler(), gen2keys=("patch",))
449# Copy Gen2 "tract" to Gen3 "tract".
450Translator.addRule(CopyKeyHandler("tract", dtype=int), gen2keys=("tract",))
452# Add valid_first, valid_last to instrument-level transmission/ datasets;
453# these are considered calibration products in Gen3.
454for datasetTypeName in ("transmission_sensor", "transmission_optics", "transmission_filter"):
455 Translator.addRule(ConstantKeyHandler("calibration_label", "unbounded"),
456 datasetTypeName=datasetTypeName)
458# Translate Gen2 pixel_id to Gen3 skypix.
459# TODO: For now, we just assume that the refcat indexer uses htm7, since that's
460# what we have generated most of our refcats at.
461Translator.addRule(CopyKeyHandler("htm7", gen2key="pixel_id", dtype=int), gen2keys=("pixel_id",))