22 __all__ = (
"Translator",
"KeyHandler",
"CopyKeyHandler",
"ConstantKeyHandler",
23 "CalibKeyHandler",
"AbstractToPhysicalFilterKeyHandler",
"PhysicalToAbstractFilterKeyHandler",
24 "makeCalibrationLabel")
27 from typing
import Optional, Any, Dict, Tuple, FrozenSet, Iterable, List
28 from abc
import ABCMeta, abstractmethod
30 from lsst.log
import Log
31 from lsst.skymap
import BaseSkyMap
35 filter: Optional[str] =
None) -> str:
36 """Make a Gen3 calibration_label string corresponding to a Gen2 data ID.
40 datasetTypeName : `str`
41 Name of the dataset type this calibration label identifies.
43 Date string used in the Gen2 template.
45 Detector ID used in the Gen2 template.
46 filter : `str`, optional
47 Filter used in the Gen2 template.
52 Calibration label string.
56 elements = [datasetTypeName, calibDate]
58 elements.append(f
"{ccd:03d}")
59 if filter
is not None:
60 elements.append(filter)
61 return "gen2/{}".format(
"_".join(elements))
65 """Base class for Translator helpers that each handle just one Gen3 Data
71 Name of the Gen3 dimension (data ID key) populated by
72 this handler (e.g. "visit" or "abstract_filter").
77 __slots__ = (
"dimension",)
80 return f
"{type(self).__name__}({self.dimension})"
83 skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
84 datasetTypeName: str):
85 """Update a Gen3 data ID dict with a single key-value pair from a Gen2
88 This method is implemented by the base class and is not expected to
89 be re-implemented by subclasses.
94 Gen2 data ID from which to draw key-value pairs from.
96 Gen3 data ID to update in-place.
97 skyMap: `BaseSkyMap`, optional
98 SkyMap that defines the tracts and patches used in the Gen2 data
101 Name of the Gen3 skymap dimension that defines the tracts and
102 patches used in the Gen3 data ID.
103 datasetTypeName: `str`
104 Name of the dataset type.
106 gen3id[self.
dimension] = self.
extract(gen2id, skyMap=skyMap, skyMapName=skyMapName,
107 datasetTypeName=datasetTypeName)
110 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
111 datasetTypeName: str) -> Any:
112 """Extract a Gen3 data ID value from a Gen2 data ID.
117 Gen2 data ID from which to draw key-value pairs from.
118 skyMap: `BaseSkyMap`, optional
119 SkyMap that defines the tracts and patches used in the Gen2 data
122 Name of the Gen3 skymap dimension that defines the tracts and
123 patches used in the Gen3 data ID.
124 datasetTypeName: `str`
125 Name of the dataset type.
127 raise NotImplementedError()
131 """A KeyHandler that adds a constant key-value pair to the Gen3 data ID.
136 Name of the Gen3 dimension (data ID key) populated by
137 this handler (e.g. "visit" or "abstract_filter").
145 __slots__ = (
"value",)
147 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
148 datasetTypeName: str) -> Any:
154 """A KeyHandler that simply copies a value from a Gen3 data ID.
159 Name of the Gen3 dimension produced by this handler.
160 dtype : `type`, optional
161 If not `None`, the type that values for this key must be an
164 def __init__(self, dimension: str, gen2key: Optional[str] =
None,
165 dtype: Optional[type] =
None):
167 self.
gen2key = gen2key
if gen2key
is not None else dimension
170 __slots__ = (
"gen2key",
"dtype")
173 return f
"{type(self).__name__}({self.gen2key}, {self.dtype})"
175 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
176 datasetTypeName: str) -> Any:
179 if self.
dtype is not None:
182 except ValueError
as err:
184 f
"'{r}' is not a valid value for {self.dimension}; "
185 f
"expected {self.dtype.__name__}, got {type(r).__name__}."
191 """A KeyHandler for skymap patches.
198 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
199 datasetTypeName: str) -> Any:
201 tract = gen2id[
"tract"]
202 tractInfo = skyMap[tract]
203 x, y = gen2id[
"patch"].split(
",")
204 patchInfo = tractInfo[int(x), int(y)]
205 return tractInfo.getSequentialPatchIndex(patchInfo)
209 """A KeyHandler for skymaps."""
215 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
216 datasetTypeName: str) -> Any:
222 """A KeyHandler for master calibration datasets.
224 __slots__ = (
"ccdKey",)
228 super().
__init__(
"calibration_label")
230 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
231 datasetTypeName: str) -> Any:
234 ccd=gen2id.get(self.
ccdKey), filter=gen2id.get(
"filter"))
238 """KeyHandler for gen2 ``filter`` keys that match ``physical_filter``
239 keys in gen3 but should be mapped to ``abstract_filter``.
241 Note that multiple physical filter can potentially map to one abstract
242 filter, so be careful to only use this translator on obs packages where
243 there is a one-to-one mapping.
246 __slots__ = (
"_map",)
250 self.
_map = {d.physical_filter: d.abstract_filter
for d
in filterDefinitions
251 if d.physical_filter
is not None}
254 physical = gen2id[
"filter"]
255 return self.
_map.get(physical, physical)
259 """KeyHandler for gen2 ``filter`` keys that match ``abstract_filter``
260 keys in gen3 but should be mapped to ``physical_filter``.
262 Note that one abstract filter can potentially map to multiple physical
263 filters, so be careful to only use this translator on obs packages where
264 there is a one-to-one mapping.
267 __slots__ = (
"_map",)
271 self.
_map = {d.abstract_filter: d.physical_filter
for d
in filterDefinitions
272 if d.abstract_filter
is not None}
275 abstract = gen2id[
"filter"]
276 return self.
_map.get(abstract, abstract)
280 """Callable object that translates Gen2 Data IDs to Gen3 Data IDs for a
281 particular DatasetType.
283 Translators should usually be constructed via the `makeMatching` method.
288 A list of KeyHandlers this Translator should use.
289 skyMap : `BaseSkyMap`, optional
290 SkyMap instance used to define any tract or patch Dimensions.
292 Gen3 SkyMap Dimension name to be associated with any tract or patch
294 datasetTypeName : `str`
295 Name of the dataset type whose data IDs this translator handles.
297 def __init__(self, handlers: List[KeyHandler], skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
298 datasetTypeName: str):
304 __slots__ = (
"handlers",
"skyMap",
"skyMapName",
"datasetTypeName")
314 Tuple[FrozenSet[str], KeyHandler, bool]
323 hstr =
",".join(str(h)
for h
in self.
handlers)
324 return f
"{type(self).__name__}(dtype={self.datasetTypeName}, handlers=[{hstr}])"
327 def addRule(cls, handler: KeyHandler, instrument: Optional[str] =
None,
328 datasetTypeName: Optional[str] =
None, gen2keys: Iterable[str] = (),
329 consume: bool =
True):
330 """Add a KeyHandler and an associated matching rule.
334 handler : `KeyHandler`
335 A KeyHandler instance to add to a Translator when this rule
338 Gen3 instrument name the Gen2 repository must be associated with
339 for this rule to match, or None to match any instrument.
340 datasetTypeName : `str`
341 Name of the DatasetType this rule matches, or None to match any
344 Sequence of Gen2 data ID keys that must all be present for this
346 consume : `bool` or `tuple`
347 If True (default), remove all entries in gen2keys from the set of
348 keys being matched to in order to prevent less-specific handlers
350 May also be a `tuple` listing only the keys to consume.
355 consume = frozenset(gen2keys)
357 consume = frozenset(consume)
359 consume = frozenset()
363 rulesForInstrument = cls._rules.setdefault(instrument, {
None: []})
364 rulesForInstrumentAndDatasetType = rulesForInstrument.setdefault(datasetTypeName, [])
365 rulesForInstrumentAndDatasetType.append((frozenset(gen2keys), handler, consume))
368 def makeMatching(cls, datasetTypeName: str, gen2keys: Dict[str, type], instrument: Optional[str] =
None,
369 skyMap: Optional[BaseSkyMap] =
None, skyMapName: Optional[str] =
None):
370 """Construct a Translator appropriate for instances of the given
375 datasetTypeName : `str`
376 Name of the dataset type.
378 Keys of a Gen2 data ID for this dataset.
379 instrument: `str`, optional
380 Name of the Gen3 instrument dimension for translated data IDs.
381 skyMap: `~lsst.skymap.BaseSkyMap`, optional
382 The skymap instance that defines any tract/patch data IDs.
383 `~lsst.skymap.BaseSkyMap` instances.
384 skyMapName : `str`, optional
385 Gen3 SkyMap Dimension name to be associated with any tract or patch
390 translator : `Translator`
391 A translator whose translate() method can be used to transform Gen2
392 data IDs to Gen3 dataIds.
394 if instrument
is not None:
395 rulesForInstrument = cls._rules.get(instrument, {
None: []})
397 rulesForInstrument = {
None: []}
398 rulesForAnyInstrument = cls._rules[
None]
399 candidateRules = itertools.chain(
400 rulesForInstrument.get(datasetTypeName, []),
401 rulesForInstrument[
None],
402 rulesForAnyInstrument.get(datasetTypeName, []),
403 rulesForAnyInstrument[
None],
406 targetKeys = set(gen2keys)
407 for ruleKeys, ruleHandlers, consume
in candidateRules:
408 if ruleKeys.issubset(targetKeys):
409 matchedHandlers.append(ruleHandlers)
410 targetKeys -= consume
411 return Translator(matchedHandlers, skyMap=skyMap, skyMapName=skyMapName,
412 datasetTypeName=datasetTypeName)
414 def __call__(self, gen2id: Dict[str, Any], *, partial: bool =
False, log: Optional[Log] =
None):
415 """Return a Gen3 data ID that corresponds to the given Gen2 data ID.
420 handler.translate(gen2id, gen3id, skyMap=self.
skyMap, skyMapName=self.
skyMapName,
425 log.debug(
"Failed to translate %s from %s.", handler.dimension, gen2id)
433 """The names of the dimensions populated by this Translator
436 return frozenset(h.dimension
for h
in self.
handlers)
443 for coaddName
in (
"deep",
"goodSeeing",
"psfMatched",
"dcr"):
450 Translator.addRule(
CopyKeyHandler(
"tract", dtype=int), gen2keys=(
"tract",))
454 for datasetTypeName
in (
"transmission_sensor",
"transmission_optics",
"transmission_filter"):
456 datasetTypeName=datasetTypeName)
461 Translator.addRule(
CopyKeyHandler(
"htm7", gen2key=
"pixel_id", dtype=int), gen2keys=(
"pixel_id",))