22 from __future__
import annotations
24 __all__ = (
"Translator",
"TranslatorFactory",
"KeyHandler",
"CopyKeyHandler",
"ConstantKeyHandler",
25 "CalibKeyHandler",
"AbstractToPhysicalFilterKeyHandler",
"PhysicalToAbstractFilterKeyHandler",
26 "makeCalibrationLabel")
29 from typing
import Optional, Any, Dict, Tuple, FrozenSet, Iterable, List
30 from abc
import ABCMeta, abstractmethod
32 from lsst.log
import Log
33 from lsst.skymap
import BaseSkyMap
37 filter: Optional[str] =
None) -> str:
38 """Make a Gen3 calibration_label string corresponding to a Gen2 data ID.
42 datasetTypeName : `str`
43 Name of the dataset type this calibration label identifies.
45 Date string used in the Gen2 template.
47 Detector ID used in the Gen2 template.
48 filter : `str`, optional
49 Filter used in the Gen2 template.
54 Calibration label string.
58 elements = [datasetTypeName, calibDate]
60 elements.append(f
"{ccd:03d}")
61 if filter
is not None:
62 elements.append(filter)
63 return "gen2/{}".format(
"_".join(elements))
67 """Base class for Translator helpers that each handle just one Gen3 Data
73 Name of the Gen3 dimension (data ID key) populated by
74 this handler (e.g. "visit" or "abstract_filter").
79 __slots__ = (
"dimension",)
82 return f
"{type(self).__name__}({self.dimension})"
85 skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
86 datasetTypeName: str):
87 """Update a Gen3 data ID dict with a single key-value pair from a Gen2
90 This method is implemented by the base class and is not expected to
91 be re-implemented by subclasses.
96 Gen2 data ID from which to draw key-value pairs from.
98 Gen3 data ID to update in-place.
99 skyMap: `BaseSkyMap`, optional
100 SkyMap that defines the tracts and patches used in the Gen2 data
103 Name of the Gen3 skymap dimension that defines the tracts and
104 patches used in the Gen3 data ID.
105 datasetTypeName: `str`
106 Name of the dataset type.
108 gen3id[self.
dimension] = self.
extract(gen2id, skyMap=skyMap, skyMapName=skyMapName,
109 datasetTypeName=datasetTypeName)
112 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
113 datasetTypeName: str) -> Any:
114 """Extract a Gen3 data ID value from a Gen2 data ID.
119 Gen2 data ID from which to draw key-value pairs from.
120 skyMap: `BaseSkyMap`, optional
121 SkyMap that defines the tracts and patches used in the Gen2 data
124 Name of the Gen3 skymap dimension that defines the tracts and
125 patches used in the Gen3 data ID.
126 datasetTypeName: `str`
127 Name of the dataset type.
129 raise NotImplementedError()
133 """A KeyHandler that adds a constant key-value pair to the Gen3 data ID.
138 Name of the Gen3 dimension (data ID key) populated by
139 this handler (e.g. "visit" or "abstract_filter").
147 __slots__ = (
"value",)
149 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
150 datasetTypeName: str) -> Any:
156 """A KeyHandler that simply copies a value from a Gen3 data ID.
161 Name of the Gen3 dimension produced by this handler.
162 dtype : `type`, optional
163 If not `None`, the type that values for this key must be an
166 def __init__(self, dimension: str, gen2key: Optional[str] =
None,
167 dtype: Optional[type] =
None):
169 self.
gen2key = gen2key
if gen2key
is not None else dimension
172 __slots__ = (
"gen2key",
"dtype")
175 return f
"{type(self).__name__}({self.gen2key}, {self.dtype})"
177 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
178 datasetTypeName: str) -> Any:
181 if self.
dtype is not None:
184 except ValueError
as err:
186 f
"'{r}' is not a valid value for {self.dimension}; "
187 f
"expected {self.dtype.__name__}, got {type(r).__name__}."
193 """A KeyHandler for skymap patches.
200 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
201 datasetTypeName: str) -> Any:
203 tract = gen2id[
"tract"]
204 tractInfo = skyMap[tract]
205 x, y = gen2id[
"patch"].split(
",")
206 patchInfo = tractInfo[int(x), int(y)]
207 return tractInfo.getSequentialPatchIndex(patchInfo)
211 """A KeyHandler for skymaps."""
217 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
218 datasetTypeName: str) -> Any:
224 """A KeyHandler for master calibration datasets.
226 __slots__ = (
"ccdKey",)
230 super().
__init__(
"calibration_label")
232 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
233 datasetTypeName: str) -> Any:
236 ccd=gen2id.get(self.
ccdKey), filter=gen2id.get(
"filter"))
240 """KeyHandler for gen2 ``filter`` keys that match ``physical_filter``
241 keys in gen3 but should be mapped to ``abstract_filter``.
243 Note that multiple physical filter can potentially map to one abstract
244 filter, so be careful to only use this translator on obs packages where
245 there is a one-to-one mapping.
248 __slots__ = (
"_map",)
252 self.
_map = {d.physical_filter: d.abstract_filter
for d
in filterDefinitions
253 if d.physical_filter
is not None}
256 physical = gen2id[
"filter"]
257 return self.
_map.get(physical, physical)
261 """KeyHandler for gen2 ``filter`` keys that match ``abstract_filter``
262 keys in gen3 but should be mapped to ``physical_filter``.
264 Note that one abstract filter can potentially map to multiple physical
265 filters, so be careful to only use this translator on obs packages where
266 there is a one-to-one mapping.
269 __slots__ = (
"_map",)
273 self.
_map = {d.abstract_filter: d.physical_filter
for d
in filterDefinitions
274 if d.abstract_filter
is not None}
277 abstract = gen2id[
"filter"]
278 return self.
_map.get(abstract, abstract)
282 """A class that manages a system of rules for translating Gen2 data IDs
283 to Gen3 data IDs, and uses these to construct translators for particular
293 List[Tuple[FrozenSet[str], KeyHandler, bool]]
304 for instrumentName, nested
in self._rules.items():
305 if instrumentName
is None:
306 instrumentName =
"[any instrument]"
307 for datasetTypeName, rules
in nested.items():
308 if datasetTypeName
is None:
309 datasetTypeName =
"[any dataset type]"
310 lines.append(f
"{instrumentName} + {datasetTypeName}:")
311 for gen2keys, handler, consume
in rules:
312 consumed =
" (consumed)" if consume
else ""
313 lines.append(f
" {gen2keys}{consumed}: {handler}")
314 return "\n".join(lines)
316 def addRule(self, handler: KeyHandler, instrument: Optional[str] =
None,
317 datasetTypeName: Optional[str] =
None, gen2keys: Iterable[str] = (),
318 consume: bool =
True):
319 """Add a KeyHandler and an associated matching rule.
323 handler : `KeyHandler`
324 A KeyHandler instance to add to a Translator when this rule
327 Gen3 instrument name the Gen2 repository must be associated with
328 for this rule to match, or None to match any instrument.
329 datasetTypeName : `str`
330 Name of the DatasetType this rule matches, or None to match any
333 Sequence of Gen2 data ID keys that must all be present for this
335 consume : `bool` or `tuple`
336 If True (default), remove all entries in gen2keys from the set of
337 keys being matched to in order to prevent less-specific handlers
339 May also be a `tuple` listing only the keys to consume.
344 consume = frozenset(gen2keys)
346 consume = frozenset(consume)
348 consume = frozenset()
352 rulesForInstrument = self._rules.setdefault(instrument, {
None: []})
353 rulesForInstrumentAndDatasetType = rulesForInstrument.setdefault(datasetTypeName, [])
354 rulesForInstrumentAndDatasetType.append((frozenset(gen2keys), handler, consume))
356 def _addDefaultRules(self):
357 """Add translator rules that should always be present, and don't depend
358 at all on the instrument whose datasets are being converted.
360 This is called by `TranslatorFactory` construction.
366 for coaddName
in (
"deep",
"goodSeeing",
"psfMatched",
"dcr"):
377 for datasetTypeName
in (
"transmission_sensor",
"transmission_optics",
"transmission_filter"):
379 datasetTypeName=datasetTypeName)
392 calibFilterType: str =
"physical_filter",
393 detectorKey: str =
"ccd",
394 exposureKey: str =
"visit"):
395 """Add translation rules that depend on some properties of the
396 instrument but are otherwise generic.
401 The short (dimension) name of the instrument that conversion is
403 calibFilterType : `str`, optional
404 One of ``physical_filter`` or ``abstract_filter``, indicating which
405 of those the gen2 calibRegistry uses as the ``filter`` key.
406 detectorKey : `str`, optional
407 The gen2 key used to identify what in gen3 is `detector`.
408 exposureKey : `str`, optional
409 The gen2 key used to identify what in gen3 is `exposure`.
415 instrument=instrumentName, gen2keys=(exposureKey,), consume=
False)
417 instrument=instrumentName, gen2keys=(detectorKey,), consume=
False)
419 instrument=instrumentName, gen2keys=(
"calibDate",), consume=
False)
424 instrument=instrumentName, datasetTypeName=
"raw", gen2keys=(exposureKey,),
425 consume=(exposureKey,
"filter"))
429 consume=(
"visit",
"filter"))
433 instrument=instrumentName,
434 gen2keys=(detectorKey,))
439 instrument=instrumentName, datasetTypeName=
"transmission_optics")
441 instrument=instrumentName, datasetTypeName=
"transmission_atmosphere")
443 instrument=instrumentName, datasetTypeName=
"transmission_filter")
445 instrument=instrumentName, datasetTypeName=
"transmission_filter")
448 for calibType
in (
'flat',
'sky',
'fringe'):
450 instrument=instrumentName, datasetTypeName=calibType)
455 def makeMatching(self, datasetTypeName: str, gen2keys: Dict[str, type], instrument: Optional[str] =
None,
456 skyMap: Optional[BaseSkyMap] =
None, skyMapName: Optional[str] =
None):
457 """Construct a Translator appropriate for instances of the given
462 datasetTypeName : `str`
463 Name of the dataset type.
465 Keys of a Gen2 data ID for this dataset.
466 instrument: `str`, optional
467 Name of the Gen3 instrument dimension for translated data IDs.
468 skyMap: `~lsst.skymap.BaseSkyMap`, optional
469 The skymap instance that defines any tract/patch data IDs.
470 `~lsst.skymap.BaseSkyMap` instances.
471 skyMapName : `str`, optional
472 Gen3 SkyMap Dimension name to be associated with any tract or patch
477 translator : `Translator`
478 A translator whose translate() method can be used to transform Gen2
479 data IDs to Gen3 dataIds.
481 if instrument
is not None:
482 rulesForInstrument = self._rules.get(instrument, {
None: []})
484 rulesForInstrument = {
None: []}
485 rulesForAnyInstrument = self._rules[
None]
486 candidateRules = itertools.chain(
487 rulesForInstrument.get(datasetTypeName, []),
488 rulesForInstrument[
None],
489 rulesForAnyInstrument.get(datasetTypeName, []),
490 rulesForAnyInstrument[
None],
493 targetKeys = set(gen2keys)
494 for ruleKeys, ruleHandlers, consume
in candidateRules:
495 if ruleKeys.issubset(targetKeys):
496 matchedHandlers.append(ruleHandlers)
497 targetKeys -= consume
498 return Translator(matchedHandlers, skyMap=skyMap, skyMapName=skyMapName,
499 datasetTypeName=datasetTypeName)
503 """Callable object that translates Gen2 Data IDs to Gen3 Data IDs for a
504 particular DatasetType.
506 Translators should usually be constructed via
507 `TranslatorFactory.makeMatching`.
512 A list of KeyHandlers this Translator should use.
513 skyMap : `BaseSkyMap`, optional
514 SkyMap instance used to define any tract or patch Dimensions.
516 Gen3 SkyMap Dimension name to be associated with any tract or patch
518 datasetTypeName : `str`
519 Name of the dataset type whose data IDs this translator handles.
521 def __init__(self, handlers: List[KeyHandler], skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
522 datasetTypeName: str):
528 __slots__ = (
"handlers",
"skyMap",
"skyMapName",
"datasetTypeName")
531 hstr =
",".join(str(h)
for h
in self.
handlers)
532 return f
"{type(self).__name__}(dtype={self.datasetTypeName}, handlers=[{hstr}])"
534 def __call__(self, gen2id: Dict[str, Any], *, partial: bool =
False, log: Optional[Log] =
None):
535 """Return a Gen3 data ID that corresponds to the given Gen2 data ID.
540 handler.translate(gen2id, gen3id, skyMap=self.
skyMap, skyMapName=self.
skyMapName,
545 log.debug(
"Failed to translate %s from %s.", handler.dimension, gen2id)
553 """The names of the dimensions populated by this Translator
556 return frozenset(h.dimension
for h
in self.
handlers)