lsst.obs.base  19.0.0-51-gb87bce2+1
translators.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 __all__ = ("Translator", "KeyHandler", "CopyKeyHandler", "ConstantKeyHandler",
23  "CalibKeyHandler", "AbstractToPhysicalFilterKeyHandler", "PhysicalToAbstractFilterKeyHandler",
24  "makeCalibrationLabel")
25 
26 import itertools
27 from typing import Optional, Any, Dict, Tuple, FrozenSet, Iterable, List
28 from abc import ABCMeta, abstractmethod
29 
30 from lsst.log import Log
31 from lsst.skymap import BaseSkyMap
32 
33 
34 def makeCalibrationLabel(datasetTypeName: str, calibDate: str, ccd: Optional[int] = None,
35  filter: Optional[str] = None) -> str:
36  """Make a Gen3 calibration_label string corresponding to a Gen2 data ID.
37 
38  Parameters
39  ----------
40  datasetTypeName : `str`
41  Name of the dataset type this calibration label identifies.
42  calibDate : `str`
43  Date string used in the Gen2 template.
44  ccd : `int`, optional
45  Detector ID used in the Gen2 template.
46  filter : `str`, optional
47  Filter used in the Gen2 template.
48 
49  Returns
50  -------
51  label : `str`
52  Calibration label string.
53  """
54  # TODO: this function is probably HSC-specific, but I don't know how other
55  # obs calib registries behave so I don't know (yet) how to generalize it.
56  elements = [datasetTypeName, calibDate]
57  if ccd is not None:
58  elements.append(f"{ccd:03d}")
59  if filter is not None:
60  elements.append(filter)
61  return "gen2/{}".format("_".join(elements))
62 
63 
64 class KeyHandler(metaclass=ABCMeta):
65  """Base class for Translator helpers that each handle just one Gen3 Data
66  ID key.
67 
68  Parameters
69  ----------
70  dimension : `str`
71  Name of the Gen3 dimension (data ID key) populated by
72  this handler (e.g. "visit" or "abstract_filter").
73  """
74  def __init__(self, dimension: str):
75  self.dimension = dimension
76 
77  __slots__ = ("dimension",)
78 
79  def __str__(self):
80  return f"{type(self).__name__}({self.dimension})"
81 
82  def translate(self, gen2id: dict, gen3id: dict,
83  skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
84  datasetTypeName: str):
85  """Update a Gen3 data ID dict with a single key-value pair from a Gen2
86  data ID.
87 
88  This method is implemented by the base class and is not expected to
89  be re-implemented by subclasses.
90 
91  Parameters
92  ----------
93  gen2id: `dict`
94  Gen2 data ID from which to draw key-value pairs from.
95  gen3id: `dict`
96  Gen3 data ID to update in-place.
97  skyMap: `BaseSkyMap`, optional
98  SkyMap that defines the tracts and patches used in the Gen2 data
99  ID, if any.
100  skyMapName: `str`
101  Name of the Gen3 skymap dimension that defines the tracts and
102  patches used in the Gen3 data ID.
103  datasetTypeName: `str`
104  Name of the dataset type.
105  """
106  gen3id[self.dimension] = self.extract(gen2id, skyMap=skyMap, skyMapName=skyMapName,
107  datasetTypeName=datasetTypeName)
108 
109  @abstractmethod
110  def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
111  datasetTypeName: str) -> Any:
112  """Extract a Gen3 data ID value from a Gen2 data ID.
113 
114  Parameters
115  ----------
116  gen2id: `dict`
117  Gen2 data ID from which to draw key-value pairs from.
118  skyMap: `BaseSkyMap`, optional
119  SkyMap that defines the tracts and patches used in the Gen2 data
120  ID, if any.
121  skyMapName: `str`
122  Name of the Gen3 skymap dimension that defines the tracts and
123  patches used in the Gen3 data ID.
124  datasetTypeName: `str`
125  Name of the dataset type.
126  """
127  raise NotImplementedError()
128 
129 
131  """A KeyHandler that adds a constant key-value pair to the Gen3 data ID.
132 
133  Parameters
134  ----------
135  dimension : `str`
136  Name of the Gen3 dimension (data ID key) populated by
137  this handler (e.g. "visit" or "abstract_filter").
138  value : `object`
139  Data ID value.
140  """
141  def __init__(self, dimension: str, value: Any):
142  super().__init__(dimension)
143  self.value = value
144 
145  __slots__ = ("value",)
146 
147  def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
148  datasetTypeName: str) -> Any:
149  # Docstring inherited from KeyHandler.extract.
150  return self.value
151 
152 
154  """A KeyHandler that simply copies a value from a Gen3 data ID.
155 
156  Parameters
157  ----------
158  dimension : `str`
159  Name of the Gen3 dimension produced by this handler.
160  dtype : `type`, optional
161  If not `None`, the type that values for this key must be an
162  instance of.
163  """
164  def __init__(self, dimension: str, gen2key: Optional[str] = None,
165  dtype: Optional[type] = None):
166  super().__init__(dimension)
167  self.gen2key = gen2key if gen2key is not None else dimension
168  self.dtype = dtype
169 
170  __slots__ = ("gen2key", "dtype")
171 
172  def __str__(self):
173  return f"{type(self).__name__}({self.gen2key}, {self.dtype})"
174 
175  def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
176  datasetTypeName: str) -> Any:
177  # Docstring inherited from KeyHandler.extract.
178  r = gen2id[self.gen2key]
179  if self.dtype is not None:
180  try:
181  r = self.dtype(r)
182  except ValueError as err:
183  raise TypeError(
184  f"'{r}' is not a valid value for {self.dimension}; "
185  f"expected {self.dtype.__name__}, got {type(r).__name__}."
186  ) from err
187  return r
188 
189 
191  """A KeyHandler for skymap patches.
192  """
193  def __init__(self):
194  super().__init__("patch")
195 
196  __slots__ = ()
197 
198  def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
199  datasetTypeName: str) -> Any:
200  # Docstring inherited from KeyHandler.extract.
201  tract = gen2id["tract"]
202  tractInfo = skyMap[tract]
203  x, y = gen2id["patch"].split(",")
204  patchInfo = tractInfo[int(x), int(y)]
205  return tractInfo.getSequentialPatchIndex(patchInfo)
206 
207 
209  """A KeyHandler for skymaps."""
210  def __init__(self):
211  super().__init__("skymap")
212 
213  __slots__ = ()
214 
215  def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
216  datasetTypeName: str) -> Any:
217  # Docstring inherited from KeyHandler.extract.
218  return skyMapName
219 
220 
222  """A KeyHandler for master calibration datasets.
223  """
224  __slots__ = ("ccdKey",)
225 
226  def __init__(self, ccdKey="ccd"):
227  self.ccdKey = ccdKey
228  super().__init__("calibration_label")
229 
230  def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
231  datasetTypeName: str) -> Any:
232  # Docstring inherited from KeyHandler.extract.
233  return makeCalibrationLabel(datasetTypeName, gen2id["calibDate"],
234  ccd=gen2id.get(self.ccdKey), filter=gen2id.get("filter"))
235 
236 
238  """KeyHandler for gen2 ``filter`` keys that match ``physical_filter``
239  keys in gen3 but should be mapped to ``abstract_filter``.
240 
241  Note that multiple physical filter can potentially map to one abstract
242  filter, so be careful to only use this translator on obs packages where
243  there is a one-to-one mapping.
244  """
245 
246  __slots__ = ("_map",)
247 
248  def __init__(self, filterDefinitions):
249  super().__init__("abstract_filter")
250  self._map = {d.physical_filter: d.abstract_filter for d in filterDefinitions
251  if d.physical_filter is not None}
252 
253  def extract(self, gen2id, *args, **kwargs):
254  physical = gen2id["filter"]
255  return self._map.get(physical, physical)
256 
257 
259  """KeyHandler for gen2 ``filter`` keys that match ``abstract_filter``
260  keys in gen3 but should be mapped to ``physical_filter``.
261 
262  Note that one abstract filter can potentially map to multiple physical
263  filters, so be careful to only use this translator on obs packages where
264  there is a one-to-one mapping.
265  """
266 
267  __slots__ = ("_map",)
268 
269  def __init__(self, filterDefinitions):
270  super().__init__("physical_filter")
271  self._map = {d.abstract_filter: d.physical_filter for d in filterDefinitions
272  if d.abstract_filter is not None}
273 
274  def extract(self, gen2id, *args, **kwargs):
275  abstract = gen2id["filter"]
276  return self._map.get(abstract, abstract)
277 
278 
280  """Callable object that translates Gen2 Data IDs to Gen3 Data IDs for a
281  particular DatasetType.
282 
283  Translators should usually be constructed via the `makeMatching` method.
284 
285  Parameters
286  ----------
287  handlers : `list`
288  A list of KeyHandlers this Translator should use.
289  skyMap : `BaseSkyMap`, optional
290  SkyMap instance used to define any tract or patch Dimensions.
291  skyMapName : `str`
292  Gen3 SkyMap Dimension name to be associated with any tract or patch
293  Dimensions.
294  datasetTypeName : `str`
295  Name of the dataset type whose data IDs this translator handles.
296  """
297  def __init__(self, handlers: List[KeyHandler], skyMap: Optional[BaseSkyMap], skyMapName: Optional[str],
298  datasetTypeName: str):
299  self.handlers = handlers
300  self.skyMap = skyMap
301  self.skyMapName = skyMapName
302  self.datasetTypeName = datasetTypeName
303 
304  __slots__ = ("handlers", "skyMap", "skyMapName", "datasetTypeName")
305 
306  # Rules used to match Handlers when constring a Translator.
307  # outer key is instrument name, or None for any
308  # inner key is DatasetType name, or None for any
309  # values are 3-tuples of (frozenset(gen2keys), handler, consume)
310  _rules: Dict[
311  Optional[str],
312  Dict[
313  Optional[str],
314  Tuple[FrozenSet[str], KeyHandler, bool]
315  ]
316  ] = {
317  None: {
318  None: []
319  }
320  }
321 
322  def __str__(self):
323  hstr = ",".join(str(h) for h in self.handlers)
324  return f"{type(self).__name__}(dtype={self.datasetTypeName}, handlers=[{hstr}])"
325 
326  @classmethod
327  def addRule(cls, handler: KeyHandler, instrument: Optional[str] = None,
328  datasetTypeName: Optional[str] = None, gen2keys: Iterable[str] = (),
329  consume: bool = True):
330  """Add a KeyHandler and an associated matching rule.
331 
332  Parameters
333  ----------
334  handler : `KeyHandler`
335  A KeyHandler instance to add to a Translator when this rule
336  matches.
337  instrument : `str`
338  Gen3 instrument name the Gen2 repository must be associated with
339  for this rule to match, or None to match any instrument.
340  datasetTypeName : `str`
341  Name of the DatasetType this rule matches, or None to match any
342  DatasetType.
343  gen2Keys : sequence
344  Sequence of Gen2 data ID keys that must all be present for this
345  rule to match.
346  consume : `bool` or `tuple`
347  If True (default), remove all entries in gen2keys from the set of
348  keys being matched to in order to prevent less-specific handlers
349  from matching them.
350  May also be a `tuple` listing only the keys to consume.
351  """
352  # Ensure consume is always a frozenset, so we can process it uniformly
353  # from here on.
354  if consume is True:
355  consume = frozenset(gen2keys)
356  elif consume:
357  consume = frozenset(consume)
358  else:
359  consume = frozenset()
360  # find the rules for this instrument, or if we haven't seen it before,
361  # add a nested dictionary that matches any DatasetType name and then
362  # append this rule.
363  rulesForInstrument = cls._rules.setdefault(instrument, {None: []})
364  rulesForInstrumentAndDatasetType = rulesForInstrument.setdefault(datasetTypeName, [])
365  rulesForInstrumentAndDatasetType.append((frozenset(gen2keys), handler, consume))
366 
367  @classmethod
368  def makeMatching(cls, datasetTypeName: str, gen2keys: Dict[str, type], instrument: Optional[str] = None,
369  skyMap: Optional[BaseSkyMap] = None, skyMapName: Optional[str] = None):
370  """Construct a Translator appropriate for instances of the given
371  dataset.
372 
373  Parameters
374  ----------
375  datasetTypeName : `str`
376  Name of the dataset type.
377  gen2keys: `dict`
378  Keys of a Gen2 data ID for this dataset.
379  instrument: `str`, optional
380  Name of the Gen3 instrument dimension for translated data IDs.
381  skyMap: `~lsst.skymap.BaseSkyMap`, optional
382  The skymap instance that defines any tract/patch data IDs.
383  `~lsst.skymap.BaseSkyMap` instances.
384  skyMapName : `str`, optional
385  Gen3 SkyMap Dimension name to be associated with any tract or patch
386  Dimensions.
387 
388  Returns
389  -------
390  translator : `Translator`
391  A translator whose translate() method can be used to transform Gen2
392  data IDs to Gen3 dataIds.
393  """
394  if instrument is not None:
395  rulesForInstrument = cls._rules.get(instrument, {None: []})
396  else:
397  rulesForInstrument = {None: []}
398  rulesForAnyInstrument = cls._rules[None]
399  candidateRules = itertools.chain(
400  rulesForInstrument.get(datasetTypeName, []), # this instrument, this DatasetType
401  rulesForInstrument[None], # this instrument, any DatasetType
402  rulesForAnyInstrument.get(datasetTypeName, []), # any instrument, this DatasetType
403  rulesForAnyInstrument[None], # any instrument, any DatasetType
404  )
405  matchedHandlers = []
406  targetKeys = set(gen2keys)
407  for ruleKeys, ruleHandlers, consume in candidateRules:
408  if ruleKeys.issubset(targetKeys):
409  matchedHandlers.append(ruleHandlers)
410  targetKeys -= consume
411  return Translator(matchedHandlers, skyMap=skyMap, skyMapName=skyMapName,
412  datasetTypeName=datasetTypeName)
413 
414  def __call__(self, gen2id: Dict[str, Any], *, partial: bool = False, log: Optional[Log] = None):
415  """Return a Gen3 data ID that corresponds to the given Gen2 data ID.
416  """
417  gen3id = {}
418  for handler in self.handlers:
419  try:
420  handler.translate(gen2id, gen3id, skyMap=self.skyMap, skyMapName=self.skyMapName,
421  datasetTypeName=self.datasetTypeName)
422  except KeyError:
423  if partial:
424  if log is not None:
425  log.debug("Failed to translate %s from %s.", handler.dimension, gen2id)
426  continue
427  else:
428  raise
429  return gen3id
430 
431  @property
432  def dimensionNames(self):
433  """The names of the dimensions populated by this Translator
434  (`frozenset`).
435  """
436  return frozenset(h.dimension for h in self.handlers)
437 
438 
439 # Add "skymap" to Gen3 ID if Gen2 ID has a "tract" key.
440 Translator.addRule(SkyMapKeyHandler(), gen2keys=("tract",), consume=False)
441 
442 # Add "skymap" to Gen3 ID if DatasetType is one of a few specific ones
443 for coaddName in ("deep", "goodSeeing", "psfMatched", "dcr"):
444  Translator.addRule(SkyMapKeyHandler(), datasetTypeName=f"{coaddName}Coadd_skyMap")
445 
446 # Translate Gen2 str patch IDs to Gen3 sequential integers.
447 Translator.addRule(PatchKeyHandler(), gen2keys=("patch",))
448 
449 # Copy Gen2 "tract" to Gen3 "tract".
450 Translator.addRule(CopyKeyHandler("tract", dtype=int), gen2keys=("tract",))
451 
452 # Add valid_first, valid_last to instrument-level transmission/ datasets;
453 # these are considered calibration products in Gen3.
454 for datasetTypeName in ("transmission_sensor", "transmission_optics", "transmission_filter"):
455  Translator.addRule(ConstantKeyHandler("calibration_label", "unbounded"),
456  datasetTypeName=datasetTypeName)
457 
458 # Translate Gen2 pixel_id to Gen3 skypix.
459 # TODO: For now, we just assume that the refcat indexer uses htm7, since that's
460 # what we have generated most of our refcats at.
461 Translator.addRule(CopyKeyHandler("htm7", gen2key="pixel_id", dtype=int), gen2keys=("pixel_id",))
lsst.obs.base.gen2to3.translators.Translator.addRule
def addRule(cls, KeyHandler handler, Optional[str] instrument=None, Optional[str] datasetTypeName=None, Iterable[str] gen2keys=(), bool consume=True)
Definition: translators.py:327
lsst.obs.base.gen2to3.translators.KeyHandler.translate
def translate(self, dict gen2id, dict gen3id, Optional[BaseSkyMap] skyMap, Optional[str] skyMapName, str datasetTypeName)
Definition: translators.py:82
lsst.obs.base.gen2to3.translators.CalibKeyHandler.__init__
def __init__(self, ccdKey="ccd")
Definition: translators.py:226
lsst.obs.base.gen2to3.translators.SkyMapKeyHandler
Definition: translators.py:208
lsst.obs.base.gen2to3.translators.CalibKeyHandler.extract
Any extract(self, dict gen2id, Optional[BaseSkyMap] skyMap, Optional[str] skyMapName, str datasetTypeName)
Definition: translators.py:230
lsst.obs.base.gen2to3.translators.Translator.skyMap
skyMap
Definition: translators.py:299
lsst.obs.base.gen2to3.translators.SkyMapKeyHandler.__init__
def __init__(self)
Definition: translators.py:210
lsst.obs.base.gen2to3.translators.CopyKeyHandler.__str__
def __str__(self)
Definition: translators.py:172
lsst.obs.base.gen2to3.translators.KeyHandler
Definition: translators.py:64
lsst.obs.base.gen2to3.translators.CopyKeyHandler.gen2key
gen2key
Definition: translators.py:166
lsst.obs.base.gen2to3.translators.PhysicalToAbstractFilterKeyHandler
Definition: translators.py:237
lsst.obs.base.gen2to3.translators.CopyKeyHandler
Definition: translators.py:153
lsst.obs.base.gen2to3.translators.Translator.makeMatching
def makeMatching(cls, str datasetTypeName, Dict[str, type] gen2keys, Optional[str] instrument=None, Optional[BaseSkyMap] skyMap=None, Optional[str] skyMapName=None)
Definition: translators.py:368
lsst.obs.base.gen2to3.translators.PhysicalToAbstractFilterKeyHandler.extract
def extract(self, gen2id, *args, **kwargs)
Definition: translators.py:253
lsst.obs.base.gen2to3.translators.makeCalibrationLabel
str makeCalibrationLabel(str datasetTypeName, str calibDate, Optional[int] ccd=None, Optional[str] filter=None)
Definition: translators.py:34
lsst.obs.base.gen2to3.translators.CalibKeyHandler.ccdKey
ccdKey
Definition: translators.py:227
lsst.obs.base.gen2to3.translators.ConstantKeyHandler.extract
Any extract(self, dict gen2id, Optional[BaseSkyMap] skyMap, Optional[str] skyMapName, str datasetTypeName)
Definition: translators.py:147
lsst.obs.base.gen2to3.translators.Translator.handlers
handlers
Definition: translators.py:298
lsst.obs.base.gen2to3.translators.PatchKeyHandler.extract
Any extract(self, dict gen2id, Optional[BaseSkyMap] skyMap, Optional[str] skyMapName, str datasetTypeName)
Definition: translators.py:198
lsst.obs.base.gen2to3.translators.KeyHandler.__init__
def __init__(self, str dimension)
Definition: translators.py:74
lsst.obs.base.gen2to3.translators.PatchKeyHandler.__init__
def __init__(self)
Definition: translators.py:193
lsst.obs.base.gen2to3.translators.KeyHandler.extract
Any extract(self, dict gen2id, Optional[BaseSkyMap] skyMap, Optional[str] skyMapName, str datasetTypeName)
Definition: translators.py:110
lsst.obs.base.gen2to3.translators.ConstantKeyHandler
Definition: translators.py:130
lsst.obs.base.gen2to3.translators.ConstantKeyHandler.value
value
Definition: translators.py:143
lsst.obs.base.gen2to3.translators.Translator.__str__
def __str__(self)
Definition: translators.py:322
lsst.obs.base.gen2to3.translators.AbstractToPhysicalFilterKeyHandler.__init__
def __init__(self, filterDefinitions)
Definition: translators.py:269
lsst.obs.base.gen2to3.translators.Translator
Definition: translators.py:279
lsst.obs.base.gen2to3.translators.CopyKeyHandler.dtype
dtype
Definition: translators.py:167
lsst.obs.base.gen2to3.translators.Translator.skyMapName
skyMapName
Definition: translators.py:300
lsst.obs.base.gen2to3.translators.KeyHandler.__str__
def __str__(self)
Definition: translators.py:79
lsst.obs.base.gen2to3.translators.AbstractToPhysicalFilterKeyHandler._map
_map
Definition: translators.py:271
lsst.obs.base.gen2to3.translators.KeyHandler.dimension
dimension
Definition: translators.py:75
lsst.obs.base.gen2to3.translators.CopyKeyHandler.extract
Any extract(self, dict gen2id, Optional[BaseSkyMap] skyMap, Optional[str] skyMapName, str datasetTypeName)
Definition: translators.py:175
lsst.obs.base.gen2to3.translators.ConstantKeyHandler.__init__
def __init__(self, str dimension, Any value)
Definition: translators.py:141
lsst.obs.base.gen2to3.translators.CopyKeyHandler.__init__
def __init__(self, str dimension, Optional[str] gen2key=None, Optional[type] dtype=None)
Definition: translators.py:164
lsst.obs.base.gen2to3.translators.Translator.__init__
def __init__(self, List[KeyHandler] handlers, Optional[BaseSkyMap] skyMap, Optional[str] skyMapName, str datasetTypeName)
Definition: translators.py:297
lsst.obs.base.gen2to3.translators.Translator.__call__
def __call__(self, Dict[str, Any] gen2id, *bool partial=False, Optional[Log] log=None)
Definition: translators.py:414
lsst.obs.base.gen2to3.translators.PhysicalToAbstractFilterKeyHandler.__init__
def __init__(self, filterDefinitions)
Definition: translators.py:248
lsst.obs.base.gen2to3.translators.SkyMapKeyHandler.extract
Any extract(self, dict gen2id, Optional[BaseSkyMap] skyMap, Optional[str] skyMapName, str datasetTypeName)
Definition: translators.py:215
lsst.obs.base.gen2to3.translators.PatchKeyHandler
Definition: translators.py:190
lsst.obs.base.gen2to3.translators.Translator.dimensionNames
def dimensionNames(self)
Definition: translators.py:432
lsst.obs.base.gen2to3.translators.AbstractToPhysicalFilterKeyHandler
Definition: translators.py:258
lsst.obs.base.gen2to3.translators.AbstractToPhysicalFilterKeyHandler.extract
def extract(self, gen2id, *args, **kwargs)
Definition: translators.py:274
lsst.obs.base.gen2to3.translators.PhysicalToAbstractFilterKeyHandler._map
_map
Definition: translators.py:250
lsst.obs.base.gen2to3.translators.CalibKeyHandler
Definition: translators.py:221
lsst.obs.base.gen2to3.translators.Translator.datasetTypeName
datasetTypeName
Definition: translators.py:301