lsst.obs.base  20.0.0-60-gad89340+e4dd172200
standardRepoConverter.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 from __future__ import annotations
22 
23 __all__ = ["StandardRepoConverter"]
24 
25 import os.path
26 from dataclasses import dataclass
27 from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Set, Tuple
28 
29 from lsst.log import Log
30 from lsst.log.utils import temporaryLogLevel
31 from lsst.daf.persistence import Butler as Butler2
32 from lsst.daf.butler import DatasetType, DatasetRef, DataCoordinate, FileDataset
33 from .repoConverter import RepoConverter
34 from .repoWalker import RepoWalker
35 
36 SKYMAP_DATASET_TYPES = {
37  coaddName: f"{coaddName}Coadd_skyMap" for coaddName in ("deep", "goodSeeing", "dcr")
38 }
39 
40 if TYPE_CHECKING:
41  from lsst.skymap import BaseSkyMap
42  from lsst.daf.butler import StorageClass, FormatterParameter
43  from .cameraMapper import CameraMapper
44  from .repoWalker.scanner import PathElementHandler
45  from ..mapping import Mapping as CameraMapperMapping # disambiguate from collections.abc.Mapping
46 
47 
48 @dataclass
50  """Struct containing information about a SkyMap found in a Gen2 repository.
51  """
52 
53  name: str
54  """Name of the skymap used in Gen3 data IDs.
55  """
56 
57  instance: BaseSkyMap
58  """An instance of the actual skymap class.
59  """
60 
61  coaddName: str
62  """The coadd name used as a prefix for the dataset type this skymap was
63  found in.
64  """
65 
66  ref: DatasetRef
67  """A `DatasetRef` that can be used to ingest the skymap dataset into a
68  Gen3 repository.
69  """
70 
71  filename: str
72  """Name of the file containing the skymap dataset, relative to the
73  repository root.
74  """
75 
76 
78  """A specialization of `RepoConverter` for non-calibration repositories.
79 
80  Parameters
81  ----------
82  kwds
83  Keyword arguments are forwarded to (and required by) `RepoConverter`.
84  """
85 
86  def __init__(self, **kwds):
87  super().__init__(**kwds)
88  # Shush noisy log messages from Gen2 Mapper classes.
89  with temporaryLogLevel("CameraMapper", Log.ERROR):
90  with temporaryLogLevel("HscMapper", Log.ERROR):
91  self.butler2 = Butler2(self.root)
92  self.mapper = self.butler2.getMapperClass(self.root)(root=self.root)
93  self._foundSkyMapsByCoaddName = {}
94  self._chain = {}
95 
96  def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool:
97  # Docstring inherited from RepoConverter.
98  return datasetTypeName in SKYMAP_DATASET_TYPES.values()
99 
100  def prep(self):
101  # Docstring inherited from RepoConverter.
102  self.task.log.info(f"Looking for skymaps in root {self.root}.")
103  for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items():
104  if not self.task.isDatasetTypeIncluded(datasetTypeName):
105  continue
106  try:
107  exists = self.butler2.datasetExists(datasetTypeName)
108  except AttributeError:
109  # This mapper doesn't even define this dataset type.
110  continue
111  if not exists:
112  continue
113  instance = self.butler2.get(datasetTypeName)
114  name = self.task.useSkyMap(instance, datasetTypeName)
115  datasetType = DatasetType(datasetTypeName, dimensions=["skymap"],
116  storageClass="SkyMap", universe=self.task.universe)
117  dataId = DataCoordinate.standardize(skymap=name, universe=self.task.universe)
118  struct = FoundSkyMap(name=name, instance=instance, coaddName=coaddName,
119  ref=DatasetRef(datasetType, dataId),
120  filename=self.butler2.getUri(datasetTypeName))
121  self._foundSkyMapsByCoaddName[coaddName] = struct
122  self.task.log.info("Found skymap %s in %s in %s.", name, datasetTypeName, self.root)
123  super().prep()
124 
125  def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
126  # Docstring inherited from RepoConverter.
127  for datasetTypeName, mapping in self.mapper.mappings.items():
128  if datasetTypeName not in self.mapper.calibrations:
129  yield datasetTypeName, mapping
130 
131  def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]:
132  """Return the appropriate SkyMap for the given dataset type.
133 
134  Parameters
135  ----------
136  datasetTypeName : `str`
137  Name of the dataset type for which a skymap is sought.
138 
139  Returns
140  -------
141  skyMap : `BaseSkyMap` or `None`
142  The `BaseSkyMap` instance, or `None` if there was no match.
143  skyMapName : `str` or `None`
144  The Gen3 name for the SkyMap, or `None` if there was no match.
145  """
146  # Use deepCoadd_skyMap by default; there are some dataset types
147  # that use it but don't have "deep" anywhere in their name.
148  struct = self._foundSkyMapsByCoaddName.get("deep")
149  for coaddName in SKYMAP_DATASET_TYPES.keys():
150  if coaddName in datasetTypeName:
151  try:
152  struct = self._foundSkyMapsByCoaddName[coaddName]
153  break
154  except KeyError:
155  # Don't use the default, since we did find a specific
156  # coaddName.
157  struct = None
158  self.task.log.debug(
159  ("Dataset %s looks like it might need a skymap, but no %sCoadd_skyMap "
160  "found in repo %s."),
161  datasetTypeName, coaddName, self.root
162  )
163  if struct is not None:
164  return struct.instance, struct.name
165  else:
166  return None, None
167 
168  def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[str, type],
169  storageClass: StorageClass,
170  formatter: FormatterParameter = None,
171  targetHandler: Optional[PathElementHandler] = None,
172  ) -> RepoWalker.Target:
173  # Docstring inherited from RepoConverter.
174  skyMap, skyMapName = self.findMatchingSkyMap(datasetTypeName)
175  return RepoWalker.Target(
176  datasetTypeName=datasetTypeName,
177  storageClass=storageClass,
178  template=template,
179  keys=keys,
180  universe=self.task.registry.dimensions,
181  instrument=self.task.instrument.getName(),
182  skyMap=skyMap,
183  skyMapName=skyMapName,
184  formatter=formatter,
185  targetHandler=targetHandler,
186  translatorFactory=self.task.translatorFactory,
187  )
188 
189  def iterDatasets(self) -> Iterator[FileDataset]:
190  # Docstring inherited from RepoConverter.
191  for struct in self._foundSkyMapsByCoaddName.values():
192  if self.task.isDatasetTypeIncluded(struct.ref.datasetType.name):
193  yield FileDataset(path=os.path.join(self.root, struct.filename), refs=struct.ref)
194  yield from super().iterDatasets()
195 
196  def getRun(self, datasetTypeName: str, calibDate: Optional[str] = None) -> str:
197  # Docstring inherited from RepoConverter.
198  run = self.task.config.runs.get(datasetTypeName)
199  if run is not None:
200  self._chain.setdefault(run, set()).add(datasetTypeName)
201  elif self._run is None:
202  raise ValueError(f"No default run for repo at {self.root}, and no "
203  f"override for dataset {datasetTypeName}.")
204  else:
205  run = self._run
206  return run
207 
208  def getCollectionChain(self) -> List[Tuple[str, Set[str]]]:
209  """Return tuples of run name and associated dataset type names that
210  can be used to construct a chained collection that refers to the
211  converted repository (`list` [ `tuple` ]).
212  """
213  return list(self._chain.items())
214 
215  # Class attributes that will be shadowed by public instance attributes;
216  # defined here only for documentation purposes.
217 
218  butler2: Butler2
219  """Gen2 butler associated with this repository.
220  """
221 
222  mapper: CameraMapper
223  """Gen2 mapper associated with this repository.
224  """
lsst.obs.base.gen2to3.repoConverter.RepoConverter.root
root
Definition: repoConverter.py:211
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.__init__
def __init__(self, **kwds)
Definition: standardRepoConverter.py:86
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.iterDatasets
Iterator[FileDataset] iterDatasets(self)
Definition: standardRepoConverter.py:189
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.getRun
str getRun(self, str datasetTypeName, Optional[str] calibDate=None)
Definition: standardRepoConverter.py:196
lsst.obs.base.gen2to3.standardRepoConverter.FoundSkyMap
Definition: standardRepoConverter.py:49
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter._foundSkyMapsByCoaddName
_foundSkyMapsByCoaddName
Definition: standardRepoConverter.py:93
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.iterMappings
Iterator[Tuple[str, CameraMapperMapping]] iterMappings(self)
Definition: standardRepoConverter.py:125
lsst.obs.base.gen2to3.repoConverter.RepoConverter.task
task
Definition: repoConverter.py:210
lsst.obs.base.gen2to3.repoConverter.RepoConverter._run
_run
Definition: repoConverter.py:214
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.findMatchingSkyMap
Tuple[Optional[BaseSkyMap], Optional[str]] findMatchingSkyMap(self, str datasetTypeName)
Definition: standardRepoConverter.py:131
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter._chain
_chain
Definition: standardRepoConverter.py:94
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.getCollectionChain
List[Tuple[str, Set[str]]] getCollectionChain(self)
Definition: standardRepoConverter.py:208
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.isDatasetTypeSpecial
bool isDatasetTypeSpecial(self, str datasetTypeName)
Definition: standardRepoConverter.py:96
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.makeRepoWalkerTarget
RepoWalker.Target makeRepoWalkerTarget(self, str datasetTypeName, str template, Dict[str, type] keys, StorageClass storageClass, FormatterParameter formatter=None, Optional[PathElementHandler] targetHandler=None)
Definition: standardRepoConverter.py:168
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter
Definition: standardRepoConverter.py:77
lsst.obs.base.gen2to3.repoConverter.RepoConverter
Definition: repoConverter.py:180
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.mapper
mapper
Definition: standardRepoConverter.py:92
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.prep
def prep(self)
Definition: standardRepoConverter.py:100
lsst.obs.base.gen2to3.standardRepoConverter.StandardRepoConverter.butler2
butler2
Definition: standardRepoConverter.py:91