lsst.obs.base  19.0.0-21-gaaa92db+6
standardRepoConverter.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 from __future__ import annotations
22 
23 __all__ = ["StandardRepoConverter"]
24 
25 import os.path
26 from dataclasses import dataclass
27 from typing import TYPE_CHECKING, Dict, Iterator, Optional, Tuple
28 
29 from lsst.log import Log
30 from lsst.log.utils import temporaryLogLevel
31 from lsst.daf.persistence import Butler as Butler2
32 from lsst.daf.butler import DatasetType, DatasetRef, DataCoordinate, FileDataset
33 from .repoConverter import RepoConverter
34 from .repoWalker import RepoWalker
35 
36 SKYMAP_DATASET_TYPES = {
37  coaddName: f"{coaddName}Coadd_skyMap" for coaddName in ("deep", "goodSeeing", "dcr")
38 }
39 
40 if TYPE_CHECKING:
41  from lsst.skymap import BaseSkyMap
42  from lsst.daf.butler import StorageClass
43  from .cameraMapper import CameraMapper
44  from ..mapping import Mapping as CameraMapperMapping # disambiguate from collections.abc.Mapping
45 
46 
47 @dataclass
49  """Struct containing information about a SkyMap found in a Gen2 repository.
50  """
51 
52  name: str
53  """Name of the skymap used in Gen3 data IDs.
54  """
55 
56  instance: BaseSkyMap
57  """An instance of the actual skymap class.
58  """
59 
60  coaddName: str
61  """The coadd name used as a prefix for the dataset type this skymap was
62  found in.
63  """
64 
65  ref: DatasetRef
66  """A `DatasetRef` that can be used to ingest the skymap dataset into a
67  Gen3 repository.
68  """
69 
70  filename: str
71  """Name of the file containing the skymap dataset, relative to the
72  repository root.
73  """
74 
75 
77  """A specialization of `RepoConverter` for non-calibration repositories.
78 
79  Parameters
80  ----------
81  kwds
82  Keyword arguments are forwarded to (and required by) `RepoConverter`.
83  """
84 
85  def __init__(self, **kwds):
86  super().__init__(**kwds)
87  # Shush noisy log messages from Gen2 Mapper classes.
88  with temporaryLogLevel("CameraMapper", Log.ERROR):
89  with temporaryLogLevel("HscMapper", Log.ERROR):
90  self.butler2 = Butler2(self.root)
91  self.mapper = self.butler2.getMapperClass(self.root)(root=self.root)
92  self._foundSkyMapsByCoaddName = {}
93 
94  def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool:
95  # Docstring inherited from RepoConverter.
96  return datasetTypeName in SKYMAP_DATASET_TYPES.values()
97 
98  def prep(self):
99  # Docstring inherited from RepoConverter.
100  self.task.log.info(f"Looking for skymaps in root {self.root}.")
101  for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items():
102  if not self.task.isDatasetTypeIncluded(datasetTypeName):
103  continue
104  try:
105  exists = self.butler2.datasetExists(datasetTypeName)
106  except AttributeError:
107  # This mapper doesn't even define this dataset type.
108  continue
109  if not exists:
110  continue
111  instance = self.butler2.get(datasetTypeName)
112  name = self.task.useSkyMap(instance)
113  datasetType = DatasetType(datasetTypeName, dimensions=["skymap"],
114  storageClass="SkyMap", universe=self.task.universe)
115  dataId = DataCoordinate.standardize(skymap=name, universe=self.task.universe)
116  struct = FoundSkyMap(name=name, instance=instance, coaddName=coaddName,
117  ref=DatasetRef(datasetType, dataId),
118  filename=self.butler2.getUri(datasetTypeName))
119  self._foundSkyMapsByCoaddName[coaddName] = struct
120  self.task.log.info("Found skymap %s in %s in %s.", name, datasetTypeName, self.root)
121  super().prep()
122 
123  def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
124  # Docstring inherited from RepoConverter.
125  for datasetTypeName, mapping in self.mapper.mappings.items():
126  if datasetTypeName not in self.mapper.calibrations:
127  yield datasetTypeName, mapping
128 
129  def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]:
130  """Return the appropriate SkyMap for the given dataset type.
131 
132  Parameters
133  ----------
134  datasetTypeName : `str`
135  Name of the dataset type for which a skymap is sought.
136 
137  Returns
138  -------
139  skyMap : `BaseSkyMap` or `None`
140  The `BaseSkyMap` instance, or `None` if there was no match.
141  skyMapName : `str` or `None`
142  The Gen3 name for the SkyMap, or `None` if there was no match.
143  """
144  # Use deepCoadd_skyMap by default; there are some dataset types
145  # that use it but don't have "deep" anywhere in their name.
146  struct = self._foundSkyMapsByCoaddName.get("deep")
147  for coaddName in SKYMAP_DATASET_TYPES.keys():
148  if coaddName in datasetTypeName:
149  try:
150  struct = self._foundSkyMapsByCoaddName[coaddName]
151  break
152  except KeyError:
153  # Don't use the default, since we did find a specific
154  # coaddName.
155  struct = None
156  self.task.log.debug(
157  ("Dataset %s looks like it might need a skymap, but no %sCoadd_skyMap "
158  "found in repo %s."),
159  datasetTypeName, coaddName, self.root
160  )
161  if struct is not None:
162  return struct.instance, struct.name
163  else:
164  return None, None
165 
166  def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[str, type],
167  storageClass: StorageClass) -> RepoWalker.Target:
168  # Docstring inherited from RepoConverter.
169  skyMap, skyMapName = self.findMatchingSkyMap(datasetTypeName)
170  return RepoWalker.Target(
171  datasetTypeName=datasetTypeName,
172  storageClass=storageClass,
173  template=template,
174  keys=keys,
175  universe=self.task.registry.dimensions,
176  instrument=self.task.instrument.getName(),
177  skyMap=skyMap,
178  skyMapName=skyMapName,
179  )
180 
181  def iterDatasets(self) -> Iterator[FileDataset]:
182  # Docstring inherited from RepoConverter.
183  for struct in self._foundSkyMapsByCoaddName.values():
184  if self.task.isDatasetTypeIncluded(struct.ref.datasetType.name):
185  yield FileDataset(path=os.path.join(self.root, struct.filename), refs=struct.ref)
186  yield from super().iterDatasets()
187 
188  # Class attributes that will be shadowed by public instance attributes;
189  # defined here only for documentation purposes.
190 
191  butler2: Butler2
192  """Gen2 butler associated with this repository.
193  """
194 
195  mapper: CameraMapper
196  """Gen2 mapper associated with this repository.
197  """