lsst.obs.base  18.1.0-21-gde80ed3+6
standardRepoConverter.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 from __future__ import annotations
22 
23 __all__ = ["StandardRepoConverter"]
24 
25 from dataclasses import dataclass
26 from typing import TYPE_CHECKING, Iterator, Tuple
27 
28 from lsst.log import Log
29 from lsst.log.utils import temporaryLogLevel
30 from lsst.daf.persistence import Butler as Butler2
31 from lsst.daf.butler import DatasetType, DatasetRef, DataCoordinate
32 from .repoConverter import RepoConverter
33 from .filePathParser import FilePathParser
34 from .dataIdExtractor import DataIdExtractor
35 
36 SKYMAP_DATASET_TYPES = {
37  coaddName: f"{coaddName}Coadd_skyMap" for coaddName in ("deep", "goodSeeing", "dcr")
38 }
39 
40 if TYPE_CHECKING:
41  from lsst.skymap import BaseSkyMap
42  from lsst.daf.butler import StorageClass
43  from .cameraMapper import CameraMapper
44  from ..mapping import Mapping as CameraMapperMapping # disambiguate from collections.abc.Mapping
45 
46 
47 @dataclass
49  """Struct containing information about a SkyMap found in a Gen2 repository.
50  """
51 
52  name: str
53  """Name of the skymap used in Gen3 data IDs.
54  """
55 
56  instance: BaseSkyMap
57  """An instance of the actual skymap class.
58  """
59 
60  coaddName: str
61  """The coadd name used as a prefix for the dataset type this skymap was
62  found in.
63  """
64 
65  ref: DatasetRef
66  """A `DatasetRef` that can be used to ingest the skymap dataset into a
67  Gen3 repository.
68  """
69 
70  filename: str
71  """Name of the file containing the skymap dataset, relative to the
72  repository root.
73  """
74 
75 
77  """A specialization of `RepoConverter` for non-calibration repositories.
78 
79  Parameters
80  ----------
81  kwds
82  Keyword arguments are forwarded to (and required by) `RepoConverter`.
83  """
84 
85  def __init__(self, **kwds):
86  super().__init__(**kwds)
87  # Shush noisy log messages from Gen2 Mapper classes.
88  with temporaryLogLevel("CameraMapper", Log.ERROR):
89  with temporaryLogLevel("HscMapper", Log.ERROR):
90  self.butler2 = Butler2(self.root)
91  self.mapper = self.butler2.getMapperClass(self.root)(root=self.root)
92  self._foundSkyMapsByCoaddName = {}
93 
94  def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool:
95  # Docstring inherited from RepoConverter.
96  return datasetTypeName in SKYMAP_DATASET_TYPES.values()
97 
98  def isDirectorySpecial(self, subdirectory: str) -> bool:
99  # Docstring inherited from RepoConverter.
100  return False
101 
102  def prep(self):
103  # Docstring inherited from RepoConverter.
104  self.task.log.info(f"Looking for skymaps in root {self.root}.")
105  for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items():
106  if not self.task.isDatasetTypeIncluded(datasetTypeName):
107  continue
108  try:
109  exists = self.butler2.datasetExists(datasetTypeName)
110  except AttributeError:
111  # This mapper doesn't even define this dataset type.
112  continue
113  if not exists:
114  continue
115  instance = self.butler2.get(datasetTypeName)
116  name = self.task.useSkyMap(instance)
117  datasetType = DatasetType(datasetTypeName, dimensions=["skymap"],
118  storageClass="SkyMap", universe=self.task.universe)
119  dataId = DataCoordinate.standardize(skymap=name, universe=self.task.universe)
120  struct = FoundSkyMap(name=name, instance=instance, coaddName=coaddName,
121  ref=DatasetRef(datasetType, dataId),
122  filename=self.butler2.getUri(datasetTypeName))
123  self._foundSkyMapsByCoaddName[coaddName] = struct
124  self.task.log.info("Found skymap %s in %s in %s.", name, datasetTypeName, self.root)
125  super().prep()
126 
127  def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
128  # Docstring inherited from RepoConverter.
129  for datasetTypeName, mapping in self.mapper.mappings.items():
130  if datasetTypeName not in self.mapper.calibrations:
131  yield datasetTypeName, mapping
132 
133  def makeDataIdExtractor(self, datasetTypeName: str, parser: FilePathParser,
134  storageClass: StorageClass) -> DataIdExtractor:
135  # Docstring inherited from RepoConverter.
136  # Use deepCoadd_skyMap by default; there are some dataset types
137  # that use it but don't have "deep" anywhere in their name.
138  struct = self._foundSkyMapsByCoaddName.get("deep")
139  for coaddName in SKYMAP_DATASET_TYPES.keys():
140  if coaddName in datasetTypeName:
141  try:
142  struct = self._foundSkyMapsByCoaddName[coaddName]
143  break
144  except KeyError:
145  # Don't use the default, since we did find a specific
146  # coaddName.
147  struct = None
148  self.task.log.debug(
149  ("Dataset %s looks like it might need a skymap, but no %sCoadd_skyMap "
150  "found in repo %s."),
151  datasetTypeName, coaddName, self.root
152  )
153  return DataIdExtractor(
154  datasetTypeName,
155  storageClass,
156  filePathParser=parser,
157  universe=self.task.universe,
158  instrument=self.task.instrument.getName(),
159  skyMap=struct.instance if struct is not None else None,
160  skyMapName=struct.name if struct is not None else None,
161  )
162 
163  def iterDatasets(self) -> Iterator[Tuple[str, DatasetRef]]:
164  # Docstring inherited from RepoConverter.
165  for struct in self._foundSkyMapsByCoaddName.values():
166  if self.task.isDatasetTypeIncluded(struct.ref.datasetType.name):
167  yield struct.filename, struct.ref
168  yield from super().iterDatasets()
169 
170  # Class attributes that will be shadowed by public instance attributes;
171  # defined here only for documentation purposes.
172 
173  butler2: Butler2
174  """Gen2 butler associated with this repository.
175  """
176 
177  mapper: CameraMapper
178  """Gen2 mapper associated with this repository.
179  """