Coverage for python/lsst/obs/base/gen2to3/standardRepoConverter.py: 34%
102 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 11:05 +0000
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 11:05 +0000
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = ["StandardRepoConverter"]
25from dataclasses import dataclass
26from typing import TYPE_CHECKING, Dict, Iterator, List, Mapping, Optional, Tuple
28from lsst.daf.butler import DataCoordinate, DatasetRef, DatasetType, FileDataset
29from lsst.daf.persistence import Butler as Butler2
30from lsst.log import Log
31from lsst.log.utils import temporaryLogLevel
32from lsst.skymap import BaseSkyMap
34from .repoConverter import RepoConverter
35from .repoWalker import RepoWalker
37SKYMAP_DATASET_TYPES = {coaddName: f"{coaddName}Coadd_skyMap" for coaddName in ("deep", "goodSeeing", "dcr")}
39if TYPE_CHECKING: 39 ↛ 40line 39 didn't jump to line 40, because the condition on line 39 was never true
40 from lsst.daf.butler import FormatterParameter, StorageClass
42 from ..mapping import Mapping as CameraMapperMapping # disambiguate from collections.abc.Mapping
43 from .cameraMapper import CameraMapper
44 from .repoWalker.scanner import PathElementHandler
47@dataclass
48class FoundSkyMap:
49 """Struct containing information about a SkyMap in a Gen2 repository."""
51 name: str
52 """Name of the skymap used in Gen3 data IDs.
53 """
55 instance: BaseSkyMap
56 """An instance of the actual skymap class.
57 """
59 coaddName: str
60 """The coadd name used as a prefix for the dataset type this skymap was
61 found in.
62 """
64 ref: DatasetRef
65 """A `DatasetRef` that can be used to ingest the skymap dataset into a
66 Gen3 repository.
67 """
69 filename: str
70 """Name of the file containing the skymap dataset, relative to the
71 repository root.
72 """
75class StandardRepoConverter(RepoConverter):
76 """A specialization of `RepoConverter` for non-calibration repositories.
78 Parameters
79 ----------
80 kwds
81 Keyword arguments are forwarded to (and required by) `RepoConverter`.
82 """
84 def __init__(self, **kwds):
85 super().__init__(**kwds)
86 # Shush noisy log messages from Gen2 Mapper classes.
87 # These are currently lsst.log loggers.
88 with temporaryLogLevel("CameraMapper", Log.ERROR):
89 with temporaryLogLevel("HscMapper", Log.ERROR):
90 self.butler2 = Butler2(self.root)
91 self.mapper = self.butler2.getMapperClass(self.root)(root=self.root)
92 self._foundSkyMapsByCoaddName = {}
93 self._chain = []
95 def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool:
96 # Docstring inherited from RepoConverter.
97 return datasetTypeName in SKYMAP_DATASET_TYPES.values()
99 def prep(self):
100 # Docstring inherited from RepoConverter.
101 self.task.log.info("Looking for skymaps in root %s", self.root)
102 for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items():
103 if not self.task.isDatasetTypeIncluded(datasetTypeName):
104 continue
105 try:
106 exists = self.butler2.datasetExists(datasetTypeName)
107 except AttributeError:
108 # This mapper doesn't even define this dataset type.
109 continue
110 if not exists:
111 continue
112 instance = self.butler2.get(datasetTypeName)
113 name = self.task.useSkyMap(instance, datasetTypeName)
114 datasetType = DatasetType(
115 datasetTypeName, dimensions=["skymap"], storageClass="SkyMap", universe=self.task.universe
116 )
117 dataId = DataCoordinate.standardize(skymap=name, universe=self.task.universe)
118 struct = FoundSkyMap(
119 name=name,
120 instance=instance,
121 coaddName=coaddName,
122 ref=DatasetRef(datasetType, dataId),
123 filename=self.butler2.getUri(datasetTypeName),
124 )
125 self._foundSkyMapsByCoaddName[coaddName] = struct
126 self.task.log.info("Found skymap %s in %s in %s.", name, datasetTypeName, self.root)
127 super().prep()
129 def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
130 # Docstring inherited from RepoConverter.
131 for datasetTypeName, mapping in self.mapper.mappings.items():
132 if datasetTypeName not in self.mapper.calibrations:
133 yield datasetTypeName, mapping
135 def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]:
136 """Return the appropriate SkyMap for the given dataset type.
138 Parameters
139 ----------
140 datasetTypeName : `str`
141 Name of the dataset type for which a skymap is sought.
143 Returns
144 -------
145 skyMap : `BaseSkyMap` or `None`
146 The `BaseSkyMap` instance, or `None` if there was no match.
147 skyMapName : `str` or `None`
148 The Gen3 name for the SkyMap, or `None` if there was no match.
149 """
150 # Use deepCoadd_skyMap by default; there are some dataset types
151 # that use it but don't have "deep" anywhere in their name.
152 struct = self._foundSkyMapsByCoaddName.get("deep")
153 for coaddName in SKYMAP_DATASET_TYPES.keys():
154 if coaddName in datasetTypeName:
155 try:
156 struct = self._foundSkyMapsByCoaddName[coaddName]
157 break
158 except KeyError:
159 # Don't use the default, since we did find a specific
160 # coaddName.
161 struct = None
162 self.task.log.debug(
163 "Dataset %s looks like it might need a skymap, but no %sCoadd_skyMap "
164 "found in repo %s.",
165 datasetTypeName,
166 coaddName,
167 self.root,
168 )
169 if struct is not None:
170 return struct.instance, struct.name
171 else:
172 return None, None
174 def makeRepoWalkerTarget(
175 self,
176 datasetTypeName: str,
177 template: str,
178 keys: Dict[str, type],
179 storageClass: StorageClass,
180 formatter: FormatterParameter = None,
181 targetHandler: Optional[PathElementHandler] = None,
182 ) -> RepoWalker.Target:
183 # Docstring inherited from RepoConverter.
184 skyMap, skyMapName = self.findMatchingSkyMap(datasetTypeName)
185 return RepoWalker.Target(
186 datasetTypeName=datasetTypeName,
187 storageClass=storageClass,
188 template=template,
189 keys=keys,
190 universe=self.task.registry.dimensions,
191 instrument=self.task.instrument.getName(),
192 skyMap=skyMap,
193 skyMapName=skyMapName,
194 formatter=formatter,
195 targetHandler=targetHandler,
196 translatorFactory=self.task.translatorFactory,
197 )
199 def iterDatasets(self) -> Iterator[FileDataset]:
200 # Docstring inherited from RepoConverter.
201 yield from super().iterDatasets()
203 def getRun(self, datasetTypeName: str, calibDate: Optional[str] = None) -> str:
204 # Docstring inherited from RepoConverter.
205 run = self.task.config.runsForced.get(datasetTypeName)
206 if run is None:
207 if self._run is not None:
208 run = self._run
209 else:
210 run = self.task.config.runs.get(datasetTypeName)
211 if run is None:
212 raise ValueError(
213 f"No default run for repo at {self.root}, and no override for dataset {datasetTypeName}."
214 )
215 if run not in self._chain:
216 self._chain.append(run)
217 return run
219 def getCollectionChain(self) -> List[str]:
220 """Return run names that can be used to construct a chained collection
221 that refers to the converted repository (`list` [ `str` ]).
222 """
223 return self._chain
225 def _finish(
226 self, datasets: Mapping[DatasetType, Mapping[Optional[str], List[FileDataset]]], count: int
227 ) -> None:
228 # Docstring inherited from RepoConverter.
229 super()._finish(datasets, count)
230 if self._foundSkyMapsByCoaddName:
231 self._chain.append(BaseSkyMap.SKYMAP_RUN_COLLECTION_NAME)
233 # Class attributes that will be shadowed by public instance attributes;
234 # defined here only for documentation purposes.
236 butler2: Butler2
237 """Gen2 butler associated with this repository.
238 """
240 mapper: CameraMapper
241 """Gen2 mapper associated with this repository.
242 """