Coverage for python/lsst/obs/base/gen2to3/standardRepoConverter.py : 34%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = ["StandardRepoConverter"]
25import os.path
26from dataclasses import dataclass
27from typing import TYPE_CHECKING, Dict, Iterator, Optional, Tuple
29from lsst.log import Log
30from lsst.log.utils import temporaryLogLevel
31from lsst.daf.persistence import Butler as Butler2
32from lsst.daf.butler import DatasetType, DatasetRef, DataCoordinate, FileDataset
33from .repoConverter import RepoConverter
34from .repoWalker import RepoWalker
36SKYMAP_DATASET_TYPES = {
37 coaddName: f"{coaddName}Coadd_skyMap" for coaddName in ("deep", "goodSeeing", "dcr")
38}
40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true
41 from lsst.skymap import BaseSkyMap
42 from lsst.daf.butler import StorageClass
43 from .cameraMapper import CameraMapper
44 from ..mapping import Mapping as CameraMapperMapping # disambiguate from collections.abc.Mapping
47@dataclass
48class FoundSkyMap:
49 """Struct containing information about a SkyMap found in a Gen2 repository.
50 """
52 name: str
53 """Name of the skymap used in Gen3 data IDs.
54 """
56 instance: BaseSkyMap
57 """An instance of the actual skymap class.
58 """
60 coaddName: str
61 """The coadd name used as a prefix for the dataset type this skymap was
62 found in.
63 """
65 ref: DatasetRef
66 """A `DatasetRef` that can be used to ingest the skymap dataset into a
67 Gen3 repository.
68 """
70 filename: str
71 """Name of the file containing the skymap dataset, relative to the
72 repository root.
73 """
76class StandardRepoConverter(RepoConverter):
77 """A specialization of `RepoConverter` for non-calibration repositories.
79 Parameters
80 ----------
81 kwds
82 Keyword arguments are forwarded to (and required by) `RepoConverter`.
83 """
85 def __init__(self, **kwds):
86 super().__init__(**kwds)
87 # Shush noisy log messages from Gen2 Mapper classes.
88 with temporaryLogLevel("CameraMapper", Log.ERROR):
89 with temporaryLogLevel("HscMapper", Log.ERROR):
90 self.butler2 = Butler2(self.root)
91 self.mapper = self.butler2.getMapperClass(self.root)(root=self.root)
92 self._foundSkyMapsByCoaddName = {}
94 def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool:
95 # Docstring inherited from RepoConverter.
96 return datasetTypeName in SKYMAP_DATASET_TYPES.values()
98 def prep(self):
99 # Docstring inherited from RepoConverter.
100 self.task.log.info(f"Looking for skymaps in root {self.root}.")
101 for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items():
102 if not self.task.isDatasetTypeIncluded(datasetTypeName):
103 continue
104 try:
105 exists = self.butler2.datasetExists(datasetTypeName)
106 except AttributeError:
107 # This mapper doesn't even define this dataset type.
108 continue
109 if not exists:
110 continue
111 instance = self.butler2.get(datasetTypeName)
112 name = self.task.useSkyMap(instance, datasetTypeName)
113 datasetType = DatasetType(datasetTypeName, dimensions=["skymap"],
114 storageClass="SkyMap", universe=self.task.universe)
115 dataId = DataCoordinate.standardize(skymap=name, universe=self.task.universe)
116 struct = FoundSkyMap(name=name, instance=instance, coaddName=coaddName,
117 ref=DatasetRef(datasetType, dataId),
118 filename=self.butler2.getUri(datasetTypeName))
119 self._foundSkyMapsByCoaddName[coaddName] = struct
120 self.task.log.info("Found skymap %s in %s in %s.", name, datasetTypeName, self.root)
121 super().prep()
123 def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
124 # Docstring inherited from RepoConverter.
125 for datasetTypeName, mapping in self.mapper.mappings.items():
126 if datasetTypeName not in self.mapper.calibrations:
127 yield datasetTypeName, mapping
129 def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]:
130 """Return the appropriate SkyMap for the given dataset type.
132 Parameters
133 ----------
134 datasetTypeName : `str`
135 Name of the dataset type for which a skymap is sought.
137 Returns
138 -------
139 skyMap : `BaseSkyMap` or `None`
140 The `BaseSkyMap` instance, or `None` if there was no match.
141 skyMapName : `str` or `None`
142 The Gen3 name for the SkyMap, or `None` if there was no match.
143 """
144 # Use deepCoadd_skyMap by default; there are some dataset types
145 # that use it but don't have "deep" anywhere in their name.
146 struct = self._foundSkyMapsByCoaddName.get("deep")
147 for coaddName in SKYMAP_DATASET_TYPES.keys():
148 if coaddName in datasetTypeName:
149 try:
150 struct = self._foundSkyMapsByCoaddName[coaddName]
151 break
152 except KeyError:
153 # Don't use the default, since we did find a specific
154 # coaddName.
155 struct = None
156 self.task.log.debug(
157 ("Dataset %s looks like it might need a skymap, but no %sCoadd_skyMap "
158 "found in repo %s."),
159 datasetTypeName, coaddName, self.root
160 )
161 if struct is not None:
162 return struct.instance, struct.name
163 else:
164 return None, None
166 def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[str, type],
167 storageClass: StorageClass) -> RepoWalker.Target:
168 # Docstring inherited from RepoConverter.
169 skyMap, skyMapName = self.findMatchingSkyMap(datasetTypeName)
170 return RepoWalker.Target(
171 datasetTypeName=datasetTypeName,
172 storageClass=storageClass,
173 template=template,
174 keys=keys,
175 universe=self.task.registry.dimensions,
176 instrument=self.task.instrument.getName(),
177 skyMap=skyMap,
178 skyMapName=skyMapName,
179 )
181 def iterDatasets(self) -> Iterator[FileDataset]:
182 # Docstring inherited from RepoConverter.
183 for struct in self._foundSkyMapsByCoaddName.values():
184 if self.task.isDatasetTypeIncluded(struct.ref.datasetType.name):
185 yield FileDataset(path=os.path.join(self.root, struct.filename), refs=struct.ref)
186 yield from super().iterDatasets()
188 # Class attributes that will be shadowed by public instance attributes;
189 # defined here only for documentation purposes.
191 butler2: Butler2
192 """Gen2 butler associated with this repository.
193 """
195 mapper: CameraMapper
196 """Gen2 mapper associated with this repository.
197 """