Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["StandardRepoConverter"] 

24 

25from dataclasses import dataclass 

26from typing import TYPE_CHECKING, Dict, Iterator, List, Mapping, Optional, Tuple 

27 

28from lsst.log import Log 

29from lsst.log.utils import temporaryLogLevel 

30from lsst.daf.persistence import Butler as Butler2 

31from lsst.daf.butler import DatasetType, DatasetRef, DataCoordinate, FileDataset 

32from lsst.skymap import BaseSkyMap 

33from .repoConverter import RepoConverter 

34from .repoWalker import RepoWalker 

35 

36SKYMAP_DATASET_TYPES = { 

37 coaddName: f"{coaddName}Coadd_skyMap" for coaddName in ("deep", "goodSeeing", "dcr") 

38} 

39 

40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true

41 from lsst.daf.butler import StorageClass, FormatterParameter 

42 from .cameraMapper import CameraMapper 

43 from .repoWalker.scanner import PathElementHandler 

44 from ..mapping import Mapping as CameraMapperMapping # disambiguate from collections.abc.Mapping 

45 

46 

47@dataclass 

48class FoundSkyMap: 

49 """Struct containing information about a SkyMap found in a Gen2 repository. 

50 """ 

51 

52 name: str 

53 """Name of the skymap used in Gen3 data IDs. 

54 """ 

55 

56 instance: BaseSkyMap 

57 """An instance of the actual skymap class. 

58 """ 

59 

60 coaddName: str 

61 """The coadd name used as a prefix for the dataset type this skymap was 

62 found in. 

63 """ 

64 

65 ref: DatasetRef 

66 """A `DatasetRef` that can be used to ingest the skymap dataset into a 

67 Gen3 repository. 

68 """ 

69 

70 filename: str 

71 """Name of the file containing the skymap dataset, relative to the 

72 repository root. 

73 """ 

74 

75 

76class StandardRepoConverter(RepoConverter): 

77 """A specialization of `RepoConverter` for non-calibration repositories. 

78 

79 Parameters 

80 ---------- 

81 kwds 

82 Keyword arguments are forwarded to (and required by) `RepoConverter`. 

83 """ 

84 

85 def __init__(self, **kwds): 

86 super().__init__(**kwds) 

87 # Shush noisy log messages from Gen2 Mapper classes. 

88 with temporaryLogLevel("CameraMapper", Log.ERROR): 

89 with temporaryLogLevel("HscMapper", Log.ERROR): 

90 self.butler2 = Butler2(self.root) 

91 self.mapper = self.butler2.getMapperClass(self.root)(root=self.root) 

92 self._foundSkyMapsByCoaddName = {} 

93 self._chain = [] 

94 

95 def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool: 

96 # Docstring inherited from RepoConverter. 

97 return datasetTypeName in SKYMAP_DATASET_TYPES.values() 

98 

99 def prep(self): 

100 # Docstring inherited from RepoConverter. 

101 self.task.log.info("Looking for skymaps in root %s", self.root) 

102 for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items(): 

103 if not self.task.isDatasetTypeIncluded(datasetTypeName): 

104 continue 

105 try: 

106 exists = self.butler2.datasetExists(datasetTypeName) 

107 except AttributeError: 

108 # This mapper doesn't even define this dataset type. 

109 continue 

110 if not exists: 

111 continue 

112 instance = self.butler2.get(datasetTypeName) 

113 name = self.task.useSkyMap(instance, datasetTypeName) 

114 datasetType = DatasetType(datasetTypeName, dimensions=["skymap"], 

115 storageClass="SkyMap", universe=self.task.universe) 

116 dataId = DataCoordinate.standardize(skymap=name, universe=self.task.universe) 

117 struct = FoundSkyMap(name=name, instance=instance, coaddName=coaddName, 

118 ref=DatasetRef(datasetType, dataId), 

119 filename=self.butler2.getUri(datasetTypeName)) 

120 self._foundSkyMapsByCoaddName[coaddName] = struct 

121 self.task.log.info("Found skymap %s in %s in %s.", name, datasetTypeName, self.root) 

122 super().prep() 

123 

124 def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]: 

125 # Docstring inherited from RepoConverter. 

126 for datasetTypeName, mapping in self.mapper.mappings.items(): 

127 if datasetTypeName not in self.mapper.calibrations: 

128 yield datasetTypeName, mapping 

129 

130 def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]: 

131 """Return the appropriate SkyMap for the given dataset type. 

132 

133 Parameters 

134 ---------- 

135 datasetTypeName : `str` 

136 Name of the dataset type for which a skymap is sought. 

137 

138 Returns 

139 ------- 

140 skyMap : `BaseSkyMap` or `None` 

141 The `BaseSkyMap` instance, or `None` if there was no match. 

142 skyMapName : `str` or `None` 

143 The Gen3 name for the SkyMap, or `None` if there was no match. 

144 """ 

145 # Use deepCoadd_skyMap by default; there are some dataset types 

146 # that use it but don't have "deep" anywhere in their name. 

147 struct = self._foundSkyMapsByCoaddName.get("deep") 

148 for coaddName in SKYMAP_DATASET_TYPES.keys(): 

149 if coaddName in datasetTypeName: 

150 try: 

151 struct = self._foundSkyMapsByCoaddName[coaddName] 

152 break 

153 except KeyError: 

154 # Don't use the default, since we did find a specific 

155 # coaddName. 

156 struct = None 

157 self.task.log.debug( 

158 "Dataset %s looks like it might need a skymap, but no %sCoadd_skyMap " 

159 "found in repo %s.", 

160 datasetTypeName, coaddName, self.root 

161 ) 

162 if struct is not None: 

163 return struct.instance, struct.name 

164 else: 

165 return None, None 

166 

167 def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[str, type], 

168 storageClass: StorageClass, 

169 formatter: FormatterParameter = None, 

170 targetHandler: Optional[PathElementHandler] = None, 

171 ) -> RepoWalker.Target: 

172 # Docstring inherited from RepoConverter. 

173 skyMap, skyMapName = self.findMatchingSkyMap(datasetTypeName) 

174 return RepoWalker.Target( 

175 datasetTypeName=datasetTypeName, 

176 storageClass=storageClass, 

177 template=template, 

178 keys=keys, 

179 universe=self.task.registry.dimensions, 

180 instrument=self.task.instrument.getName(), 

181 skyMap=skyMap, 

182 skyMapName=skyMapName, 

183 formatter=formatter, 

184 targetHandler=targetHandler, 

185 translatorFactory=self.task.translatorFactory, 

186 ) 

187 

188 def iterDatasets(self) -> Iterator[FileDataset]: 

189 # Docstring inherited from RepoConverter. 

190 yield from super().iterDatasets() 

191 

192 def getRun(self, datasetTypeName: str, calibDate: Optional[str] = None) -> str: 

193 # Docstring inherited from RepoConverter. 

194 run = self.task.config.runsForced.get(datasetTypeName) 

195 if run is None: 

196 if self._run is not None: 

197 run = self._run 

198 else: 

199 run = self.task.config.runs.get(datasetTypeName) 

200 if run is None: 

201 raise ValueError(f"No default run for repo at {self.root}, and no " 

202 f"override for dataset {datasetTypeName}.") 

203 if run not in self._chain: 

204 self._chain.append(run) 

205 return run 

206 

207 def getCollectionChain(self) -> List[str]: 

208 """Return run names that can be used to construct a chained collection 

209 that refers to the converted repository (`list` [ `str` ]). 

210 """ 

211 return self._chain 

212 

213 def _finish(self, datasets: Mapping[DatasetType, Mapping[Optional[str], List[FileDataset]]], 

214 count: int) -> None: 

215 # Docstring inherited from RepoConverter. 

216 super()._finish(datasets, count) 

217 if self._foundSkyMapsByCoaddName: 

218 self._chain.append(BaseSkyMap.SKYMAP_RUN_COLLECTION_NAME) 

219 

220 # Class attributes that will be shadowed by public instance attributes; 

221 # defined here only for documentation purposes. 

222 

223 butler2: Butler2 

224 """Gen2 butler associated with this repository. 

225 """ 

226 

227 mapper: CameraMapper 

228 """Gen2 mapper associated with this repository. 

229 """