Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["StandardRepoConverter"] 

24 

25import os.path 

26from dataclasses import dataclass 

27from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Set, Tuple 

28 

29from lsst.log import Log 

30from lsst.log.utils import temporaryLogLevel 

31from lsst.daf.persistence import Butler as Butler2 

32from lsst.daf.butler import DatasetType, DatasetRef, DataCoordinate, FileDataset 

33from .repoConverter import RepoConverter 

34from .repoWalker import RepoWalker 

35 

36SKYMAP_DATASET_TYPES = { 

37 coaddName: f"{coaddName}Coadd_skyMap" for coaddName in ("deep", "goodSeeing", "dcr") 

38} 

39 

40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true

41 from lsst.skymap import BaseSkyMap 

42 from lsst.daf.butler import StorageClass, FormatterParameter 

43 from .cameraMapper import CameraMapper 

44 from .repoWalker.scanner import PathElementHandler 

45 from ..mapping import Mapping as CameraMapperMapping # disambiguate from collections.abc.Mapping 

46 

47 

48@dataclass 

49class FoundSkyMap: 

50 """Struct containing information about a SkyMap found in a Gen2 repository. 

51 """ 

52 

53 name: str 

54 """Name of the skymap used in Gen3 data IDs. 

55 """ 

56 

57 instance: BaseSkyMap 

58 """An instance of the actual skymap class. 

59 """ 

60 

61 coaddName: str 

62 """The coadd name used as a prefix for the dataset type this skymap was 

63 found in. 

64 """ 

65 

66 ref: DatasetRef 

67 """A `DatasetRef` that can be used to ingest the skymap dataset into a 

68 Gen3 repository. 

69 """ 

70 

71 filename: str 

72 """Name of the file containing the skymap dataset, relative to the 

73 repository root. 

74 """ 

75 

76 

77class StandardRepoConverter(RepoConverter): 

78 """A specialization of `RepoConverter` for non-calibration repositories. 

79 

80 Parameters 

81 ---------- 

82 kwds 

83 Keyword arguments are forwarded to (and required by) `RepoConverter`. 

84 """ 

85 

86 def __init__(self, **kwds): 

87 super().__init__(**kwds) 

88 # Shush noisy log messages from Gen2 Mapper classes. 

89 with temporaryLogLevel("CameraMapper", Log.ERROR): 

90 with temporaryLogLevel("HscMapper", Log.ERROR): 

91 self.butler2 = Butler2(self.root) 

92 self.mapper = self.butler2.getMapperClass(self.root)(root=self.root) 

93 self._foundSkyMapsByCoaddName = {} 

94 self._chain = {} 

95 

96 def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool: 

97 # Docstring inherited from RepoConverter. 

98 return datasetTypeName in SKYMAP_DATASET_TYPES.values() 

99 

100 def prep(self): 

101 # Docstring inherited from RepoConverter. 

102 self.task.log.info(f"Looking for skymaps in root {self.root}.") 

103 for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items(): 

104 if not self.task.isDatasetTypeIncluded(datasetTypeName): 

105 continue 

106 try: 

107 exists = self.butler2.datasetExists(datasetTypeName) 

108 except AttributeError: 

109 # This mapper doesn't even define this dataset type. 

110 continue 

111 if not exists: 

112 continue 

113 instance = self.butler2.get(datasetTypeName) 

114 name = self.task.useSkyMap(instance, datasetTypeName) 

115 datasetType = DatasetType(datasetTypeName, dimensions=["skymap"], 

116 storageClass="SkyMap", universe=self.task.universe) 

117 dataId = DataCoordinate.standardize(skymap=name, universe=self.task.universe) 

118 struct = FoundSkyMap(name=name, instance=instance, coaddName=coaddName, 

119 ref=DatasetRef(datasetType, dataId), 

120 filename=self.butler2.getUri(datasetTypeName)) 

121 self._foundSkyMapsByCoaddName[coaddName] = struct 

122 self.task.log.info("Found skymap %s in %s in %s.", name, datasetTypeName, self.root) 

123 super().prep() 

124 

125 def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]: 

126 # Docstring inherited from RepoConverter. 

127 for datasetTypeName, mapping in self.mapper.mappings.items(): 

128 if datasetTypeName not in self.mapper.calibrations: 

129 yield datasetTypeName, mapping 

130 

131 def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]: 

132 """Return the appropriate SkyMap for the given dataset type. 

133 

134 Parameters 

135 ---------- 

136 datasetTypeName : `str` 

137 Name of the dataset type for which a skymap is sought. 

138 

139 Returns 

140 ------- 

141 skyMap : `BaseSkyMap` or `None` 

142 The `BaseSkyMap` instance, or `None` if there was no match. 

143 skyMapName : `str` or `None` 

144 The Gen3 name for the SkyMap, or `None` if there was no match. 

145 """ 

146 # Use deepCoadd_skyMap by default; there are some dataset types 

147 # that use it but don't have "deep" anywhere in their name. 

148 struct = self._foundSkyMapsByCoaddName.get("deep") 

149 for coaddName in SKYMAP_DATASET_TYPES.keys(): 

150 if coaddName in datasetTypeName: 

151 try: 

152 struct = self._foundSkyMapsByCoaddName[coaddName] 

153 break 

154 except KeyError: 

155 # Don't use the default, since we did find a specific 

156 # coaddName. 

157 struct = None 

158 self.task.log.debug( 

159 ("Dataset %s looks like it might need a skymap, but no %sCoadd_skyMap " 

160 "found in repo %s."), 

161 datasetTypeName, coaddName, self.root 

162 ) 

163 if struct is not None: 

164 return struct.instance, struct.name 

165 else: 

166 return None, None 

167 

168 def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[str, type], 

169 storageClass: StorageClass, 

170 formatter: FormatterParameter = None, 

171 targetHandler: Optional[PathElementHandler] = None, 

172 ) -> RepoWalker.Target: 

173 # Docstring inherited from RepoConverter. 

174 skyMap, skyMapName = self.findMatchingSkyMap(datasetTypeName) 

175 return RepoWalker.Target( 

176 datasetTypeName=datasetTypeName, 

177 storageClass=storageClass, 

178 template=template, 

179 keys=keys, 

180 universe=self.task.registry.dimensions, 

181 instrument=self.task.instrument.getName(), 

182 skyMap=skyMap, 

183 skyMapName=skyMapName, 

184 formatter=formatter, 

185 targetHandler=targetHandler, 

186 translatorFactory=self.task.translatorFactory, 

187 ) 

188 

189 def iterDatasets(self) -> Iterator[FileDataset]: 

190 # Docstring inherited from RepoConverter. 

191 for struct in self._foundSkyMapsByCoaddName.values(): 

192 if self.task.isDatasetTypeIncluded(struct.ref.datasetType.name): 

193 yield FileDataset(path=os.path.join(self.root, struct.filename), refs=struct.ref) 

194 yield from super().iterDatasets() 

195 

196 def getRun(self, datasetTypeName: str, calibDate: Optional[str] = None) -> str: 

197 # Docstring inherited from RepoConverter. 

198 if self._run is not None: 

199 run = self._run 

200 else: 

201 run = self.task.config.runs.get(datasetTypeName) 

202 if run is None: 

203 raise ValueError(f"No default run for repo at {self.root}, and no " 

204 f"override for dataset {datasetTypeName}.") 

205 self._chain.setdefault(run, set()).add(datasetTypeName) 

206 return run 

207 

208 def getCollectionChain(self) -> List[Tuple[str, Set[str]]]: 

209 """Return tuples of run name and associated dataset type names that 

210 can be used to construct a chained collection that refers to the 

211 converted repository (`list` [ `tuple` ]). 

212 """ 

213 return list(self._chain.items()) 

214 

215 # Class attributes that will be shadowed by public instance attributes; 

216 # defined here only for documentation purposes. 

217 

218 butler2: Butler2 

219 """Gen2 butler associated with this repository. 

220 """ 

221 

222 mapper: CameraMapper 

223 """Gen2 mapper associated with this repository. 

224 """