Coverage for python/lsst/obs/base/gen2to3/standardRepoConverter.py: 34%

102 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-24 02:02 -0700

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["StandardRepoConverter"] 

24 

25from dataclasses import dataclass 

26from typing import TYPE_CHECKING, Dict, Iterator, List, Mapping, Optional, Tuple 

27 

28from lsst.daf.butler import DataCoordinate, DatasetRef, DatasetType, FileDataset 

29from lsst.daf.persistence import Butler as Butler2 

30from lsst.log import Log 

31from lsst.log.utils import temporaryLogLevel 

32from lsst.skymap import BaseSkyMap 

33 

34from .repoConverter import RepoConverter 

35from .repoWalker import RepoWalker 

36 

37SKYMAP_DATASET_TYPES = {coaddName: f"{coaddName}Coadd_skyMap" for coaddName in ("deep", "goodSeeing", "dcr")} 

38 

39if TYPE_CHECKING: 39 ↛ 40line 39 didn't jump to line 40, because the condition on line 39 was never true

40 from lsst.daf.butler import FormatterParameter, StorageClass 

41 

42 from ..mapping import Mapping as CameraMapperMapping # disambiguate from collections.abc.Mapping 

43 from .cameraMapper import CameraMapper 

44 from .repoWalker.scanner import PathElementHandler 

45 

46 

47@dataclass 

48class FoundSkyMap: 

49 """Struct containing information about a SkyMap in a Gen2 repository.""" 

50 

51 name: str 

52 """Name of the skymap used in Gen3 data IDs. 

53 """ 

54 

55 instance: BaseSkyMap 

56 """An instance of the actual skymap class. 

57 """ 

58 

59 coaddName: str 

60 """The coadd name used as a prefix for the dataset type this skymap was 

61 found in. 

62 """ 

63 

64 ref: DatasetRef 

65 """A `DatasetRef` that can be used to ingest the skymap dataset into a 

66 Gen3 repository. 

67 """ 

68 

69 filename: str 

70 """Name of the file containing the skymap dataset, relative to the 

71 repository root. 

72 """ 

73 

74 

75class StandardRepoConverter(RepoConverter): 

76 """A specialization of `RepoConverter` for non-calibration repositories. 

77 

78 Parameters 

79 ---------- 

80 kwds 

81 Keyword arguments are forwarded to (and required by) `RepoConverter`. 

82 """ 

83 

84 def __init__(self, **kwds): 

85 super().__init__(**kwds) 

86 # Shush noisy log messages from Gen2 Mapper classes. 

87 # These are currently lsst.log loggers. 

88 with temporaryLogLevel("CameraMapper", Log.ERROR): 

89 with temporaryLogLevel("HscMapper", Log.ERROR): 

90 self.butler2 = Butler2(self.root) 

91 self.mapper = self.butler2.getMapperClass(self.root)(root=self.root) 

92 self._foundSkyMapsByCoaddName = {} 

93 self._chain = [] 

94 

95 def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool: 

96 # Docstring inherited from RepoConverter. 

97 return datasetTypeName in SKYMAP_DATASET_TYPES.values() 

98 

99 def prep(self): 

100 # Docstring inherited from RepoConverter. 

101 self.task.log.info("Looking for skymaps in root %s", self.root) 

102 for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items(): 

103 if not self.task.isDatasetTypeIncluded(datasetTypeName): 

104 continue 

105 try: 

106 exists = self.butler2.datasetExists(datasetTypeName) 

107 except AttributeError: 

108 # This mapper doesn't even define this dataset type. 

109 continue 

110 if not exists: 

111 continue 

112 instance = self.butler2.get(datasetTypeName) 

113 name = self.task.useSkyMap(instance, datasetTypeName) 

114 datasetType = DatasetType( 

115 datasetTypeName, dimensions=["skymap"], storageClass="SkyMap", universe=self.task.universe 

116 ) 

117 dataId = DataCoordinate.standardize(skymap=name, universe=self.task.universe) 

118 struct = FoundSkyMap( 

119 name=name, 

120 instance=instance, 

121 coaddName=coaddName, 

122 ref=DatasetRef(datasetType, dataId), 

123 filename=self.butler2.getUri(datasetTypeName), 

124 ) 

125 self._foundSkyMapsByCoaddName[coaddName] = struct 

126 self.task.log.info("Found skymap %s in %s in %s.", name, datasetTypeName, self.root) 

127 super().prep() 

128 

129 def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]: 

130 # Docstring inherited from RepoConverter. 

131 for datasetTypeName, mapping in self.mapper.mappings.items(): 

132 if datasetTypeName not in self.mapper.calibrations: 

133 yield datasetTypeName, mapping 

134 

135 def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]: 

136 """Return the appropriate SkyMap for the given dataset type. 

137 

138 Parameters 

139 ---------- 

140 datasetTypeName : `str` 

141 Name of the dataset type for which a skymap is sought. 

142 

143 Returns 

144 ------- 

145 skyMap : `BaseSkyMap` or `None` 

146 The `BaseSkyMap` instance, or `None` if there was no match. 

147 skyMapName : `str` or `None` 

148 The Gen3 name for the SkyMap, or `None` if there was no match. 

149 """ 

150 # Use deepCoadd_skyMap by default; there are some dataset types 

151 # that use it but don't have "deep" anywhere in their name. 

152 struct = self._foundSkyMapsByCoaddName.get("deep") 

153 for coaddName in SKYMAP_DATASET_TYPES.keys(): 

154 if coaddName in datasetTypeName: 

155 try: 

156 struct = self._foundSkyMapsByCoaddName[coaddName] 

157 break 

158 except KeyError: 

159 # Don't use the default, since we did find a specific 

160 # coaddName. 

161 struct = None 

162 self.task.log.debug( 

163 "Dataset %s looks like it might need a skymap, but no %sCoadd_skyMap " 

164 "found in repo %s.", 

165 datasetTypeName, 

166 coaddName, 

167 self.root, 

168 ) 

169 if struct is not None: 

170 return struct.instance, struct.name 

171 else: 

172 return None, None 

173 

174 def makeRepoWalkerTarget( 

175 self, 

176 datasetTypeName: str, 

177 template: str, 

178 keys: Dict[str, type], 

179 storageClass: StorageClass, 

180 formatter: FormatterParameter = None, 

181 targetHandler: Optional[PathElementHandler] = None, 

182 ) -> RepoWalker.Target: 

183 # Docstring inherited from RepoConverter. 

184 skyMap, skyMapName = self.findMatchingSkyMap(datasetTypeName) 

185 return RepoWalker.Target( 

186 datasetTypeName=datasetTypeName, 

187 storageClass=storageClass, 

188 template=template, 

189 keys=keys, 

190 universe=self.task.registry.dimensions, 

191 instrument=self.task.instrument.getName(), 

192 skyMap=skyMap, 

193 skyMapName=skyMapName, 

194 formatter=formatter, 

195 targetHandler=targetHandler, 

196 translatorFactory=self.task.translatorFactory, 

197 ) 

198 

199 def iterDatasets(self) -> Iterator[FileDataset]: 

200 # Docstring inherited from RepoConverter. 

201 yield from super().iterDatasets() 

202 

203 def getRun(self, datasetTypeName: str, calibDate: Optional[str] = None) -> str: 

204 # Docstring inherited from RepoConverter. 

205 run = self.task.config.runsForced.get(datasetTypeName) 

206 if run is None: 

207 if self._run is not None: 

208 run = self._run 

209 else: 

210 run = self.task.config.runs.get(datasetTypeName) 

211 if run is None: 

212 raise ValueError( 

213 f"No default run for repo at {self.root}, and no override for dataset {datasetTypeName}." 

214 ) 

215 if run not in self._chain: 

216 self._chain.append(run) 

217 return run 

218 

219 def getCollectionChain(self) -> List[str]: 

220 """Return run names that can be used to construct a chained collection 

221 that refers to the converted repository (`list` [ `str` ]). 

222 """ 

223 return self._chain 

224 

225 def _finish( 

226 self, datasets: Mapping[DatasetType, Mapping[Optional[str], List[FileDataset]]], count: int 

227 ) -> None: 

228 # Docstring inherited from RepoConverter. 

229 super()._finish(datasets, count) 

230 if self._foundSkyMapsByCoaddName: 

231 self._chain.append(BaseSkyMap.SKYMAP_RUN_COLLECTION_NAME) 

232 

233 # Class attributes that will be shadowed by public instance attributes; 

234 # defined here only for documentation purposes. 

235 

236 butler2: Butler2 

237 """Gen2 butler associated with this repository. 

238 """ 

239 

240 mapper: CameraMapper 

241 """Gen2 mapper associated with this repository. 

242 """