Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["StandardRepoConverter"] 

24 

25import os.path 

26from dataclasses import dataclass 

27from typing import TYPE_CHECKING, Dict, Iterator, Optional, Tuple 

28 

29from lsst.log import Log 

30from lsst.log.utils import temporaryLogLevel 

31from lsst.daf.persistence import Butler as Butler2 

32from lsst.daf.butler import DatasetType, DatasetRef, DataCoordinate, FileDataset 

33from .repoConverter import RepoConverter 

34from .repoWalker import RepoWalker 

35 

36SKYMAP_DATASET_TYPES = { 

37 coaddName: f"{coaddName}Coadd_skyMap" for coaddName in ("deep", "goodSeeing", "dcr") 

38} 

39 

40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true

41 from lsst.skymap import BaseSkyMap 

42 from lsst.daf.butler import StorageClass 

43 from .cameraMapper import CameraMapper 

44 from ..mapping import Mapping as CameraMapperMapping # disambiguate from collections.abc.Mapping 

45 

46 

47@dataclass 

48class FoundSkyMap: 

49 """Struct containing information about a SkyMap found in a Gen2 repository. 

50 """ 

51 

52 name: str 

53 """Name of the skymap used in Gen3 data IDs. 

54 """ 

55 

56 instance: BaseSkyMap 

57 """An instance of the actual skymap class. 

58 """ 

59 

60 coaddName: str 

61 """The coadd name used as a prefix for the dataset type this skymap was 

62 found in. 

63 """ 

64 

65 ref: DatasetRef 

66 """A `DatasetRef` that can be used to ingest the skymap dataset into a 

67 Gen3 repository. 

68 """ 

69 

70 filename: str 

71 """Name of the file containing the skymap dataset, relative to the 

72 repository root. 

73 """ 

74 

75 

76class StandardRepoConverter(RepoConverter): 

77 """A specialization of `RepoConverter` for non-calibration repositories. 

78 

79 Parameters 

80 ---------- 

81 kwds 

82 Keyword arguments are forwarded to (and required by) `RepoConverter`. 

83 """ 

84 

85 def __init__(self, **kwds): 

86 super().__init__(**kwds) 

87 # Shush noisy log messages from Gen2 Mapper classes. 

88 with temporaryLogLevel("CameraMapper", Log.ERROR): 

89 with temporaryLogLevel("HscMapper", Log.ERROR): 

90 self.butler2 = Butler2(self.root) 

91 self.mapper = self.butler2.getMapperClass(self.root)(root=self.root) 

92 self._foundSkyMapsByCoaddName = {} 

93 

94 def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool: 

95 # Docstring inherited from RepoConverter. 

96 return datasetTypeName in SKYMAP_DATASET_TYPES.values() 

97 

98 def prep(self): 

99 # Docstring inherited from RepoConverter. 

100 self.task.log.info(f"Looking for skymaps in root {self.root}.") 

101 for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items(): 

102 if not self.task.isDatasetTypeIncluded(datasetTypeName): 

103 continue 

104 try: 

105 exists = self.butler2.datasetExists(datasetTypeName) 

106 except AttributeError: 

107 # This mapper doesn't even define this dataset type. 

108 continue 

109 if not exists: 

110 continue 

111 instance = self.butler2.get(datasetTypeName) 

112 name = self.task.useSkyMap(instance) 

113 datasetType = DatasetType(datasetTypeName, dimensions=["skymap"], 

114 storageClass="SkyMap", universe=self.task.universe) 

115 dataId = DataCoordinate.standardize(skymap=name, universe=self.task.universe) 

116 struct = FoundSkyMap(name=name, instance=instance, coaddName=coaddName, 

117 ref=DatasetRef(datasetType, dataId), 

118 filename=self.butler2.getUri(datasetTypeName)) 

119 self._foundSkyMapsByCoaddName[coaddName] = struct 

120 self.task.log.info("Found skymap %s in %s in %s.", name, datasetTypeName, self.root) 

121 super().prep() 

122 

123 def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]: 

124 # Docstring inherited from RepoConverter. 

125 for datasetTypeName, mapping in self.mapper.mappings.items(): 

126 if datasetTypeName not in self.mapper.calibrations: 

127 yield datasetTypeName, mapping 

128 

129 def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]: 

130 """Return the appropriate SkyMap for the given dataset type. 

131 

132 Parameters 

133 ---------- 

134 datasetTypeName : `str` 

135 Name of the dataset type for which a skymap is sought. 

136 

137 Returns 

138 ------- 

139 skyMap : `BaseSkyMap` or `None` 

140 The `BaseSkyMap` instance, or `None` if there was no match. 

141 skyMapName : `str` or `None` 

142 The Gen3 name for the SkyMap, or `None` if there was no match. 

143 """ 

144 # Use deepCoadd_skyMap by default; there are some dataset types 

145 # that use it but don't have "deep" anywhere in their name. 

146 struct = self._foundSkyMapsByCoaddName.get("deep") 

147 for coaddName in SKYMAP_DATASET_TYPES.keys(): 

148 if coaddName in datasetTypeName: 

149 try: 

150 struct = self._foundSkyMapsByCoaddName[coaddName] 

151 break 

152 except KeyError: 

153 # Don't use the default, since we did find a specific 

154 # coaddName. 

155 struct = None 

156 self.task.log.debug( 

157 ("Dataset %s looks like it might need a skymap, but no %sCoadd_skyMap " 

158 "found in repo %s."), 

159 datasetTypeName, coaddName, self.root 

160 ) 

161 if struct is not None: 

162 return struct.instance, struct.name 

163 else: 

164 return None, None 

165 

166 def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[str, type], 

167 storageClass: StorageClass) -> RepoWalker.Target: 

168 # Docstring inherited from RepoConverter. 

169 skyMap, skyMapName = self.findMatchingSkyMap(datasetTypeName) 

170 return RepoWalker.Target( 

171 datasetTypeName=datasetTypeName, 

172 storageClass=storageClass, 

173 template=template, 

174 keys=keys, 

175 universe=self.task.registry.dimensions, 

176 instrument=self.task.instrument.getName(), 

177 skyMap=skyMap, 

178 skyMapName=skyMapName, 

179 ) 

180 

181 def iterDatasets(self) -> Iterator[FileDataset]: 

182 # Docstring inherited from RepoConverter. 

183 for struct in self._foundSkyMapsByCoaddName.values(): 

184 if self.task.isDatasetTypeIncluded(struct.ref.datasetType.name): 

185 yield FileDataset(path=os.path.join(self.root, struct.filename), refs=struct.ref) 

186 yield from super().iterDatasets() 

187 

188 # Class attributes that will be shadowed by public instance attributes; 

189 # defined here only for documentation purposes. 

190 

191 butler2: Butler2 

192 """Gen2 butler associated with this repository. 

193 """ 

194 

195 mapper: CameraMapper 

196 """Gen2 mapper associated with this repository. 

197 """