Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import os 

25 

26import lsst.daf.persistence as dafPersistence 

27import lsst.daf.butler as dafButler 

28import lsst.obs.base as obsBase 

29import lsst.pex.exceptions as pexExcept 

30from lsst.utils import getPackageDir 

31 

32from .config import Config 

33 

34 

35class Dataset: 

36 """A dataset supported by ``ap_verify``. 

37 

38 Any object of this class is guaranteed to represent a ready-for-use 

39 ap_verify dataset, barring concurrent changes to the file system or EUPS 

40 operations. Constructing a Dataset does not create a compatible output 

41 repository(ies), which can be done by calling `makeCompatibleRepo`. 

42 

43 Parameters 

44 ---------- 

45 datasetId : `str` 

46 A tag identifying the dataset. 

47 

48 Raises 

49 ------ 

50 RuntimeError 

51 Raised if `datasetId` exists, but is not correctly organized or incomplete 

52 ValueError 

53 Raised if `datasetId` is not a recognized ap_verify dataset. No side 

54 effects if this exception is raised. 

55 """ 

56 

57 def __init__(self, datasetId): 

58 # daf.persistence.Policy's behavior on missing keys is apparently undefined 

59 # test for __getattr__ *either* raising KeyError or returning None 

60 try: 

61 datasetPackage = self._getDatasetInfo()[datasetId] 

62 if datasetPackage is None: 

63 raise KeyError 

64 except KeyError: 

65 raise ValueError('Unsupported dataset: ' + datasetId) 

66 

67 try: 

68 self._dataRootDir = getPackageDir(datasetPackage) 

69 except pexExcept.NotFoundError as e: 

70 error = 'Dataset %s requires the %s package, which has not been set up.' \ 

71 % (datasetId, datasetPackage) 

72 raise RuntimeError(error) from e 

73 else: 

74 self._validatePackage() 

75 

76 self._initPackage(datasetPackage) 

77 

78 def _initPackage(self, name): 

79 """Prepare the package backing this ap_verify dataset. 

80 

81 Parameters 

82 ---------- 

83 name : `str` 

84 The EUPS package identifier for the desired package. 

85 """ 

86 # No initialization required at present 

87 pass 

88 

89 @staticmethod 

90 def getSupportedDatasets(): 

91 """The ap_verify dataset IDs that can be passed to this class's constructor. 

92 

93 Returns 

94 ------- 

95 datasets : `set` of `str` 

96 the set of IDs that will be accepted 

97 

98 Raises 

99 ------ 

100 IoError 

101 Raised if the config file does not exist or is not readable 

102 RuntimeError 

103 Raised if the config file exists, but does not contain the expected data 

104 """ 

105 return Dataset._getDatasetInfo().keys() 

106 

107 @staticmethod 

108 def _getDatasetInfo(): 

109 """Return external data on supported ap_verify datasets. 

110 

111 If an exception is raised, the program state shall be unchanged. 

112 

113 Returns 

114 ------- 

115 datasetToPackage : `dict`-like 

116 a map from dataset IDs to package names. 

117 

118 Raises 

119 ------ 

120 RuntimeError 

121 Raised if the config file exists, but does not contain the expected data 

122 """ 

123 return Config.instance['datasets'] 

124 

125 @property 

126 def datasetRoot(self): 

127 """The parent directory containing everything related to the 

128 ap_verify dataset (`str`, read-only). 

129 """ 

130 return self._dataRootDir 

131 

132 @property 

133 def rawLocation(self): 

134 """The directory containing the "raw" input data (`str`, read-only). 

135 """ 

136 return os.path.join(self.datasetRoot, 'raw') 

137 

138 @property 

139 def calibLocation(self): 

140 """The directory containing the calibration data (`str`, read-only). 

141 """ 

142 return os.path.join(self.datasetRoot, 'calib') 

143 

144 @property 

145 def refcatsLocation(self): 

146 """The directory containing external astrometric and photometric 

147 reference catalogs (`str`, read-only). 

148 """ 

149 return os.path.join(self.datasetRoot, 'refcats') 

150 

151 @property 

152 def templateLocation(self): 

153 """The directory containing the image subtraction templates (`str`, read-only). 

154 """ 

155 return os.path.join(self.datasetRoot, 'templates') 

156 

157 @property 

158 def configLocation(self): 

159 """The directory containing configs that can be used to process the data (`str`, read-only). 

160 """ 

161 return os.path.join(self.datasetRoot, 'config') 

162 

163 @property 

164 def obsPackage(self): 

165 """The name of the obs package associated with this data (`str`, read-only). 

166 """ 

167 return dafPersistence.Butler.getMapperClass(self._stubInputRepo).getPackageName() 

168 

169 @property 

170 def camera(self): 

171 """The name of the Gen 2 camera associated with this data (`str`, read-only). 

172 """ 

173 return dafPersistence.Butler.getMapperClass(self._stubInputRepo).getCameraName() 

174 

175 @property 

176 def instrument(self): 

177 """The Gen 3 instrument associated with this data (`lsst.obs.base.Instrument`, read-only). 

178 """ 

179 butler = dafButler.Butler(self._preloadedRepo, writeable=False) 

180 instruments = list(butler.registry.queryDimensions('instrument')) 

181 if len(instruments) != 1: 

182 raise RuntimeError(f"Dataset does not have exactly one instrument; got {instruments}.") 

183 else: 

184 return obsBase.Instrument.fromName(instruments[0]["instrument"], butler.registry) 

185 

186 @property 

187 def _stubInputRepo(self): 

188 """The directory containing the data set's input stub (`str`, read-only). 

189 """ 

190 return os.path.join(self.datasetRoot, 'repo') 

191 

192 @property 

193 def _preloadedRepo(self): 

194 """The directory containing the pre-ingested Gen 3 repo (`str`, read-only). 

195 """ 

196 return os.path.join(self.datasetRoot, 'preloaded') 

197 

198 @property 

199 def _preloadedExport(self): 

200 """The file containing an exported registry of `_preloadedRepo` (`str`, read-only). 

201 """ 

202 return os.path.join(self.configLocation, 'export.yaml') 

203 

204 def _validatePackage(self): 

205 """Confirm that the dataset directory satisfies all assumptions. 

206 

207 Raises 

208 ------ 

209 RuntimeError 

210 Raised if the package represented by this object does not conform to the 

211 dataset framework 

212 

213 Notes 

214 ----- 

215 Requires that `self._dataRootDir` has been initialized. 

216 """ 

217 if not os.path.exists(self.datasetRoot): 

218 raise RuntimeError('Could not find dataset at ' + self.datasetRoot) 

219 if not os.path.exists(self.rawLocation): 

220 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing data directory') 

221 if not os.path.exists(self.calibLocation): 

222 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing calibration directory') 

223 # Template and refcat directories might not be subdirectories of self.datasetRoot 

224 if not os.path.exists(self.templateLocation): 

225 raise RuntimeError('Dataset is missing template directory at ' + self.templateLocation) 

226 if not os.path.exists(self.refcatsLocation): 

227 raise RuntimeError('Dataset is missing reference catalog directory at ' + self.refcatsLocation) 

228 if not os.path.exists(self._stubInputRepo): 

229 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing stub repo') 

230 if not _isRepo(self._stubInputRepo): 

231 raise RuntimeError('Stub repo at ' + self._stubInputRepo + 'is missing mapper file') 

232 

233 def makeCompatibleRepo(self, repoDir, calibRepoDir): 

234 """Set up a directory as a Gen 2 repository compatible with this ap_verify dataset. 

235 

236 If the directory already exists, any files required by the dataset will 

237 be added if absent; otherwise the directory will remain unchanged. 

238 

239 Parameters 

240 ---------- 

241 repoDir : `str` 

242 The directory where the output repository will be created. 

243 calibRepoDir : `str` 

244 The directory where the output calibration repository will be created. 

245 """ 

246 mapperArgs = {'mapperArgs': {'calibRoot': calibRepoDir}} 

247 if _isRepo(self.templateLocation): 

248 # Stub repo is not a parent because can't mix v1 and v2 repositories in parents list 

249 dafPersistence.Butler(inputs=[{"root": self.templateLocation, "mode": "r"}], 

250 outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}]) 

251 else: 

252 dafPersistence.Butler(inputs=[{"root": self._stubInputRepo, "mode": "r"}], 

253 outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}]) 

254 

255 def makeCompatibleRepoGen3(self, repoDir): 

256 """Set up a directory as a Gen 3 repository compatible with this ap_verify dataset. 

257 

258 If the directory already exists, any files required by the dataset will 

259 be added if absent; otherwise the directory will remain unchanged. 

260 

261 Parameters 

262 ---------- 

263 repoDir : `str` 

264 The directory where the output repository will be created. 

265 """ 

266 repoConfig = dafButler.Butler.makeRepo(repoDir, overwrite=True) 

267 butler = dafButler.Butler(repoConfig, writeable=True) 

268 butler.import_(directory=self._preloadedRepo, filename=self._preloadedExport, 

269 transfer="auto") 

270 

271 

272def _isRepo(repoDir): 

273 """Test whether a directory has been set up as a repository. 

274 """ 

275 return os.path.exists(os.path.join(repoDir, '_mapper')) \ 

276 or os.path.exists(os.path.join(repoDir, 'repositoryCfg.yaml'))