Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import os 

25 

26from lsst.daf.persistence import Butler 

27import lsst.pex.exceptions as pexExcept 

28from lsst.utils import getPackageDir 

29 

30from .config import Config 

31 

32 

33class Dataset: 

34 """A dataset supported by ``ap_verify``. 

35 

36 Any object of this class is guaranteed to represent a ready-for-use 

37 dataset, barring concurrent changes to the file system or EUPS operations. 

38 Constructing a Dataset does not create a compatible output repository(ies), 

39 which can be done by calling `makeCompatibleRepo`. 

40 

41 Parameters 

42 ---------- 

43 datasetId : `str` 

44 A tag identifying the dataset. 

45 

46 Raises 

47 ------ 

48 RuntimeError 

49 Raised if `datasetId` exists, but is not correctly organized or incomplete 

50 ValueError 

51 Raised if `datasetId` is not a recognized dataset. No side effects if this 

52 exception is raised. 

53 """ 

54 

55 def __init__(self, datasetId): 

56 # daf.persistence.Policy's behavior on missing keys is apparently undefined 

57 # test for __getattr__ *either* raising KeyError or returning None 

58 try: 

59 datasetPackage = self._getDatasetInfo()[datasetId] 

60 if datasetPackage is None: 

61 raise KeyError 

62 except KeyError: 

63 raise ValueError('Unsupported dataset: ' + datasetId) 

64 

65 try: 

66 self._dataRootDir = getPackageDir(datasetPackage) 

67 except pexExcept.NotFoundError as e: 

68 error = 'Dataset %s requires the %s package, which has not been set up.' \ 

69 % (datasetId, datasetPackage) 

70 raise RuntimeError(error) from e 

71 else: 

72 self._validatePackage() 

73 

74 self._initPackage(datasetPackage) 

75 

76 def _initPackage(self, name): 

77 """Prepare the package backing this dataset. 

78 

79 Parameters 

80 ---------- 

81 name : `str` 

82 The EUPS package identifier for the desired package. 

83 """ 

84 # No initialization required at present 

85 pass 

86 

87 @staticmethod 

88 def getSupportedDatasets(): 

89 """The dataset IDs that can be passed to this class's constructor. 

90 

91 Returns 

92 ------- 

93 datasets : `set` of `str` 

94 the set of IDs that will be accepted 

95 

96 Raises 

97 ------ 

98 IoError 

99 Raised if the config file does not exist or is not readable 

100 RuntimeError 

101 Raised if the config file exists, but does not contain the expected data 

102 """ 

103 return Dataset._getDatasetInfo().keys() 

104 

105 @staticmethod 

106 def _getDatasetInfo(): 

107 """Return external data on supported datasets. 

108 

109 If an exception is raised, the program state shall be unchanged. 

110 

111 Returns 

112 ------- 

113 datasetToPackage : `dict`-like 

114 a map from dataset IDs to package names. 

115 

116 Raises 

117 ------ 

118 RuntimeError 

119 Raised if the config file exists, but does not contain the expected data 

120 """ 

121 return Config.instance['datasets'] 

122 

123 @property 

124 def datasetRoot(self): 

125 """The parent directory containing everything related to the dataset (`str`, read-only). 

126 """ 

127 return self._dataRootDir 

128 

129 @property 

130 def rawLocation(self): 

131 """The directory containing the "raw" input data (`str`, read-only). 

132 """ 

133 return os.path.join(self.datasetRoot, 'raw') 

134 

135 @property 

136 def calibLocation(self): 

137 """The directory containing the calibration data (`str`, read-only). 

138 """ 

139 return os.path.join(self.datasetRoot, 'calib') 

140 

141 @property 

142 def refcatsLocation(self): 

143 """The directory containing external astrometric and photometric 

144 reference catalogs (`str`, read-only). 

145 """ 

146 return os.path.join(self.datasetRoot, 'refcats') 

147 

148 @property 

149 def templateLocation(self): 

150 """The directory containing the image subtraction templates (`str`, read-only). 

151 """ 

152 return os.path.join(self.datasetRoot, 'templates') 

153 

154 @property 

155 def configLocation(self): 

156 """The directory containing configs that can be used to process the dataset (`str`, read-only). 

157 """ 

158 return os.path.join(self.datasetRoot, 'config') 

159 

160 @property 

161 def obsPackage(self): 

162 """The name of the obs package associated with this dataset (`str`, read-only). 

163 """ 

164 return Butler.getMapperClass(self._stubInputRepo).getPackageName() 

165 

166 @property 

167 def camera(self): 

168 """The name of the camera associated with this dataset (`str`, read-only). 

169 """ 

170 return Butler.getMapperClass(self._stubInputRepo).getCameraName() 

171 

172 @property 

173 def _stubInputRepo(self): 

174 """The directory containing the data set's input stub (`str`, read-only). 

175 """ 

176 return os.path.join(self.datasetRoot, 'repo') 

177 

178 def _validatePackage(self): 

179 """Confirm that the dataset directory satisfies all assumptions. 

180 

181 Raises 

182 ------ 

183 RuntimeError 

184 Raised if the package represented by this object does not conform to the 

185 dataset framework 

186 

187 Notes 

188 ----- 

189 Requires that `self._dataRootDir` has been initialized. 

190 """ 

191 if not os.path.exists(self.datasetRoot): 

192 raise RuntimeError('Could not find dataset at ' + self.datasetRoot) 

193 if not os.path.exists(self.rawLocation): 

194 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing data directory') 

195 if not os.path.exists(self.calibLocation): 

196 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing calibration directory') 

197 # Template and refcat directories might not be subdirectories of self.datasetRoot 

198 if not os.path.exists(self.templateLocation): 

199 raise RuntimeError('Dataset is missing template directory at ' + self.templateLocation) 

200 if not os.path.exists(self.refcatsLocation): 

201 raise RuntimeError('Dataset is missing reference catalog directory at ' + self.refcatsLocation) 

202 if not os.path.exists(self._stubInputRepo): 

203 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing stub repo') 

204 if not _isRepo(self._stubInputRepo): 

205 raise RuntimeError('Stub repo at ' + self._stubInputRepo + 'is missing mapper file') 

206 

207 def makeCompatibleRepo(self, repoDir, calibRepoDir): 

208 """Set up a directory as a repository compatible with this dataset. 

209 

210 If the directory already exists, any files required by the dataset will 

211 be added if absent; otherwise the directory will remain unchanged. 

212 

213 Parameters 

214 ---------- 

215 repoDir : `str` 

216 The directory where the output repository will be created. 

217 calibRepoDir : `str` 

218 The directory where the output calibration repository will be created. 

219 """ 

220 mapperArgs = {'mapperArgs': {'calibRoot': calibRepoDir}} 

221 if _isRepo(self.templateLocation): 

222 # Stub repo is not a parent because can't mix v1 and v2 repositories in parents list 

223 Butler(inputs=[{"root": self.templateLocation, "mode": "r"}], 

224 outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}]) 

225 else: 

226 Butler(inputs=[{"root": self._stubInputRepo, "mode": "r"}], 

227 outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}]) 

228 

229 

230def _isRepo(repoDir): 

231 """Test whether a directory has been set up as a repository. 

232 """ 

233 return os.path.exists(os.path.join(repoDir, '_mapper')) \ 

234 or os.path.exists(os.path.join(repoDir, 'repositoryCfg.yaml'))