Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24import tempfile 

25import shutil 

26 

27import lsst.utils.tests 

28from lsst.utils import doImport 

29 

30from lsst.daf.butler import StorageClassFactory 

31from lsst.daf.butler import DatastoreConfig 

32 

33from lsst.daf.butler.tests import (FitsCatalogDatasetsHelper, DatasetTestHelper, DatastoreTestHelper, 

34 DummyRegistry) 

35 

36try: 

37 import lsst.afw.table 

38 import lsst.afw.image 

39 import lsst.geom 

40except ImportError: 

41 lsst.afw = None 

42 

43TESTDIR = os.path.dirname(__file__) 

44 

45 

46class DatastoreFitsTests(FitsCatalogDatasetsHelper, DatasetTestHelper, DatastoreTestHelper): 

47 root = None 

48 

49 @classmethod 

50 def setUpClass(cls): 

51 if lsst.afw is None: 

52 raise unittest.SkipTest("afw not available.") 

53 

54 # Base classes need to know where the test directory is 

55 cls.testDir = TESTDIR 

56 

57 # Storage Classes are fixed for all datastores in these tests 

58 scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml") 

59 cls.storageClassFactory = StorageClassFactory() 

60 cls.storageClassFactory.addFromConfig(scConfigFile) 

61 

62 # Read the Datastore config so we can get the class 

63 # information (since we should not assume the constructor 

64 # name here, but rely on the configuration file itself) 

65 datastoreConfig = DatastoreConfig(cls.configFile) 

66 cls.datastoreType = doImport(datastoreConfig["cls"]) 

67 

68 def setUp(self): 

69 self.setUpDatastoreTests(DummyRegistry, DatastoreConfig) 

70 

71 def tearDown(self): 

72 if self.root is not None and os.path.exists(self.root): 

73 shutil.rmtree(self.root, ignore_errors=True) 

74 

75 def testConstructor(self): 

76 datastore = self.makeDatastore() 

77 self.assertIsNotNone(datastore) 

78 

79 def testBasicPutGet(self): 

80 catalog = self.makeExampleCatalog() 

81 datastore = self.makeDatastore() 

82 

83 # Put 

84 dimensions = self.registry.dimensions.extract(("visit", "physical_filter")) 

85 dataId = {"visit": 123456, "physical_filter": "blue", "instrument": "dummy"} 

86 storageClass = self.storageClassFactory.getStorageClass("SourceCatalog") 

87 

88 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId) 

89 

90 datastore.put(catalog, ref) 

91 

92 # Does it exist? 

93 self.assertTrue(datastore.exists(ref)) 

94 

95 uri = datastore.getUri(ref) 

96 if self.fileExt is not None: 

97 self.assertTrue(uri.endswith(self.fileExt)) 

98 self.assertTrue(uri.startswith(self.uriScheme)) 

99 

100 # Get 

101 catalogOut = datastore.get(ref, parameters=None) 

102 self.assertCatalogEqual(catalog, catalogOut) 

103 

104 # These should raise 

105 ref = self.makeDatasetRef("calexp2", dimensions, storageClass, dataId) 

106 with self.assertRaises(FileNotFoundError): 

107 # non-existing file 

108 datastore.get(ref, parameters=None) 

109 

110 def testRemove(self): 

111 catalog = self.makeExampleCatalog() 

112 datastore = self.makeDatastore() 

113 

114 # Put 

115 storageClass = self.storageClassFactory.getStorageClass("SourceCatalog") 

116 dimensions = self.registry.dimensions.extract(("visit", "physical_filter")) 

117 dataId = {"visit": 1234567, "physical_filter": "blue", "instrument": "dummy"} 

118 

119 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId) 

120 datastore.put(catalog, ref) 

121 

122 # Does it exist? 

123 self.assertTrue(datastore.exists(ref)) 

124 

125 # Get 

126 catalogOut = datastore.get(ref) 

127 self.assertCatalogEqual(catalog, catalogOut) 

128 

129 # Remove 

130 datastore.remove(ref) 

131 

132 # Does it exist? 

133 self.assertFalse(datastore.exists(ref)) 

134 

135 # Get should now fail 

136 with self.assertRaises(FileNotFoundError): 

137 datastore.get(ref) 

138 # Can only delete once 

139 with self.assertRaises(FileNotFoundError): 

140 datastore.remove(ref) 

141 

142 def testTransfer(self): 

143 catalog = self.makeExampleCatalog() 

144 dimensions = self.registry.dimensions.extract(("visit", "physical_filter")) 

145 dataId = {"visit": 12345, "physical_filter": "red", "instrument": "dummy"} 

146 

147 storageClass = self.storageClassFactory.getStorageClass("SourceCatalog") 

148 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId) 

149 

150 inputDatastore = self.makeDatastore("test_input_datastore") 

151 outputDatastore = self.makeDatastore("test_output_datastore") 

152 

153 inputDatastore.put(catalog, ref) 

154 outputDatastore.transfer(inputDatastore, ref) 

155 

156 catalogOut = outputDatastore.get(ref) 

157 self.assertCatalogEqual(catalog, catalogOut) 

158 

159 def testExposurePutGet(self): 

160 example = os.path.join(self.testDir, "data", "basic", "small.fits") 

161 exposure = lsst.afw.image.ExposureF(example) 

162 datastore = self.makeDatastore() 

163 # Put 

164 dimensions = self.registry.dimensions.extract(("visit", "physical_filter")) 

165 dataId = {"visit": 231, "physical_filter": "Fc", "instrument": "dummy"} 

166 storageClass = datastore.storageClassFactory.getStorageClass("ExposureF") 

167 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId) 

168 

169 datastore.put(exposure, ref) 

170 

171 # Does it exist? 

172 self.assertTrue(datastore.exists(ref)) 

173 

174 # Get 

175 exposureOut = datastore.get(ref) 

176 self.assertEqual(type(exposure), type(exposureOut)) 

177 

178 # Get some components 

179 # Could not test the following components as they were not known: 

180 # bbox, xy0, filter, polygon, detector, extras, and exposureInfo 

181 for compName, isNone in (("wcs", False), 

182 ("image", False), 

183 ("mask", False), 

184 ("coaddInputs", False), 

185 ("psf", False), 

186 ("variance", False), 

187 ("photoCalib", False), 

188 ("metadata", False), 

189 ("visitInfo", False), 

190 ("apCorrMap", True), 

191 ("transmissionCurve", True), 

192 ("metadata", False)): 

193 with self.subTest(component=compName): 

194 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions, 

195 storageClass.components[compName], dataId, id=ref.id) 

196 component = datastore.get(compRef) 

197 if isNone: 

198 self.assertIsNone(component) 

199 else: 

200 self.assertIsInstance(component, compRef.datasetType.storageClass.pytype) 

201 

202 # Get the WCS component to check it 

203 wcsRef = self.makeDatasetRef(ref.datasetType.componentTypeName("wcs"), dimensions, 

204 storageClass.components["wcs"], dataId, id=ref.id) 

205 wcs = datastore.get(wcsRef) 

206 

207 # Simple check of WCS 

208 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), 

209 lsst.geom.Extent2I(9, 9)) 

210 self.assertWcsAlmostEqualOverBBox(wcs, exposure.getWcs(), bbox) 

211 

212 # Check basic metadata 

213 metadataRef = self.makeDatasetRef(ref.datasetType.componentTypeName("metadata"), dimensions, 

214 storageClass.components["metadata"], dataId, id=ref.id) 

215 metadata = datastore.get(metadataRef) 

216 self.assertEqual(metadata["WCS_ID"], 3) 

217 

218 def testExposureCompositePutGet(self): 

219 example = os.path.join(self.testDir, "data", "basic", "small.fits") 

220 exposure = lsst.afw.image.ExposureF(example) 

221 datastore = self.makeDatastore() 

222 # Put 

223 dimensions = self.registry.dimensions.extract(("visit", "physical_filter")) 

224 dataId = {"visit": 23, "physical_filter": "F", "instrument": "dummy"} 

225 storageClass = datastore.storageClassFactory.getStorageClass("ExposureCompositeF") 

226 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId) 

227 

228 # Get the predicted URI 

229 self.assertFalse(datastore.exists(ref)) 

230 uri = datastore.getUri(ref, predict=True) 

231 self.assertTrue(uri.endswith("#predicted")) 

232 

233 components = storageClass.assembler().disassemble(exposure) 

234 self.assertTrue(components) 

235 

236 # Get a component 

237 compsRead = {} 

238 for compName in ("wcs", "image", "mask", "coaddInputs", "psf"): 

239 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions, 

240 components[compName].storageClass, dataId) 

241 

242 datastore.put(components[compName].component, compRef) 

243 

244 # Does it exist? 

245 self.assertTrue(datastore.exists(compRef)) 

246 

247 component = datastore.get(compRef) 

248 self.assertIsInstance(component, compRef.datasetType.storageClass.pytype) 

249 compsRead[compName] = component 

250 

251 # Simple check of WCS 

252 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), 

253 lsst.geom.Extent2I(9, 9)) 

254 self.assertWcsAlmostEqualOverBBox(compsRead["wcs"], exposure.getWcs(), bbox) 

255 

256 # Try to reassemble the exposure 

257 retrievedExposure = storageClass.assembler().assemble(compsRead) 

258 self.assertIsInstance(retrievedExposure, type(exposure)) 

259 

260 

261class PosixDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase): 

262 """PosixDatastore specialization""" 

263 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

264 uriScheme = "file:" 

265 fileExt = ".fits" 

266 

267 def setUp(self): 

268 # Override the working directory before calling the base class 

269 self.root = tempfile.mkdtemp(dir=TESTDIR) 

270 super().setUp() 

271 

272 

273class InMemoryDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase): 

274 """PosixDatastore specialization""" 

275 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml") 

276 uriScheme = "mem:" 

277 fileExt = None 

278 

279 

280class ChainedDatastoreTestCase(PosixDatastoreTestCase): 

281 """PosixDatastore specialization""" 

282 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml") 

283 

284 

285if __name__ == "__main__": 285 ↛ 286line 285 didn't jump to line 286, because the condition on line 285 was never true

286 unittest.main()