Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23import tempfile 

24import shutil 

25import pickle 

26import os 

27 

28import lsst.utils.tests 

29import lsst.geom as geom 

30import lsst.daf.persistence as dafPersist 

31import lsst.obs.base 

32 

33# Define paths used for testing 

34ROOT = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37def setup_module(module): 

38 lsst.utils.tests.init() 

39 

40 

41class MinMapper1(lsst.obs.base.CameraMapper): 

42 packageName = 'larry' 

43 

44 def __init__(self, **kwargs): 

45 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

46 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs) 

47 return 

48 

49 def std_x(self, item, dataId): 

50 return float(item) 

51 

52 @classmethod 

53 def getPackageDir(cls): 

54 return "/path/to/nowhere" 

55 

56 

57class OutputRootTestCase(unittest.TestCase): 

58 """A test case for output roots.""" 

59 

60 def setUp(self): 

61 self.tempDirs = {} 

62 

63 def tearDown(self): 

64 for d in self.tempDirs: 

65 shutil.rmtree(self.tempDirs[d]) 

66 

67 def mkdtemp(self, prefix): 

68 """Create a temporary directory and return its name. 

69 The resulting path is stored in a dict (self.tempDirs) with 

70 the supplied prefix as key. This allows the name to be retrieved 

71 later. The directory path is returned. 

72 A '-' is appended to the prefix if not there.""" 

73 dprefix = prefix 

74 if not dprefix.endswith('-'): 

75 dprefix = dprefix + '-' 

76 tempDir = tempfile.mkdtemp(dir=ROOT, prefix=dprefix) 

77 self.tempDirs[prefix] = tempDir 

78 return tempDir 

79 

80 def testCreateOutputRoot(self): 

81 """Create an input repository and a related output repo, and verify there is a parent relationship 

82 from the output repo to the input repo.""" 

83 testOutput = self.mkdtemp("testOutput") 

84 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}, 

85 outputs=testOutput) 

86 self.assertTrue(butler) 

87 self.assertTrue(os.path.exists(testOutput)) 

88 self.assertTrue(os.path.isdir(testOutput)) 

89 self.assertTrue(os.path.exists(os.path.join(testOutput, "repositoryCfg.yaml"))) 

90 cfg = dafPersist.PosixStorage.getRepositoryCfg(testOutput) 

91 expectedCfg = dafPersist.RepositoryCfg(root=ROOT, 

92 mapper=MinMapper1, 

93 mapperArgs=None, 

94 parents=None, 

95 policy=None) 

96 self.assertEqual(len(cfg.parents), 1) 

97 self.assertEqual(cfg.parents[0], expectedCfg) 

98 

99 def testParentNormal(self): 

100 """Test that an object can be found at root location and put into an output location. 

101 Then test that when the output locaiton is used as an input location, and with a new output location, 

102 that the object is found in the first output location. 

103 """ 

104 testOutput = self.mkdtemp("testOutput") 

105 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}, outputs=testOutput) 

106 mapper1 = butler._repos.inputs()[0].repo._mapper 

107 loc = mapper1.map("x", dict(sensor="1,1"), write=True) 

108 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

109 self.assertEqual(loc.getCppType(), "BoxI") 

110 self.assertEqual(loc.getStorageName(), "PickleStorage") 

111 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

112 self.assertEqual(loc.getAdditionalData().toString(), "sensor = \"1,1\"\n") 

113 box = geom.BoxI(geom.PointI(0, 1), geom.PointI(2, 3)) 

114 butler.put(box, "x", sensor="1,1") 

115 self.assertTrue(os.path.exists(os.path.join(testOutput, loc.getLocations()[0]))) 

116 del butler 

117 

118 testOutput2 = self.mkdtemp("testOutput2") 

119 butler = dafPersist.Butler(inputs={'root': testOutput, 'mapper': MinMapper1}, outputs=testOutput2) 

120 mapper2 = butler._repos.inputs()[0].repo._mapper 

121 loc = mapper2.map("x", dict(sensor="1,1")) 

122 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

123 self.assertEqual(loc.getCppType(), "BoxI") 

124 self.assertEqual(loc.getStorageName(), "PickleStorage") 

125 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

126 self.assertEqual(loc.getStorage().root, testOutput) 

127 self.assertEqual(loc.getAdditionalData().toString(), "sensor = \"1,1\"\n") 

128 

129 def testParentTrailingSlash2527(self): 

130 """Just like testParentNormal, but put a trailing slash on the root paths. 

131 Test that an object can be found at root location and put into an output location. 

132 Then test that when the output locaiton is used as an input location, and with a new output location, 

133 that the object is found in the first output location.""" 

134 # todo these shouldn't be commented out, I think the test wants the trailing slash. 

135 testOutput = self.mkdtemp("testOutput") + '/' 

136 butler = dafPersist.Butler(inputs={'root': ROOT + '/', 'mapper': MinMapper1}, 

137 outputs=testOutput) 

138 mapper1 = butler._repos.inputs()[0].repo._mapper 

139 loc = mapper1.map("x", dict(sensor="1,1"), write=True) 

140 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

141 self.assertEqual(loc.getCppType(), "BoxI") 

142 self.assertEqual(loc.getStorageName(), "PickleStorage") 

143 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

144 self.assertEqual(loc.getStorage().root, ROOT) 

145 self.assertEqual(loc.getAdditionalData().toString(), "sensor = \"1,1\"\n") 

146 box = geom.BoxI(geom.PointI(0, 1), geom.PointI(2, 3)) 

147 butler.put(box, "x", sensor="1,1") 

148 self.assertTrue(os.path.exists(os.path.join(testOutput, loc.getLocations()[0]))) 

149 del butler 

150 del mapper1 

151 

152 testOutput2 = self.mkdtemp("testOutput2") + '/' 

153 butler = dafPersist.Butler(inputs={'root': testOutput, 'mapper': MinMapper1}, 

154 outputs=testOutput2) 

155 mapper2 = butler._repos.inputs()[0].repo._mapper 

156 loc = mapper2.map("x", dict(sensor="1,1")) 

157 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

158 self.assertEqual(loc.getCppType(), "BoxI") 

159 self.assertEqual(loc.getStorageName(), "PickleStorage") 

160 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

161 self.assertEqual(os.path.normpath(loc.getStorage().root), 

162 os.path.normpath(testOutput)) 

163 self.assertEqual(loc.getAdditionalData().toString(), "sensor = \"1,1\"\n") 

164 

165 def testReuseOutputRoot(self): 

166 """Set up an output repositoriy and verify its parent relationship to the input repository. 

167 Then set up an output repository with the first output as an input, and verify the parent 

168 relationships.""" 

169 testOutput = self.mkdtemp("testOutput") 

170 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}, 

171 outputs=testOutput) 

172 self.assertTrue(os.path.exists(testOutput)) 

173 self.assertTrue(os.path.isdir(testOutput)) 

174 cfg = dafPersist.Storage().getRepositoryCfg(testOutput) 

175 expectedCfg = dafPersist.RepositoryCfg(root=ROOT, 

176 mapper=MinMapper1, 

177 mapperArgs=None, 

178 parents=None, 

179 policy=None) 

180 self.assertEqual(cfg.parents, [expectedCfg]) 

181 del butler 

182 

183 testOutput2 = self.mkdtemp("testOutput2") 

184 butler = dafPersist.Butler(inputs={'root': testOutput, 'mapper': MinMapper1}, 

185 outputs=testOutput2) 

186 self.assertTrue(os.path.exists(testOutput2)) 

187 self.assertTrue(os.path.isdir(testOutput2)) 

188 cfg = dafPersist.Storage().getRepositoryCfg(testOutput2) 

189 self.assertEqual(cfg.parents, [testOutput]) 

190 del butler 

191 

192 def testDiffInput(self): 

193 """Verify that if an output repository is loaded/created twice, and the second time it has a different 

194 parent than the first time, then the second instantiation should raise an exception.""" 

195 testInput1 = self.mkdtemp("testInput1") 

196 butler = dafPersist.Butler(outputs={'root': testInput1, 'mapper': MinMapper1}) 

197 del butler 

198 testInput2 = self.mkdtemp("testInput2") 

199 butler = dafPersist.Butler(outputs={'root': testInput2, 'mapper': MinMapper1}) 

200 del butler 

201 testOutput = self.mkdtemp("testOutput") 

202 butler = dafPersist.Butler(inputs=testInput1, outputs=testOutput) 

203 del butler 

204 # should raise: 

205 with self.assertRaises(RuntimeError): 

206 butler = dafPersist.Butler(inputs=testInput2, outputs=testOutput) 

207 del butler 

208 

209 @unittest.expectedFailure # this is flagged to be fixed in DM-9048 

210 def testBackup(self): 

211 testOutput = self.mkdtemp("testOutput") 

212 mapper1 = MinMapper1(outputRoot=testOutput) 

213 butler1 = dafPersist.Butler(outputs=dafPersist.RepositoryArgs(mode='w', 

214 root=testOutput, 

215 mapper=mapper1)) 

216 b1 = geom.Box2I(geom.Point2I(3, 4), geom.Point2I(7, 6)) 

217 butler1.put(b1, "x") 

218 self.assertTrue(os.path.exists(os.path.join(testOutput, "foo-1,1.pickle"))) 

219 b2 = geom.Box2I(b1) 

220 b2.grow(1) 

221 butler1.put(b2, "x", doBackup=True) 

222 self.assertTrue(os.path.exists(os.path.join(testOutput, "foo-1,1.pickle"))) 

223 self.assertTrue(os.path.exists(os.path.join(testOutput, "foo-1,1.pickle~1"))) 

224 testOutput2 = self.mkdtemp("testOutput2") 

225 mapper2 = MinMapper1(root=testOutput, outputRoot=testOutput2) 

226 butler2 = dafPersist.Butler( 

227 # MinMapper is a little unconventional in that it takes its root and output root as separate 

228 # arguments, meaning that in effect it's a mapper for 2 different repositories 

229 outputs=dafPersist.RepositoryArgs( 

230 mode='rw', 

231 root=testOutput2, 

232 mapper=mapper2), 

233 ) 

234 b3 = geom.Box2I(b2) 

235 b3.grow(1) 

236 butler2.put(b3, "x", doBackup=True) 

237 

238 self.assertTrue(os.path.exists(os.path.join(testOutput2, "foo-1,1.pickle"))) 

239 self.assertTrue(os.path.exists(os.path.join(testOutput2, "foo-1,1.pickle~1"))) 

240 self.assertTrue(os.path.exists(os.path.join(testOutput2, "foo-1,1.pickle~2"))) 

241 with open(os.path.join(testOutput2, "foo-1,1.pickle~2"), 'rb') as f: 

242 b1Check = pickle.load(f) 

243 self.assertEqual(b1Check, b1) 

244 with open(os.path.join(testOutput2, "foo-1,1.pickle~1"), 'rb') as f: 

245 b2Check = pickle.load(f) 

246 self.assertEqual(b2Check, b2) 

247 with open(os.path.join(testOutput2, "foo-1,1.pickle"), 'rb') as f: 

248 b3Check = pickle.load(f) 

249 self.assertEqual(b3Check, b3) 

250 

251 

252class MemoryTester(lsst.utils.tests.MemoryTestCase): 

253 pass 

254 

255 

256if __name__ == '__main__': 256 ↛ 257line 256 didn't jump to line 257, because the condition on line 256 was never true

257 lsst.utils.tests.init() 

258 unittest.main()