Coverage for tests/test_outputRoot.py: 19%

162 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:01 -0800

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23import tempfile 

24import shutil 

25import pickle 

26import os 

27 

28import lsst.utils.tests 

29import lsst.geom as geom 

30import lsst.daf.persistence as dafPersist 

31import lsst.obs.base 

32 

33# Define paths used for testing 

34ROOT = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37def setup_module(module): 

38 lsst.utils.tests.init() 

39 

40 

41class MinMapper1(lsst.obs.base.CameraMapper): 

42 packageName = 'larry' 

43 

44 def __init__(self, **kwargs): 

45 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

46 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs) 

47 return 

48 

49 def std_x(self, item, dataId): 

50 return float(item) 

51 

52 @classmethod 

53 def getPackageDir(cls): 

54 return "/path/to/nowhere" 

55 

56 

57class OutputRootTestCase(unittest.TestCase): 

58 """A test case for output roots.""" 

59 

60 def setUp(self): 

61 self.tempDirs = {} 

62 

63 def tearDown(self): 

64 for d in self.tempDirs: 

65 shutil.rmtree(self.tempDirs[d]) 

66 

67 def mkdtemp(self, prefix): 

68 """Create a temporary directory and return its name. 

69 The resulting path is stored in a dict (self.tempDirs) with 

70 the supplied prefix as key. This allows the name to be retrieved 

71 later. The directory path is returned. 

72 A '-' is appended to the prefix if not there.""" 

73 dprefix = prefix 

74 if not dprefix.endswith('-'): 

75 dprefix = dprefix + '-' 

76 tempDir = tempfile.mkdtemp(dir=ROOT, prefix=dprefix) 

77 self.tempDirs[prefix] = tempDir 

78 return tempDir 

79 

80 def testCreateOutputRoot(self): 

81 """Create an input repository and a related output repo, and verify 

82 there is a parent relationship from the output repo to the input repo. 

83 """ 

84 testOutput = self.mkdtemp("testOutput") 

85 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}, 

86 outputs=testOutput) 

87 self.assertTrue(butler) 

88 self.assertTrue(os.path.exists(testOutput)) 

89 self.assertTrue(os.path.isdir(testOutput)) 

90 self.assertTrue(os.path.exists(os.path.join(testOutput, "repositoryCfg.yaml"))) 

91 cfg = dafPersist.PosixStorage.getRepositoryCfg(testOutput) 

92 expectedCfg = dafPersist.RepositoryCfg(root=ROOT, 

93 mapper=MinMapper1, 

94 mapperArgs=None, 

95 parents=None, 

96 policy=None) 

97 self.assertEqual(len(cfg.parents), 1) 

98 self.assertEqual(cfg.parents[0], expectedCfg) 

99 

100 def testParentNormal(self): 

101 """Test that an object can be found at root location and put into an 

102 output location. Then test that when the output locaiton is used as an 

103 input location, and with a new output location, that the object is 

104 found in the first output location. 

105 """ 

106 testOutput = self.mkdtemp("testOutput") 

107 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}, outputs=testOutput) 

108 mapper1 = butler._repos.inputs()[0].repo._mapper 

109 loc = mapper1.map("x", dict(sensor="1,1"), write=True) 

110 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

111 self.assertEqual(loc.getCppType(), "BoxI") 

112 self.assertEqual(loc.getStorageName(), "PickleStorage") 

113 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

114 self.assertEqual(loc.getAdditionalData().toString(), "sensor = \"1,1\"\n") 

115 box = geom.BoxI(geom.PointI(0, 1), geom.PointI(2, 3)) 

116 butler.put(box, "x", sensor="1,1") 

117 self.assertTrue(os.path.exists(os.path.join(testOutput, loc.getLocations()[0]))) 

118 del butler 

119 

120 testOutput2 = self.mkdtemp("testOutput2") 

121 butler = dafPersist.Butler(inputs={'root': testOutput, 'mapper': MinMapper1}, outputs=testOutput2) 

122 mapper2 = butler._repos.inputs()[0].repo._mapper 

123 loc = mapper2.map("x", dict(sensor="1,1")) 

124 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

125 self.assertEqual(loc.getCppType(), "BoxI") 

126 self.assertEqual(loc.getStorageName(), "PickleStorage") 

127 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

128 self.assertEqual(loc.getStorage().root, testOutput) 

129 self.assertEqual(loc.getAdditionalData().toString(), "sensor = \"1,1\"\n") 

130 

131 def testParentTrailingSlash2527(self): 

132 """Just like testParentNormal, but put a trailing slash on the root 

133 paths. 

134 

135 Test that an object can be found at root location and put into an 

136 output location. Then test that when the output locaiton is used as 

137 an input location, and with a new output location, that the object is 

138 found in the first output location.""" 

139 # todo these shouldn't be commented out, I think the test wants the 

140 # trailing slash. 

141 testOutput = self.mkdtemp("testOutput") + '/' 

142 butler = dafPersist.Butler(inputs={'root': ROOT + '/', 'mapper': MinMapper1}, 

143 outputs=testOutput) 

144 mapper1 = butler._repos.inputs()[0].repo._mapper 

145 loc = mapper1.map("x", dict(sensor="1,1"), write=True) 

146 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

147 self.assertEqual(loc.getCppType(), "BoxI") 

148 self.assertEqual(loc.getStorageName(), "PickleStorage") 

149 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

150 self.assertEqual(loc.getStorage().root, ROOT) 

151 self.assertEqual(loc.getAdditionalData().toString(), "sensor = \"1,1\"\n") 

152 box = geom.BoxI(geom.PointI(0, 1), geom.PointI(2, 3)) 

153 butler.put(box, "x", sensor="1,1") 

154 self.assertTrue(os.path.exists(os.path.join(testOutput, loc.getLocations()[0]))) 

155 del butler 

156 del mapper1 

157 

158 testOutput2 = self.mkdtemp("testOutput2") + '/' 

159 butler = dafPersist.Butler(inputs={'root': testOutput, 'mapper': MinMapper1}, 

160 outputs=testOutput2) 

161 mapper2 = butler._repos.inputs()[0].repo._mapper 

162 loc = mapper2.map("x", dict(sensor="1,1")) 

163 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

164 self.assertEqual(loc.getCppType(), "BoxI") 

165 self.assertEqual(loc.getStorageName(), "PickleStorage") 

166 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

167 self.assertEqual(os.path.normpath(loc.getStorage().root), 

168 os.path.normpath(testOutput)) 

169 self.assertEqual(loc.getAdditionalData().toString(), "sensor = \"1,1\"\n") 

170 

171 def testReuseOutputRoot(self): 

172 """Set up an output repositoriy and verify its parent relationship to 

173 the input repository. 

174 

175 Then set up an output repository with the first output as an input, 

176 and verify the parent relationships. 

177 """ 

178 testOutput = self.mkdtemp("testOutput") 

179 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}, 

180 outputs=testOutput) 

181 self.assertTrue(os.path.exists(testOutput)) 

182 self.assertTrue(os.path.isdir(testOutput)) 

183 cfg = dafPersist.Storage().getRepositoryCfg(testOutput) 

184 expectedCfg = dafPersist.RepositoryCfg(root=ROOT, 

185 mapper=MinMapper1, 

186 mapperArgs=None, 

187 parents=None, 

188 policy=None) 

189 self.assertEqual(cfg.parents, [expectedCfg]) 

190 del butler 

191 

192 testOutput2 = self.mkdtemp("testOutput2") 

193 butler = dafPersist.Butler(inputs={'root': testOutput, 'mapper': MinMapper1}, 

194 outputs=testOutput2) 

195 self.assertTrue(os.path.exists(testOutput2)) 

196 self.assertTrue(os.path.isdir(testOutput2)) 

197 cfg = dafPersist.Storage().getRepositoryCfg(testOutput2) 

198 self.assertEqual(cfg.parents, [testOutput]) 

199 del butler 

200 

201 def testDiffInput(self): 

202 """Verify that if an output repository is loaded/created twice, and the 

203 second time it has a different parent than the first time, then the 

204 second instantiation should raise an exception. 

205 """ 

206 testInput1 = self.mkdtemp("testInput1") 

207 butler = dafPersist.Butler(outputs={'root': testInput1, 'mapper': MinMapper1}) 

208 del butler 

209 testInput2 = self.mkdtemp("testInput2") 

210 butler = dafPersist.Butler(outputs={'root': testInput2, 'mapper': MinMapper1}) 

211 del butler 

212 testOutput = self.mkdtemp("testOutput") 

213 butler = dafPersist.Butler(inputs=testInput1, outputs=testOutput) 

214 del butler 

215 # should raise: 

216 with self.assertRaises(RuntimeError): 

217 butler = dafPersist.Butler(inputs=testInput2, outputs=testOutput) 

218 del butler 

219 

220 @unittest.expectedFailure # this is flagged to be fixed in DM-9048 

221 def testBackup(self): 

222 testOutput = self.mkdtemp("testOutput") 

223 mapper1 = MinMapper1(outputRoot=testOutput) 

224 butler1 = dafPersist.Butler(outputs=dafPersist.RepositoryArgs(mode='w', 

225 root=testOutput, 

226 mapper=mapper1)) 

227 b1 = geom.Box2I(geom.Point2I(3, 4), geom.Point2I(7, 6)) 

228 butler1.put(b1, "x") 

229 self.assertTrue(os.path.exists(os.path.join(testOutput, "foo-1,1.pickle"))) 

230 b2 = geom.Box2I(b1) 

231 b2.grow(1) 

232 butler1.put(b2, "x", doBackup=True) 

233 self.assertTrue(os.path.exists(os.path.join(testOutput, "foo-1,1.pickle"))) 

234 self.assertTrue(os.path.exists(os.path.join(testOutput, "foo-1,1.pickle~1"))) 

235 testOutput2 = self.mkdtemp("testOutput2") 

236 mapper2 = MinMapper1(root=testOutput, outputRoot=testOutput2) 

237 butler2 = dafPersist.Butler( 

238 # MinMapper is a little unconventional in that it takes its root 

239 # and output root as separate arguments, meaning that in effect 

240 # it's a mapper for 2 different repositories 

241 outputs=dafPersist.RepositoryArgs( 

242 mode='rw', 

243 root=testOutput2, 

244 mapper=mapper2), 

245 ) 

246 b3 = geom.Box2I(b2) 

247 b3.grow(1) 

248 butler2.put(b3, "x", doBackup=True) 

249 

250 self.assertTrue(os.path.exists(os.path.join(testOutput2, "foo-1,1.pickle"))) 

251 self.assertTrue(os.path.exists(os.path.join(testOutput2, "foo-1,1.pickle~1"))) 

252 self.assertTrue(os.path.exists(os.path.join(testOutput2, "foo-1,1.pickle~2"))) 

253 with open(os.path.join(testOutput2, "foo-1,1.pickle~2"), 'rb') as f: 

254 b1Check = pickle.load(f) 

255 self.assertEqual(b1Check, b1) 

256 with open(os.path.join(testOutput2, "foo-1,1.pickle~1"), 'rb') as f: 

257 b2Check = pickle.load(f) 

258 self.assertEqual(b2Check, b2) 

259 with open(os.path.join(testOutput2, "foo-1,1.pickle"), 'rb') as f: 

260 b3Check = pickle.load(f) 

261 self.assertEqual(b3Check, b3) 

262 

263 

264class MemoryTester(lsst.utils.tests.MemoryTestCase): 

265 pass 

266 

267 

268if __name__ == '__main__': 268 ↛ 269line 268 didn't jump to line 269, because the condition on line 268 was never true

269 lsst.utils.tests.init() 

270 unittest.main()