Coverage for tests/test_outputRoot.py: 21%

162 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-14 02:56 -0700

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import pickle 

24import shutil 

25import tempfile 

26import unittest 

27 

28import lsst.daf.persistence as dafPersist 

29import lsst.geom as geom 

30import lsst.obs.base 

31import lsst.utils.tests 

32 

33# Define paths used for testing 

34ROOT = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37def setup_module(module): 

38 lsst.utils.tests.init() 

39 

40 

41class MinMapper1(lsst.obs.base.CameraMapper): 

42 packageName = "larry" 

43 

44 def __init__(self, **kwargs): 

45 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

46 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs) 

47 return 

48 

49 def std_x(self, item, dataId): 

50 return float(item) 

51 

52 @classmethod 

53 def getPackageDir(cls): 

54 return "/path/to/nowhere" 

55 

56 

57class OutputRootTestCase(unittest.TestCase): 

58 """A test case for output roots.""" 

59 

60 def setUp(self): 

61 self.tempDirs = {} 

62 

63 def tearDown(self): 

64 for d in self.tempDirs: 

65 shutil.rmtree(self.tempDirs[d]) 

66 

67 def mkdtemp(self, prefix): 

68 """Create a temporary directory and return its name. 

69 The resulting path is stored in a dict (self.tempDirs) with 

70 the supplied prefix as key. This allows the name to be retrieved 

71 later. The directory path is returned. 

72 A '-' is appended to the prefix if not there.""" 

73 dprefix = prefix 

74 if not dprefix.endswith("-"): 

75 dprefix = dprefix + "-" 

76 tempDir = tempfile.mkdtemp(dir=ROOT, prefix=dprefix) 

77 self.tempDirs[prefix] = tempDir 

78 return tempDir 

79 

80 def testCreateOutputRoot(self): 

81 """Create an input repository and a related output repo, and verify 

82 there is a parent relationship from the output repo to the input repo. 

83 """ 

84 testOutput = self.mkdtemp("testOutput") 

85 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}, outputs=testOutput) 

86 self.assertTrue(butler) 

87 self.assertTrue(os.path.exists(testOutput)) 

88 self.assertTrue(os.path.isdir(testOutput)) 

89 self.assertTrue(os.path.exists(os.path.join(testOutput, "repositoryCfg.yaml"))) 

90 cfg = dafPersist.PosixStorage.getRepositoryCfg(testOutput) 

91 expectedCfg = dafPersist.RepositoryCfg( 

92 root=ROOT, mapper=MinMapper1, mapperArgs=None, parents=None, policy=None 

93 ) 

94 self.assertEqual(len(cfg.parents), 1) 

95 self.assertEqual(cfg.parents[0], expectedCfg) 

96 

97 def testParentNormal(self): 

98 """Test that an object can be found at root location and put into an 

99 output location. Then test that when the output locaiton is used as an 

100 input location, and with a new output location, that the object is 

101 found in the first output location. 

102 """ 

103 testOutput = self.mkdtemp("testOutput") 

104 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}, outputs=testOutput) 

105 mapper1 = butler._repos.inputs()[0].repo._mapper 

106 loc = mapper1.map("x", dict(sensor="1,1"), write=True) 

107 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

108 self.assertEqual(loc.getCppType(), "BoxI") 

109 self.assertEqual(loc.getStorageName(), "PickleStorage") 

110 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

111 self.assertEqual(loc.getAdditionalData().toString(), 'sensor = "1,1"\n') 

112 box = geom.BoxI(geom.PointI(0, 1), geom.PointI(2, 3)) 

113 butler.put(box, "x", sensor="1,1") 

114 self.assertTrue(os.path.exists(os.path.join(testOutput, loc.getLocations()[0]))) 

115 del butler 

116 

117 testOutput2 = self.mkdtemp("testOutput2") 

118 butler = dafPersist.Butler(inputs={"root": testOutput, "mapper": MinMapper1}, outputs=testOutput2) 

119 mapper2 = butler._repos.inputs()[0].repo._mapper 

120 loc = mapper2.map("x", dict(sensor="1,1")) 

121 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

122 self.assertEqual(loc.getCppType(), "BoxI") 

123 self.assertEqual(loc.getStorageName(), "PickleStorage") 

124 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

125 self.assertEqual(loc.getStorage().root, testOutput) 

126 self.assertEqual(loc.getAdditionalData().toString(), 'sensor = "1,1"\n') 

127 

128 def testParentTrailingSlash2527(self): 

129 """Just like testParentNormal, but put a trailing slash on the root 

130 paths. 

131 

132 Test that an object can be found at root location and put into an 

133 output location. Then test that when the output locaiton is used as 

134 an input location, and with a new output location, that the object is 

135 found in the first output location.""" 

136 # todo these shouldn't be commented out, I think the test wants the 

137 # trailing slash. 

138 testOutput = self.mkdtemp("testOutput") + "/" 

139 butler = dafPersist.Butler(inputs={"root": ROOT + "/", "mapper": MinMapper1}, outputs=testOutput) 

140 mapper1 = butler._repos.inputs()[0].repo._mapper 

141 loc = mapper1.map("x", dict(sensor="1,1"), write=True) 

142 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

143 self.assertEqual(loc.getCppType(), "BoxI") 

144 self.assertEqual(loc.getStorageName(), "PickleStorage") 

145 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

146 self.assertEqual(loc.getStorage().root, ROOT) 

147 self.assertEqual(loc.getAdditionalData().toString(), 'sensor = "1,1"\n') 

148 box = geom.BoxI(geom.PointI(0, 1), geom.PointI(2, 3)) 

149 butler.put(box, "x", sensor="1,1") 

150 self.assertTrue(os.path.exists(os.path.join(testOutput, loc.getLocations()[0]))) 

151 del butler 

152 del mapper1 

153 

154 testOutput2 = self.mkdtemp("testOutput2") + "/" 

155 butler = dafPersist.Butler(inputs={"root": testOutput, "mapper": MinMapper1}, outputs=testOutput2) 

156 mapper2 = butler._repos.inputs()[0].repo._mapper 

157 loc = mapper2.map("x", dict(sensor="1,1")) 

158 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

159 self.assertEqual(loc.getCppType(), "BoxI") 

160 self.assertEqual(loc.getStorageName(), "PickleStorage") 

161 self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"]) 

162 self.assertEqual(os.path.normpath(loc.getStorage().root), os.path.normpath(testOutput)) 

163 self.assertEqual(loc.getAdditionalData().toString(), 'sensor = "1,1"\n') 

164 

165 def testReuseOutputRoot(self): 

166 """Set up an output repositoriy and verify its parent relationship to 

167 the input repository. 

168 

169 Then set up an output repository with the first output as an input, 

170 and verify the parent relationships. 

171 """ 

172 testOutput = self.mkdtemp("testOutput") 

173 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}, outputs=testOutput) 

174 self.assertTrue(os.path.exists(testOutput)) 

175 self.assertTrue(os.path.isdir(testOutput)) 

176 cfg = dafPersist.Storage().getRepositoryCfg(testOutput) 

177 expectedCfg = dafPersist.RepositoryCfg( 

178 root=ROOT, mapper=MinMapper1, mapperArgs=None, parents=None, policy=None 

179 ) 

180 self.assertEqual(cfg.parents, [expectedCfg]) 

181 del butler 

182 

183 testOutput2 = self.mkdtemp("testOutput2") 

184 butler = dafPersist.Butler(inputs={"root": testOutput, "mapper": MinMapper1}, outputs=testOutput2) 

185 self.assertTrue(os.path.exists(testOutput2)) 

186 self.assertTrue(os.path.isdir(testOutput2)) 

187 cfg = dafPersist.Storage().getRepositoryCfg(testOutput2) 

188 self.assertEqual(cfg.parents, [testOutput]) 

189 del butler 

190 

191 def testDiffInput(self): 

192 """Verify that if an output repository is loaded/created twice, and the 

193 second time it has a different parent than the first time, then the 

194 second instantiation should raise an exception. 

195 """ 

196 testInput1 = self.mkdtemp("testInput1") 

197 butler = dafPersist.Butler(outputs={"root": testInput1, "mapper": MinMapper1}) 

198 del butler 

199 testInput2 = self.mkdtemp("testInput2") 

200 butler = dafPersist.Butler(outputs={"root": testInput2, "mapper": MinMapper1}) 

201 del butler 

202 testOutput = self.mkdtemp("testOutput") 

203 butler = dafPersist.Butler(inputs=testInput1, outputs=testOutput) 

204 del butler 

205 # should raise: 

206 with self.assertRaises(RuntimeError): 

207 butler = dafPersist.Butler(inputs=testInput2, outputs=testOutput) 

208 del butler 

209 

210 @unittest.expectedFailure # this is flagged to be fixed in DM-9048 

211 def testBackup(self): 

212 testOutput = self.mkdtemp("testOutput") 

213 mapper1 = MinMapper1(outputRoot=testOutput) 

214 butler1 = dafPersist.Butler( 

215 outputs=dafPersist.RepositoryArgs(mode="w", root=testOutput, mapper=mapper1) 

216 ) 

217 b1 = geom.Box2I(geom.Point2I(3, 4), geom.Point2I(7, 6)) 

218 butler1.put(b1, "x") 

219 self.assertTrue(os.path.exists(os.path.join(testOutput, "foo-1,1.pickle"))) 

220 b2 = geom.Box2I(b1) 

221 b2.grow(1) 

222 butler1.put(b2, "x", doBackup=True) 

223 self.assertTrue(os.path.exists(os.path.join(testOutput, "foo-1,1.pickle"))) 

224 self.assertTrue(os.path.exists(os.path.join(testOutput, "foo-1,1.pickle~1"))) 

225 testOutput2 = self.mkdtemp("testOutput2") 

226 mapper2 = MinMapper1(root=testOutput, outputRoot=testOutput2) 

227 butler2 = dafPersist.Butler( 

228 # MinMapper is a little unconventional in that it takes its root 

229 # and output root as separate arguments, meaning that in effect 

230 # it's a mapper for 2 different repositories 

231 outputs=dafPersist.RepositoryArgs(mode="rw", root=testOutput2, mapper=mapper2), 

232 ) 

233 b3 = geom.Box2I(b2) 

234 b3.grow(1) 

235 butler2.put(b3, "x", doBackup=True) 

236 

237 self.assertTrue(os.path.exists(os.path.join(testOutput2, "foo-1,1.pickle"))) 

238 self.assertTrue(os.path.exists(os.path.join(testOutput2, "foo-1,1.pickle~1"))) 

239 self.assertTrue(os.path.exists(os.path.join(testOutput2, "foo-1,1.pickle~2"))) 

240 with open(os.path.join(testOutput2, "foo-1,1.pickle~2"), "rb") as f: 

241 b1Check = pickle.load(f) 

242 self.assertEqual(b1Check, b1) 

243 with open(os.path.join(testOutput2, "foo-1,1.pickle~1"), "rb") as f: 

244 b2Check = pickle.load(f) 

245 self.assertEqual(b2Check, b2) 

246 with open(os.path.join(testOutput2, "foo-1,1.pickle"), "rb") as f: 

247 b3Check = pickle.load(f) 

248 self.assertEqual(b3Check, b3) 

249 

250 

251class MemoryTester(lsst.utils.tests.MemoryTestCase): 

252 pass 

253 

254 

255if __name__ == "__main__": 255 ↛ 256line 255 didn't jump to line 256, because the condition on line 255 was never true

256 lsst.utils.tests.init() 

257 unittest.main()