Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import multiprocessing 

24import os 

25import shutil 

26import stat 

27import time 

28import unittest 

29import tempfile 

30 

31import lsst.daf.persistence as dp 

32import lsst.utils.tests 

33from lsst.log import Log 

34 

35 

36# Define the root of the tests relative to this file 

37ROOT = os.path.abspath(os.path.dirname(__file__)) 

38 

39 

40def setup_module(module): 

41 lsst.utils.tests.init() 

42 

43 

44class WriteOnceCompareSameTest(unittest.TestCase): 

45 

46 def setUp(self): 

47 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='WriteOnceCompareSameTest-') 

48 

49 def tearDown(self): 

50 if os.path.exists(self.testDir): 

51 shutil.rmtree(self.testDir) 

52 

53 def testCompareSame(self): 

54 

55 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f: 

56 f.write('bar\n') 

57 f.write('baz\n') 

58 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt'))) 

59 self.assertEqual(len(os.listdir(self.testDir)), 1) 

60 

61 # write the same file, verify the dir & file stay the same 

62 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f: 

63 f.write('bar\n') 

64 f.write('baz\n') 

65 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt'))) 

66 self.assertEqual(len(os.listdir(self.testDir)), 1) 

67 

68 def testCompareDifferent(self): 

69 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f: 

70 f.write('bar\n') 

71 f.write('baz\n') 

72 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt'))) 

73 self.assertEqual(len(os.listdir(self.testDir)), 1) 

74 

75 # write the same file, verify the dir & file stay the same 

76 def writeNonMatchingFile(): 

77 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f: 

78 f.write('boo\n') 

79 f.write('fop\n') 

80 self.assertRaises(RuntimeError, writeNonMatchingFile) 

81 

82 def testPermissions(self): 

83 """Check that the file is created with the current umask.""" 

84 # The only way to get the umask is to set it. 

85 umask = os.umask(0) 

86 os.umask(umask) 

87 

88 fileName = os.path.join(self.testDir, 'test.txt') 

89 with dp.safeFileIo.FileForWriteOnceCompareSame(fileName) as f: 

90 f.write('bar\n') 

91 f.write('baz\n') 

92 

93 filePerms = stat.S_IMODE(os.lstat(fileName).st_mode) 

94 self.assertEqual(~umask & 0o666, filePerms) 

95 

96 

97def readFile(filename, readQueue): 

98 readQueue.put("waiting") 

99 readQueue.get() 

100 with dp.safeFileIo.SafeLockedFileForRead(filename) as f: 

101 readQueue.put(f.read()) 

102 

103 

104class TestFileLocking(unittest.TestCase): 

105 """A test case for safeFileIo file read and write locking""" 

106 

107 def setUp(self): 

108 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='TestFileLocking-') 

109 

110 def tearDown(self): 

111 if os.path.exists(self.testDir): 

112 shutil.rmtree(self.testDir) 

113 

114 def testWriteLock(self): 

115 """Test SafeLockedFileForWrite by 

116 1. open a file for write 

117 2. spawn a second process that tries to read the file but should be blocked by the file lock 

118 3. then write the file it and closing it (in the first process) 

119 4. the second process should then be unblocked 

120 5. read the file in the second process and return the result to the first process 

121 6. compare what was written and read 

122 """ 

123 readQueue = multiprocessing.Queue() 

124 fileName = os.path.join(self.testDir, "testfile.txt") 

125 proc = multiprocessing.Process(target=readFile, args=(fileName, readQueue)) 

126 testStr = "foobarbaz" 

127 proc.start() 

128 self.assertEqual(readQueue.get(), "waiting") 

129 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f: 

130 readQueue.put("go") 

131 time.sleep(1) 

132 f.write(testStr) 

133 self.assertEqual(readQueue.get(), testStr) 

134 proc.join() 

135 

136 def testNoChange(self): 

137 """Test that if a file is opened and not changed that the file does not get changed""" 

138 fileName = os.path.join(self.testDir, "testfile.txt") 

139 # create the file with some contents 

140 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f: 

141 f.write("some test string") 

142 # open the file but do not change it 

143 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f: 

144 pass 

145 # open the file for read and test that it still contains the original test contents 

146 with dp.safeFileIo.SafeLockedFileForRead(fileName) as f: 

147 self.assertEqual(f.read(), "some test string") 

148 

149 

150class TestOneThousandWriters(unittest.TestCase): 

151 """Test for efficient file updating with shared & exclusive locks by serializing a RepositoryCfg to a 

152 location 1000 times. When this was tested on a 2.8 GHz Intel Core i7 macbook pro it took about 1.3 seconds 

153 to run. When the squash performance monitoring framework is done, this test could be monitored in that 

154 system.""" 

155 

156 def setUp(self): 

157 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='TestOneThousandWriters-') 

158 

159 def tearDown(self): 

160 if os.path.exists(self.testDir): 

161 shutil.rmtree(self.testDir) 

162 

163 @staticmethod 

164 def writeCfg(cfg, go): 

165 while go is False: 

166 pass 

167 dp.PosixStorage.putRepositoryCfg(cfg) 

168 

169 def testWriteCfg(self): 

170 # The number of writers to use can result in too many open files 

171 # We calculate this as the 80% of the maximum allowed number for this 

172 # process, or 1000, whichever is smaller. 

173 # Reduce the number on macOS due to the cost of multiprocessing spawn 

174 if os.uname()[0] == "Darwin": 

175 numWriters = 3 

176 else: 

177 numWriters = 1000 

178 try: 

179 import resource 

180 limit = resource.getrlimit(resource.RLIMIT_NOFILE) 

181 allowedOpen = int(limit[0] * 0.8) 

182 if allowedOpen < numWriters: 

183 numWriters = allowedOpen 

184 except Exception: 

185 # Use the default number if we had trouble obtaining resources 

186 pass 

187 startTime = time.time() 

188 go = multiprocessing.Value('b', False) 

189 cfg = dp.RepositoryCfg(root=os.path.join(self.testDir), mapper='bar', mapperArgs={}, 

190 parents=None, policy=None) 

191 procs = [multiprocessing.Process(target=TestOneThousandWriters.writeCfg, args=(cfg, go)) 

192 for x in range(numWriters)] 

193 for proc in procs: 

194 proc.start() 

195 go = True 

196 for proc in procs: 

197 proc.join() 

198 endTime = time.time() 

199 log = Log.getLogger("daf.persistence") 

200 log.trace("TestOneThousandWriters took {} seconds.".format(endTime-startTime)) 

201 

202 

203class MemoryTester(lsst.utils.tests.MemoryTestCase): 

204 pass 

205 

206 

207if __name__ == '__main__': 207 ↛ 208line 207 didn't jump to line 208, because the condition on line 207 was never true

208 lsst.utils.tests.init() 

209 unittest.main()