Coverage for tests/test_safeFileIo.py: 27%

112 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-13 02:46 -0700

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import multiprocessing 

24import os 

25import shutil 

26import stat 

27import time 

28import unittest 

29import tempfile 

30 

31import lsst.daf.persistence as dp 

32import lsst.utils.tests 

33from lsst.log import Log 

34 

35 

36# Define the root of the tests relative to this file 

37ROOT = os.path.abspath(os.path.dirname(__file__)) 

38 

39 

40def setup_module(module): 

41 lsst.utils.tests.init() 

42 

43 

44class WriteOnceCompareSameTest(unittest.TestCase): 

45 

46 def setUp(self): 

47 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='WriteOnceCompareSameTest-') 

48 

49 def tearDown(self): 

50 if os.path.exists(self.testDir): 

51 shutil.rmtree(self.testDir) 

52 

53 def testCompareSame(self): 

54 

55 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f: 

56 f.write('bar\n') 

57 f.write('baz\n') 

58 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt'))) 

59 self.assertEqual(len(os.listdir(self.testDir)), 1) 

60 

61 # write the same file, verify the dir & file stay the same 

62 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f: 

63 f.write('bar\n') 

64 f.write('baz\n') 

65 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt'))) 

66 self.assertEqual(len(os.listdir(self.testDir)), 1) 

67 

68 def testCompareDifferent(self): 

69 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f: 

70 f.write('bar\n') 

71 f.write('baz\n') 

72 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt'))) 

73 self.assertEqual(len(os.listdir(self.testDir)), 1) 

74 

75 # write the same file, verify the dir & file stay the same 

76 def writeNonMatchingFile(): 

77 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f: 

78 f.write('boo\n') 

79 f.write('fop\n') 

80 self.assertRaises(RuntimeError, writeNonMatchingFile) 

81 

82 def testPermissions(self): 

83 """Check that the file is created with the current umask.""" 

84 # The only way to get the umask is to set it. 

85 umask = os.umask(0) 

86 os.umask(umask) 

87 

88 fileName = os.path.join(self.testDir, 'test.txt') 

89 with dp.safeFileIo.FileForWriteOnceCompareSame(fileName) as f: 

90 f.write('bar\n') 

91 f.write('baz\n') 

92 

93 filePerms = stat.S_IMODE(os.lstat(fileName).st_mode) 

94 self.assertEqual(~umask & 0o666, filePerms) 

95 

96 

97def readFile(filename, readQueue): 

98 readQueue.put("waiting") 

99 readQueue.get() 

100 with dp.safeFileIo.SafeLockedFileForRead(filename) as f: 

101 readQueue.put(f.read()) 

102 

103 

104class TestFileLocking(unittest.TestCase): 

105 """A test case for safeFileIo file read and write locking""" 

106 

107 def setUp(self): 

108 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='TestFileLocking-') 

109 

110 def tearDown(self): 

111 if os.path.exists(self.testDir): 

112 shutil.rmtree(self.testDir) 

113 

114 def testWriteLock(self): 

115 """Test SafeLockedFileForWrite by 

116 1. open a file for write 

117 2. spawn a second process that tries to read the file but should be blocked by the file lock 

118 3. then write the file it and closing it (in the first process) 

119 4. the second process should then be unblocked 

120 5. read the file in the second process and return the result to the first process 

121 6. compare what was written and read 

122 """ 

123 readQueue = multiprocessing.Queue() 

124 fileName = os.path.join(self.testDir, "testfile.txt") 

125 proc = multiprocessing.Process(target=readFile, args=(fileName, readQueue)) 

126 testStr = "foobarbaz" 

127 proc.start() 

128 self.assertEqual(readQueue.get(), "waiting") 

129 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f: 

130 readQueue.put("go") 

131 time.sleep(1) 

132 f.write(testStr) 

133 self.assertEqual(readQueue.get(), testStr) 

134 proc.join() 

135 

136 def testNoChange(self): 

137 """Test that if a file is opened and not changed that the file does not get changed""" 

138 fileName = os.path.join(self.testDir, "testfile.txt") 

139 # create the file with some contents 

140 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f: 

141 f.write("some test string") 

142 # open the file but do not change it 

143 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f: 

144 pass 

145 # open the file for read and test that it still contains the original test contents 

146 with dp.safeFileIo.SafeLockedFileForRead(fileName) as f: 

147 self.assertEqual(f.read(), "some test string") 

148 

149 

150class TestMultipleWriters(unittest.TestCase): 

151 """Test for efficient file updating with shared & exclusive locks by 

152 serializing a RepositoryCfg to a location several times in parallel.""" 

153 

154 def setUp(self): 

155 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='TestMultipleWriters-') 

156 

157 def tearDown(self): 

158 if os.path.exists(self.testDir): 

159 shutil.rmtree(self.testDir) 

160 

161 @staticmethod 

162 def writeCfg(cfg, go): 

163 """Write a configuration file after waiting on a condition variable.""" 

164 while go is False: 

165 pass 

166 dp.PosixStorage.putRepositoryCfg(cfg) 

167 

168 def testWriteCfg(self): 

169 """Test parallel writes to a configuration file. 

170 

171 multiprocessing is used to spawn several writer function executions, 

172 all of which wait to be released by the condition variable "go". 

173 

174 There are no asserts here, so success is measured solely by not 

175 failing with an exception, but the time it took to do the writes can 

176 be logged as a potential performance metric. 

177 """ 

178 numWriters = 3 

179 startTime = time.time() 

180 go = multiprocessing.Value('b', False) 

181 cfg = dp.RepositoryCfg(root=os.path.join(self.testDir), mapper='bar', mapperArgs={}, 

182 parents=None, policy=None) 

183 procs = [multiprocessing.Process(target=TestMultipleWriters.writeCfg, args=(cfg, go)) 

184 for x in range(numWriters)] 

185 for proc in procs: 

186 proc.start() 

187 go = True 

188 for proc in procs: 

189 proc.join() 

190 endTime = time.time() 

191 log = Log.getLogger("daf.persistence") 

192 log.trace("TestMultipleWriters took {} seconds.".format(endTime-startTime)) 

193 

194 

195class MemoryTester(lsst.utils.tests.MemoryTestCase): 

196 pass 

197 

198 

199if __name__ == '__main__': 199 ↛ 200line 199 didn't jump to line 200, because the condition on line 199 was never true

200 lsst.utils.tests.init() 

201 unittest.main()