Coverage for tests/test_safeFileIo.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import multiprocessing
24import os
25import shutil
26import stat
27import time
28import unittest
29import tempfile
31import lsst.daf.persistence as dp
32import lsst.utils.tests
33from lsst.log import Log
36# Define the root of the tests relative to this file
37ROOT = os.path.abspath(os.path.dirname(__file__))
40def setup_module(module):
41 lsst.utils.tests.init()
44class WriteOnceCompareSameTest(unittest.TestCase):
46 def setUp(self):
47 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='WriteOnceCompareSameTest-')
49 def tearDown(self):
50 if os.path.exists(self.testDir):
51 shutil.rmtree(self.testDir)
53 def testCompareSame(self):
55 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f:
56 f.write('bar\n')
57 f.write('baz\n')
58 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt')))
59 self.assertEqual(len(os.listdir(self.testDir)), 1)
61 # write the same file, verify the dir & file stay the same
62 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f:
63 f.write('bar\n')
64 f.write('baz\n')
65 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt')))
66 self.assertEqual(len(os.listdir(self.testDir)), 1)
68 def testCompareDifferent(self):
69 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f:
70 f.write('bar\n')
71 f.write('baz\n')
72 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt')))
73 self.assertEqual(len(os.listdir(self.testDir)), 1)
75 # write the same file, verify the dir & file stay the same
76 def writeNonMatchingFile():
77 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f:
78 f.write('boo\n')
79 f.write('fop\n')
80 self.assertRaises(RuntimeError, writeNonMatchingFile)
82 def testPermissions(self):
83 """Check that the file is created with the current umask."""
84 # The only way to get the umask is to set it.
85 umask = os.umask(0)
86 os.umask(umask)
88 fileName = os.path.join(self.testDir, 'test.txt')
89 with dp.safeFileIo.FileForWriteOnceCompareSame(fileName) as f:
90 f.write('bar\n')
91 f.write('baz\n')
93 filePerms = stat.S_IMODE(os.lstat(fileName).st_mode)
94 self.assertEqual(~umask & 0o666, filePerms)
97def readFile(filename, readQueue):
98 readQueue.put("waiting")
99 readQueue.get()
100 with dp.safeFileIo.SafeLockedFileForRead(filename) as f:
101 readQueue.put(f.read())
104class TestFileLocking(unittest.TestCase):
105 """A test case for safeFileIo file read and write locking"""
107 def setUp(self):
108 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='TestFileLocking-')
110 def tearDown(self):
111 if os.path.exists(self.testDir):
112 shutil.rmtree(self.testDir)
114 def testWriteLock(self):
115 """Test SafeLockedFileForWrite by
116 1. open a file for write
117 2. spawn a second process that tries to read the file but should be blocked by the file lock
118 3. then write the file it and closing it (in the first process)
119 4. the second process should then be unblocked
120 5. read the file in the second process and return the result to the first process
121 6. compare what was written and read
122 """
123 readQueue = multiprocessing.Queue()
124 fileName = os.path.join(self.testDir, "testfile.txt")
125 proc = multiprocessing.Process(target=readFile, args=(fileName, readQueue))
126 testStr = "foobarbaz"
127 proc.start()
128 self.assertEqual(readQueue.get(), "waiting")
129 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f:
130 readQueue.put("go")
131 time.sleep(1)
132 f.write(testStr)
133 self.assertEqual(readQueue.get(), testStr)
134 proc.join()
136 def testNoChange(self):
137 """Test that if a file is opened and not changed that the file does not get changed"""
138 fileName = os.path.join(self.testDir, "testfile.txt")
139 # create the file with some contents
140 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f:
141 f.write("some test string")
142 # open the file but do not change it
143 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f:
144 pass
145 # open the file for read and test that it still contains the original test contents
146 with dp.safeFileIo.SafeLockedFileForRead(fileName) as f:
147 self.assertEqual(f.read(), "some test string")
150class TestOneThousandWriters(unittest.TestCase):
151 """Test for efficient file updating with shared & exclusive locks by serializing a RepositoryCfg to a
152 location 1000 times. When this was tested on a 2.8 GHz Intel Core i7 macbook pro it took about 1.3 seconds
153 to run. When the squash performance monitoring framework is done, this test could be monitored in that
154 system."""
156 def setUp(self):
157 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='TestOneThousandWriters-')
159 def tearDown(self):
160 if os.path.exists(self.testDir):
161 shutil.rmtree(self.testDir)
163 @staticmethod
164 def writeCfg(cfg, go):
165 while go is False:
166 pass
167 dp.PosixStorage.putRepositoryCfg(cfg)
169 def testWriteCfg(self):
170 # The number of writers to use can result in too many open files
171 # We calculate this as the 80% of the maximum allowed number for this
172 # process, or 1000, whichever is smaller.
173 numWriters = 1000
174 try:
175 import resource
176 limit = resource.getrlimit(resource.RLIMIT_NOFILE)
177 allowedOpen = int(limit[0] * 0.8)
178 if allowedOpen < numWriters:
179 numWriters = allowedOpen
180 except Exception:
181 # Use the default number if we had trouble obtaining resources
182 pass
183 startTime = time.time()
184 go = multiprocessing.Value('b', False)
185 cfg = dp.RepositoryCfg(root=os.path.join(self.testDir), mapper='bar', mapperArgs={},
186 parents=None, policy=None)
187 procs = [multiprocessing.Process(target=TestOneThousandWriters.writeCfg, args=(cfg, go))
188 for x in range(numWriters)]
189 for proc in procs:
190 proc.start()
191 go = True
192 for proc in procs:
193 proc.join()
194 endTime = time.time()
195 log = Log.getLogger("daf.persistence")
196 log.trace("TestOneThousandWriters took {} seconds.".format(endTime-startTime))
199class MemoryTester(lsst.utils.tests.MemoryTestCase):
200 pass
203if __name__ == '__main__': 203 ↛ 204line 203 didn't jump to line 204, because the condition on line 203 was never true
204 lsst.utils.tests.init()
205 unittest.main()