Coverage for tests/test_safeFileIo.py: 27%
112 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-23 02:39 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-23 02:39 -0700
1#
2# LSST Data Management System
3# Copyright 2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import multiprocessing
24import os
25import shutil
26import stat
27import time
28import unittest
29import tempfile
31import lsst.daf.persistence as dp
32import lsst.utils.tests
33from lsst.log import Log
36# Define the root of the tests relative to this file
37ROOT = os.path.abspath(os.path.dirname(__file__))
40def setup_module(module):
41 lsst.utils.tests.init()
44class WriteOnceCompareSameTest(unittest.TestCase):
46 def setUp(self):
47 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='WriteOnceCompareSameTest-')
49 def tearDown(self):
50 if os.path.exists(self.testDir):
51 shutil.rmtree(self.testDir)
53 def testCompareSame(self):
55 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f:
56 f.write('bar\n')
57 f.write('baz\n')
58 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt')))
59 self.assertEqual(len(os.listdir(self.testDir)), 1)
61 # write the same file, verify the dir & file stay the same
62 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f:
63 f.write('bar\n')
64 f.write('baz\n')
65 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt')))
66 self.assertEqual(len(os.listdir(self.testDir)), 1)
68 def testCompareDifferent(self):
69 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f:
70 f.write('bar\n')
71 f.write('baz\n')
72 self.assertTrue(os.path.exists(os.path.join(self.testDir, 'test.txt')))
73 self.assertEqual(len(os.listdir(self.testDir)), 1)
75 # write the same file, verify the dir & file stay the same
76 def writeNonMatchingFile():
77 with dp.safeFileIo.FileForWriteOnceCompareSame(os.path.join(self.testDir, 'test.txt')) as f:
78 f.write('boo\n')
79 f.write('fop\n')
80 self.assertRaises(RuntimeError, writeNonMatchingFile)
82 def testPermissions(self):
83 """Check that the file is created with the current umask."""
84 # The only way to get the umask is to set it.
85 umask = os.umask(0)
86 os.umask(umask)
88 fileName = os.path.join(self.testDir, 'test.txt')
89 with dp.safeFileIo.FileForWriteOnceCompareSame(fileName) as f:
90 f.write('bar\n')
91 f.write('baz\n')
93 filePerms = stat.S_IMODE(os.lstat(fileName).st_mode)
94 self.assertEqual(~umask & 0o666, filePerms)
97def readFile(filename, readQueue):
98 readQueue.put("waiting")
99 readQueue.get()
100 with dp.safeFileIo.SafeLockedFileForRead(filename) as f:
101 readQueue.put(f.read())
104class TestFileLocking(unittest.TestCase):
105 """A test case for safeFileIo file read and write locking"""
107 def setUp(self):
108 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='TestFileLocking-')
110 def tearDown(self):
111 if os.path.exists(self.testDir):
112 shutil.rmtree(self.testDir)
114 def testWriteLock(self):
115 """Test SafeLockedFileForWrite by
116 1. open a file for write
117 2. spawn a second process that tries to read the file but should be blocked by the file lock
118 3. then write the file it and closing it (in the first process)
119 4. the second process should then be unblocked
120 5. read the file in the second process and return the result to the first process
121 6. compare what was written and read
122 """
123 readQueue = multiprocessing.Queue()
124 fileName = os.path.join(self.testDir, "testfile.txt")
125 proc = multiprocessing.Process(target=readFile, args=(fileName, readQueue))
126 testStr = "foobarbaz"
127 proc.start()
128 self.assertEqual(readQueue.get(), "waiting")
129 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f:
130 readQueue.put("go")
131 time.sleep(1)
132 f.write(testStr)
133 self.assertEqual(readQueue.get(), testStr)
134 proc.join()
136 def testNoChange(self):
137 """Test that if a file is opened and not changed that the file does not get changed"""
138 fileName = os.path.join(self.testDir, "testfile.txt")
139 # create the file with some contents
140 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f:
141 f.write("some test string")
142 # open the file but do not change it
143 with dp.safeFileIo.SafeLockedFileForWrite(fileName) as f:
144 pass
145 # open the file for read and test that it still contains the original test contents
146 with dp.safeFileIo.SafeLockedFileForRead(fileName) as f:
147 self.assertEqual(f.read(), "some test string")
150class TestMultipleWriters(unittest.TestCase):
151 """Test for efficient file updating with shared & exclusive locks by
152 serializing a RepositoryCfg to a location several times in parallel."""
154 def setUp(self):
155 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='TestMultipleWriters-')
157 def tearDown(self):
158 if os.path.exists(self.testDir):
159 shutil.rmtree(self.testDir)
161 @staticmethod
162 def writeCfg(cfg, go):
163 """Write a configuration file after waiting on a condition variable."""
164 while go is False:
165 pass
166 dp.PosixStorage.putRepositoryCfg(cfg)
168 def testWriteCfg(self):
169 """Test parallel writes to a configuration file.
171 multiprocessing is used to spawn several writer function executions,
172 all of which wait to be released by the condition variable "go".
174 There are no asserts here, so success is measured solely by not
175 failing with an exception, but the time it took to do the writes can
176 be logged as a potential performance metric.
177 """
178 numWriters = 3
179 startTime = time.time()
180 go = multiprocessing.Value('b', False)
181 cfg = dp.RepositoryCfg(root=os.path.join(self.testDir), mapper='bar', mapperArgs={},
182 parents=None, policy=None)
183 procs = [multiprocessing.Process(target=TestMultipleWriters.writeCfg, args=(cfg, go))
184 for x in range(numWriters)]
185 for proc in procs:
186 proc.start()
187 go = True
188 for proc in procs:
189 proc.join()
190 endTime = time.time()
191 log = Log.getLogger("daf.persistence")
192 log.trace("TestMultipleWriters took {} seconds.".format(endTime-startTime))
195class MemoryTester(lsst.utils.tests.MemoryTestCase):
196 pass
199if __name__ == '__main__': 199 ↛ 200line 199 didn't jump to line 200, because the condition on line 199 was never true
200 lsst.utils.tests.init()
201 unittest.main()