lsst.daf.persistence  13.0-11-gfc17871
 All Classes Namespaces Files Functions Variables Typedefs Friends Macros
safeFileIo.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 """
24 Utilities for safe file IO
25 """
26 from contextlib import contextmanager
27 import errno
28 import filecmp
29 import os
30 import tempfile
31 
32 
33 def safeMakeDir(directory):
34  """Make a directory in a manner avoiding race conditions"""
35  if directory != "" and not os.path.exists(directory):
36  try:
37  os.makedirs(directory)
38  except OSError as e:
39  # Don't fail if directory exists due to race
40  if e.errno != errno.EEXIST:
41  raise e
42 
43 
44 def setFileMode(filename):
45  """Set a file mode according to the user's umask"""
46  # Get the current umask, which we can only do by setting it and then reverting to the original.
47  umask = os.umask(0o077)
48  os.umask(umask)
49  # chmod the new file to match what it would have been if it hadn't started life as a temporary
50  # file (which have more restricted permissions).
51  os.chmod(filename, (~umask & 0o666))
52 
53 
54 @contextmanager
56  """Context manager to get a file that can be written only once and all other writes will succeed only if
57  they match the inital write.
58 
59  The context manager provides a temporary file object. After the user is done, the temporary file becomes
60  the permanent file if the file at name does not already exist. If the file at name does exist the
61  temporary file is compared to the file at name. If they are the same then this is good and the temp file
62  is silently thrown away. If they are not the same then a runtime error is raised.
63  """
64  outDir, outName = os.path.split(name)
65  safeMakeDir(outDir)
66  temp = tempfile.NamedTemporaryFile(mode="w", dir=outDir, prefix=outName, delete=False)
67  try:
68  yield temp
69  finally:
70  try:
71  temp.close()
72  # If the symlink cannot be created then it will raise. If it can't be created because a file at
73  # 'name' already exists then we'll do a compare-same check.
74  os.symlink(temp.name, name)
75  # If the symlink was created then this is the process that created the first instance of the
76  # file, and we know its contents match. Move the temp file over the symlink.
77  os.rename(temp.name, name)
78  # At this point, we know the file has just been created. Set permissions according to the
79  # current umask.
80  setFileMode(name)
81  except OSError as e:
82  if e.errno != errno.EEXIST:
83  raise e
84  filesMatch = filecmp.cmp(temp.name, name, shallow=False)
85  os.remove(temp.name)
86  if filesMatch:
87  # if the files match then the compare-same check succeeded and we can silently return.
88  return
89  else:
90  # if the files do not match then the calling code was trying to write a non-matching file over
91  # the previous file, maybe it's a race condition? Iny any event, raise a runtime error.
92  raise RuntimeError("Written file does not match existing file.")
93 
94 
95 @contextmanager
96 def SafeFile(name):
97  """Context manager to create a file in a manner avoiding race conditions
98 
99  The context manager provides a temporary file object. After the user is done,
100  we move that file into the desired place and close the fd to avoid resource
101  leakage.
102  """
103  outDir, outName = os.path.split(name)
104  safeMakeDir(outDir)
105  with tempfile.NamedTemporaryFile(mode="w", dir=outDir, prefix=outName, delete=False) as temp:
106  try:
107  yield temp
108  finally:
109  os.rename(temp.name, name)
110  setFileMode(name)
111 
112 
113 @contextmanager
114 def SafeFilename(name):
115  """Context manager for creating a file in a manner avoiding race conditions
116 
117  The context manager provides a temporary filename with no open file descriptors
118  (as this can cause trouble on some systems). After the user is done, we move the
119  file into the desired place.
120  """
121  outDir, outName = os.path.split(name)
122  safeMakeDir(outDir)
123  temp = tempfile.NamedTemporaryFile(mode="w", dir=outDir, prefix=outName, delete=False)
124  tempName = temp.name
125  temp.close() # We don't use the fd, just want a filename
126  try:
127  yield tempName
128  finally:
129  os.rename(tempName, name)
130  setFileMode(name)