Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

# This file is part of daf_butler. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

 

""" 

Utilities for safe file IO 

""" 

from contextlib import contextmanager 

import errno 

import fcntl 

import filecmp 

import os 

import tempfile 

import logging 

 

__all__ = ("DoNotWrite", 

"safeMakeDir", 

"setFileMode", 

"FileForWriteOnceCompareSameFailure", 

"FileForWriteOnceCompareSame", 

"SafeFile", 

"SafeFilename", 

"SafeLockedFileForRead", 

"SafeLockedFileForWrite") 

 

 

class DoNotWrite(RuntimeError): 

pass 

 

 

def safeMakeDir(directory): 

"""Make a directory in a manner avoiding race conditions""" 

if directory != "" and not os.path.exists(directory): 

try: 

os.makedirs(directory) 

except OSError as e: 

# Don't fail if directory exists due to race 

if e.errno != errno.EEXIST: 

raise e 

 

 

def setFileMode(filename): 

"""Set a file mode according to the user's umask""" 

# Get the current umask, which we can only do by setting it and then 

# reverting to the original. 

umask = os.umask(0o077) 

os.umask(umask) 

# chmod the new file to match what it would have been if it hadn't started 

# life as a temporary file (which have more restricted permissions). 

os.chmod(filename, (~umask & 0o666)) 

 

 

class FileForWriteOnceCompareSameFailure(RuntimeError): 

pass 

 

 

@contextmanager 

def FileForWriteOnceCompareSame(name): 

"""Context manager to get a file that can be written only once and all 

other writes will succeed only if they match the initial write. 

 

The context manager provides a temporary file object. After the user is 

done, the temporary file becomes the permanent file if the file at name 

does not already exist. If the file at name does exist the temporary file 

is compared to the file at name. If they are the same then this is good 

and the temp file is silently thrown away. If they are not the same then 

a runtime error is raised. 

""" 

outDir, outName = os.path.split(name) 

safeMakeDir(outDir) 

temp = tempfile.NamedTemporaryFile(mode="w", dir=outDir, prefix=outName, delete=False) 

try: 

yield temp 

finally: 

try: 

temp.close() 

# If the symlink cannot be created then it will raise. If it can't 

# be created because a file at "name" already exists then we"ll 

# do a compare-same check. 

os.symlink(temp.name, name) 

# If the symlink was created then this is the process that created 

# the first instance of the file, and we know its contents match. 

# Move the temp file over the symlink. 

os.rename(temp.name, name) 

# At this point, we know the file has just been created. Set 

# permissions according to the current umask. 

setFileMode(name) 

except OSError as e: 

if e.errno != errno.EEXIST: 

raise e 

filesMatch = filecmp.cmp(temp.name, name, shallow=False) 

os.remove(temp.name) 

if filesMatch: 

# if the files match then the compare-same check succeeded and 

# we can silently return. 

return 

else: 

# if the files do not match then the calling code was trying 

# to write a non-matching file over the previous file, maybe 

# it's a race condition? In any event, raise a runtime error. 

raise FileForWriteOnceCompareSameFailure("Written file does not match existing file.") 

 

 

@contextmanager 

def SafeFile(name): 

"""Context manager to create a file in a manner avoiding race conditions 

 

The context manager provides a temporary file object. After the user is 

done, we move that file into the desired place and close the fd to avoid 

resource leakage. 

""" 

outDir, outName = os.path.split(name) 

safeMakeDir(outDir) 

doWrite = True 

with tempfile.NamedTemporaryFile(mode="w", dir=outDir, prefix=outName, delete=False) as temp: 

try: 

yield temp 

except DoNotWrite: 

doWrite = False 

finally: 

if doWrite: 

os.rename(temp.name, name) 

setFileMode(name) 

 

 

@contextmanager 

def SafeFilename(name): 

"""Context manager for creating a file in a manner avoiding race 

conditions. 

 

The context manager provides a temporary filename with no open file 

descriptors (as this can cause trouble on some systems). After the user is 

done, we move the file into the desired place. 

""" 

outDir, outName = os.path.split(name) 

safeMakeDir(outDir) 

temp = tempfile.NamedTemporaryFile(mode="w", dir=outDir, prefix=outName, delete=False) 

tempName = temp.name 

temp.close() # We don't use the fd, just want a filename 

try: 

yield tempName 

finally: 

os.rename(tempName, name) 

setFileMode(name) 

 

 

@contextmanager 

def SafeLockedFileForRead(name): 

"""Context manager for reading a file that may be locked with an exclusive 

lock via SafeLockedFileForWrite. 

 

This will first acquire a shared lock before returning the file. When 

the file is closed the shared lock will be unlocked. 

 

Parameters 

---------- 

name : string 

The file name to be opened, may include path. 

 

Yields 

------ 

file object 

The file to be read from. 

""" 

log = logging.getLogger("daf.butler") 

try: 

with open(name, "r") as f: 

log.debug("Acquiring shared lock on {}".format(name)) 

fcntl.flock(f, fcntl.LOCK_SH) 

log.debug("Acquired shared lock on {}".format(name)) 

yield f 

finally: 

log.debug("Releasing shared lock on {}".format(name)) 

 

 

class SafeLockedFileForWrite: 

"""File-like object that is used to create a file if needed, lock it with 

an exclusive lock, and contain file descriptors to readable and writable 

versions of the file. 

 

This will only open a file descriptor in "write" mode if a write operation 

is performed. If no write operation is performed, the existing file (if 

there is one) will not be overwritten. 

 

Contains __enter__ and __exit__ functions so this can be used by a 

context manager. 

""" 

 

def __init__(self, name): 

self.log = logging.getLogger("daf.butler") 

self.name = name 

self._readable = None 

self._writeable = None 

safeMakeDir(os.path.split(name)[0]) 

 

def __enter__(self): 

self.open() 

return self 

 

def __exit__(self, type, value, traceback): 

self.close() 

 

def open(self): 

self._fileHandle = open(self.name, "a") 

self.log.debug("Acquiring exclusive lock on {}".format(self.name)) 

fcntl.flock(self._fileHandle, fcntl.LOCK_EX) 

self.log.debug("Acquired exclusive lock on {}".format(self.name)) 

 

def close(self): 

self.log.debug("Releasing exclusive lock on {}".format(self.name)) 

if self._writeable is not None: 

self._writeable.close() 

if self._readable is not None: 

self._readable.close() 

self._fileHandle.close() 

 

@property 

def readable(self): 

if self._readable is None: 

self._readable = open(self.name, "r") 

return self._readable 

 

@property 

def writeable(self): 

if self._writeable is None: 

self._writeable = open(self.name, "w") 

return self._writeable 

 

def read(self, size=None): 

if size is not None: 

return self.readable.read(size) 

return self.readable.read() 

 

def write(self, str): 

self.writeable.write(str)