2 Futz with files like a pro.
5 import sys, exceptions, os, stat, tempfile, time, binascii
7 from twisted.python import log
9 from pycryptopp.cipher.aes import AES
12 def rename(src, dst, tries=4, basedelay=0.1):
13 """ Here is a superkludge to workaround the fact that occasionally on
14 Windows some other process (e.g. an anti-virus scanner, a local search
15 engine, etc.) is looking at your file when you want to delete or move it,
16 and hence you can't. The horrible workaround is to sit and spin, trying
17 to delete it, for a short time and then give up.
19 With the default values of tries and basedelay this can block for less
22 @param tries: number of tries -- each time after the first we wait twice
23 as long as the previous wait
24 @param basedelay: how long to wait before the second try
26 for i in range(tries-1):
28 return os.rename(src, dst)
29 except EnvironmentError, le:
30 # XXX Tighten this to check if this is a permission denied error (possibly due to another Windows process having the file open and execute the superkludge only in this case.
31 log.msg("XXX KLUDGE Attempting to move file %s => %s; got %s; sleeping %s seconds" % (src, dst, le, basedelay,))
34 return os.rename(src, dst) # The last try.
36 def remove(f, tries=4, basedelay=0.1):
37 """ Here is a superkludge to workaround the fact that occasionally on
38 Windows some other process (e.g. an anti-virus scanner, a local search
39 engine, etc.) is looking at your file when you want to delete or move it,
40 and hence you can't. The horrible workaround is to sit and spin, trying
41 to delete it, for a short time and then give up.
43 With the default values of tries and basedelay this can block for less
46 @param tries: number of tries -- each time after the first we wait twice
47 as long as the previous wait
48 @param basedelay: how long to wait before the second try
51 os.chmod(f, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD)
54 for i in range(tries-1):
57 except EnvironmentError, le:
58 # XXX Tighten this to check if this is a permission denied error (possibly due to another Windows process having the file open and execute the superkludge only in this case.
59 if not os.path.exists(f):
61 log.msg("XXX KLUDGE Attempting to remove file %s; got %s; sleeping %s seconds" % (f, le, basedelay,))
64 return os.remove(f) # The last try.
66 class ReopenableNamedTemporaryFile:
68 This uses tempfile.mkstemp() to generate a secure temp file. It then closes
69 the file, leaving a zero-length file as a placeholder. You can get the
70 filename with ReopenableNamedTemporaryFile.name. When the
71 ReopenableNamedTemporaryFile instance is garbage collected or its shutdown()
72 method is called, it deletes the file.
74 def __init__(self, *args, **kwargs):
75 fd, self.name = tempfile.mkstemp(*args, **kwargs)
79 return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name)
82 return self.__repr__()
90 class NamedTemporaryDirectory:
92 This calls tempfile.mkdtemp(), stores the name of the dir in
93 self.name, and rmrf's the dir when it gets garbage collected or
96 def __init__(self, cleanup=True, *args, **kwargs):
97 """ If cleanup, then the directory will be rmrf'ed when the object is shutdown. """
98 self.cleanup = cleanup
99 self.name = tempfile.mkdtemp(*args, **kwargs)
102 return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name)
105 return self.__repr__()
112 traceback.print_exc()
115 if self.cleanup and hasattr(self, 'name'):
118 class EncryptedTemporaryFile:
119 # not implemented: next, readline, readlines, xreadlines, writelines
122 self.file = tempfile.TemporaryFile()
123 self.key = os.urandom(16) # AES-128
125 def _crypt(self, offset, data):
126 offset_big = offset // 16
127 offset_small = offset % 16
128 iv = binascii.unhexlify("%032x" % offset_big)
129 cipher = AES(self.key, iv=iv)
130 cipher.process("\x00"*offset_small)
131 return cipher.process(data)
139 def seek(self, offset, whence=0): # 0 = SEEK_SET
140 self.file.seek(offset, whence)
143 offset = self.file.tell()
146 def read(self, size=-1):
147 """A read must not follow a write, or vice-versa, without an intervening seek."""
148 index = self.file.tell()
149 ciphertext = self.file.read(size)
150 plaintext = self._crypt(index, ciphertext)
153 def write(self, plaintext):
154 """A read must not follow a write, or vice-versa, without an intervening seek.
155 If seeking and then writing causes a 'hole' in the file, the contents of the
156 hole are unspecified."""
157 index = self.file.tell()
158 ciphertext = self._crypt(index, plaintext)
159 self.file.write(ciphertext)
161 def truncate(self, newsize):
162 """Truncate or extend the file to 'newsize'. If it is extended, the contents after the
163 old end-of-file are unspecified. The file position after this operation is unspecified."""
164 self.file.truncate(newsize)
167 def make_dirs(dirname, mode=0777):
169 An idempotent version of os.makedirs(). If the dir already exists, do
170 nothing and return without raising an exception. If this call creates the
171 dir, return without raising an exception. If there is an error that
172 prevents creation or if the directory gets deleted after make_dirs() creates
173 it and before make_dirs() checks that it exists, raise an exception.
177 os.makedirs(dirname, mode)
181 if not os.path.isdir(dirname):
184 raise exceptions.IOError, "unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname # careful not to construct an IOError with a 2-tuple, as that has a special meaning...
188 A threadsafe and idempotent version of shutil.rmtree(). If the dir is
189 already gone, do nothing and return without raising an exception. If this
190 call removes the dir, return without raising an exception. If there is an
191 error that prevents deletion or if the directory gets created again after
192 rm_dir() deletes it and before rm_dir() checks that it is gone, raise an
197 os.chmod(dirname, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD)
198 for f in os.listdir(dirname):
199 fullname = os.path.join(dirname, f)
200 if os.path.isdir(fullname):
205 except Exception, le:
206 # Ignore "No such file or directory"
207 if (not isinstance(le, OSError)) or le.args[0] != 2:
210 # Okay, now we've recursively removed everything, ignoring any "No
211 # such file or directory" errors, and collecting any other errors.
213 if os.path.exists(dirname):
217 raise OSError, "Failed to remove dir for unknown reason."
221 def remove_if_possible(f):
230 for root, dirs, files in os.walk(basedir):
232 fn = os.path.join(root, f)
233 size += os.path.getsize(fn)
237 def move_into_place(source, dest):
238 """Atomically replace a file, or as near to it as the platform allows.
239 The dest file may or may not exist."""
240 if "win32" in sys.platform.lower():
241 remove_if_possible(dest)
242 os.rename(source, dest)
244 def write_atomically(target, contents, mode="b"):
245 f = open(target+".tmp", "w"+mode)
250 move_into_place(target+".tmp", target)
252 def write(path, data, mode="wb"):
253 wf = open(path, mode)
260 rf = open(path, "rb")
266 def put_file(path, inf):
267 precondition_abspath(path)
269 # TODO: create temporary file and move into place?
270 outf = open(path, "wb")
273 data = inf.read(32768)
281 def precondition_abspath(path):
282 if not isinstance(path, unicode):
283 raise AssertionError("an abspath must be a Unicode string")
285 if sys.platform == "win32":
286 # This intentionally doesn't view absolute paths starting with a drive specification, or
287 # paths relative to the current drive, as acceptable.
288 if not path.startswith("\\\\"):
289 raise AssertionError("an abspath should be normalized using abspath_expanduser_unicode")
291 # This intentionally doesn't view the path '~' or paths starting with '~/' as acceptable.
292 if not os.path.isabs(path):
293 raise AssertionError("an abspath should be normalized using abspath_expanduser_unicode")
295 # Work around <http://bugs.python.org/issue3426>. This code is adapted from
296 # <http://svn.python.org/view/python/trunk/Lib/ntpath.py?revision=78247&view=markup>
297 # with some simplifications.
299 _getfullpathname = None
301 from nt import _getfullpathname
305 def abspath_expanduser_unicode(path, base=None):
307 Return the absolute version of a path. If 'base' is given and 'path' is relative,
308 the path will be expanded relative to 'base'.
309 'path' must be a Unicode string. 'base', if given, must be a Unicode string
310 corresponding to an absolute path as returned by a previous call to
311 abspath_expanduser_unicode.
313 if not isinstance(path, unicode):
314 raise AssertionError("paths must be Unicode strings")
316 precondition_abspath(base)
318 path = expanduser(path)
321 # On Windows, os.path.isabs will incorrectly return True
322 # for paths without a drive letter (that are not UNC paths),
323 # e.g. "\\". See <http://bugs.python.org/issue1669539>.
326 path = _getfullpathname(path or u".")
328 path = _getfullpathname(os.path.join(base, path))
332 if not os.path.isabs(path):
334 path = os.path.join(os.getcwdu(), path)
336 path = os.path.join(base, path)
338 # We won't hit <http://bugs.python.org/issue5827> because
339 # there is always at least one Unicode path component.
340 path = os.path.normpath(path)
342 if sys.platform == "win32":
343 path = to_windows_long_path(path)
347 def to_windows_long_path(path):
348 # '/' is normally a perfectly valid path component separator in Windows.
349 # However, when using the "\\?\" syntax it is not recognized, so we
350 # replace it with '\' here.
351 path = path.replace(u"/", u"\\")
353 # Note that other normalizations such as removing '.' and '..' should
354 # be done outside this function.
356 if path.startswith(u"\\\\?\\") or path.startswith(u"\\\\.\\"):
358 elif path.startswith(u"\\\\"):
359 return u"\\\\?\\UNC\\" + path[2 :]
361 return u"\\\\?\\" + path
364 have_GetDiskFreeSpaceExW = False
365 if sys.platform == "win32":
366 from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_ulonglong, create_unicode_buffer
367 from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPWSTR
369 # <http://msdn.microsoft.com/en-us/library/ms679360%28v=VS.85%29.aspx>
370 GetLastError = WINFUNCTYPE(DWORD)(("GetLastError", windll.kernel32))
372 # <http://msdn.microsoft.com/en-us/library/windows/desktop/ms683188%28v=vs.85%29.aspx>
373 GetEnvironmentVariableW = WINFUNCTYPE(DWORD, LPCWSTR, LPWSTR, DWORD)(
374 ("GetEnvironmentVariableW", windll.kernel32))
377 # <http://msdn.microsoft.com/en-us/library/aa383742%28v=VS.85%29.aspx>
378 PULARGE_INTEGER = POINTER(c_ulonglong)
380 # <http://msdn.microsoft.com/en-us/library/aa364937%28VS.85%29.aspx>
381 GetDiskFreeSpaceExW = WINFUNCTYPE(BOOL, LPCWSTR, PULARGE_INTEGER, PULARGE_INTEGER, PULARGE_INTEGER)(
382 ("GetDiskFreeSpaceExW", windll.kernel32))
384 have_GetDiskFreeSpaceExW = True
387 traceback.print_exc()
389 def expanduser(path):
390 # os.path.expanduser is hopelessly broken for Unicode paths on Windows (ticket #1674).
391 if sys.platform == "win32":
392 return windows_expanduser(path)
394 return os.path.expanduser(path)
396 def windows_expanduser(path):
397 if not path.startswith('~'):
399 home_drive = windows_getenv(u'HOMEDRIVE')
400 home_path = windows_getenv(u'HOMEPATH')
402 return os.path.join(home_drive, home_path)
403 elif path.startswith('~/') or path.startswith('~\\'):
404 return os.path.join(home_drive, home_path, path[2 :])
408 def windows_getenv(name):
409 # Based on <http://stackoverflow.com/questions/2608200/problems-with-umlauts-in-python-appdata-environvent-variable/2608368#2608368>,
410 # with improved error handling.
411 if not isinstance(name, unicode):
412 raise AssertionError("name must be Unicode")
414 n = GetEnvironmentVariableW(name, None, 0)
415 # GetEnvironmentVariableW returns DWORD, so n cannot be negative.
418 raise OSError("Windows error %d attempting to read size of environment variable %r"
421 # Avoid an ambiguity between a zero-length string and an error in the return value of the
422 # call to GetEnvironmentVariableW below.
425 buf = create_unicode_buffer(u'\0'*n)
426 retval = GetEnvironmentVariableW(name, buf, n)
429 raise OSError("Windows error %d attempting to read environment variable %r"
432 raise OSError("Unexpected result %d (expected less than %d) from GetEnvironmentVariableW attempting to read environment variable %r"
437 def get_disk_stats(whichdir, reserved_space=0):
438 """Return disk statistics for the storage disk, in the form of a dict
439 with the following fields.
440 total: total bytes on disk
441 free_for_root: bytes actually free on disk
442 free_for_nonroot: bytes free for "a non-privileged user" [Unix] or
443 the current user [Windows]; might take into
444 account quotas depending on platform
445 used: bytes used on disk
446 avail: bytes available excluding reserved space
447 An AttributeError can occur if the OS has no API to get disk information.
448 An EnvironmentError can occur if the OS call fails.
450 whichdir is a directory on the filesystem in question -- the
451 answer is about the filesystem, not about the directory, so the
452 directory is used only to specify which filesystem.
454 reserved_space is how many bytes to subtract from the answer, so
455 you can pass how many bytes you would like to leave unused on this
456 filesystem as reserved_space.
459 if have_GetDiskFreeSpaceExW:
460 # If this is a Windows system and GetDiskFreeSpaceExW is available, use it.
461 # (This might put up an error dialog unless
462 # SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX) has been called,
463 # which we do in allmydata.windows.fixups.initialize().)
465 n_free_for_nonroot = c_ulonglong(0)
466 n_total = c_ulonglong(0)
467 n_free_for_root = c_ulonglong(0)
468 retval = GetDiskFreeSpaceExW(whichdir, byref(n_free_for_nonroot),
470 byref(n_free_for_root))
472 raise OSError("Windows error %d attempting to get disk statistics for %r"
473 % (GetLastError(), whichdir))
474 free_for_nonroot = n_free_for_nonroot.value
475 total = n_total.value
476 free_for_root = n_free_for_root.value
478 # For Unix-like systems.
479 # <http://docs.python.org/library/os.html#os.statvfs>
480 # <http://opengroup.org/onlinepubs/7990989799/xsh/fstatvfs.html>
481 # <http://opengroup.org/onlinepubs/7990989799/xsh/sysstatvfs.h.html>
482 s = os.statvfs(whichdir)
485 # statvfs(2) is a wrapper around statfs(2).
486 # statvfs.f_frsize = statfs.f_bsize :
487 # "minimum unit of allocation" (statvfs)
488 # "fundamental file system block size" (statfs)
489 # statvfs.f_bsize = statfs.f_iosize = stat.st_blocks : preferred IO size
490 # on an encrypted home directory ("FileVault"), it gets f_blocks
491 # wrong, and s.f_blocks*s.f_frsize is twice the size of my disk,
492 # but s.f_bavail*s.f_frsize is correct
494 total = s.f_frsize * s.f_blocks
495 free_for_root = s.f_frsize * s.f_bfree
496 free_for_nonroot = s.f_frsize * s.f_bavail
498 # valid for all platforms:
499 used = total - free_for_root
500 avail = max(free_for_nonroot - reserved_space, 0)
502 return { 'total': total,
503 'free_for_root': free_for_root,
504 'free_for_nonroot': free_for_nonroot,
509 def get_available_space(whichdir, reserved_space):
510 """Returns available space for share storage in bytes, or None if no
511 API to get this information is available.
513 whichdir is a directory on the filesystem in question -- the
514 answer is about the filesystem, not about the directory, so the
515 directory is used only to specify which filesystem.
517 reserved_space is how many bytes to subtract from the answer, so
518 you can pass how many bytes you would like to leave unused on this
519 filesystem as reserved_space.
522 return get_disk_stats(whichdir, reserved_space)['avail']
523 except AttributeError:
525 except EnvironmentError:
526 log.msg("OS call to get disk statistics failed")