storage: use fileutil's version of get_disk_stats() and get_available_space(), use...
authorZooko O'Whielacronx <zooko@zooko.com>
Fri, 10 Sep 2010 16:36:29 +0000 (08:36 -0800)
committerZooko O'Whielacronx <zooko@zooko.com>
Fri, 10 Sep 2010 16:36:29 +0000 (08:36 -0800)
src/allmydata/storage/server.py
src/allmydata/test/test_storage.py
src/allmydata/test/test_system.py

index 1af9c8901d1a940b18b734b77d895beebd779b3c..cb58d082b890747f83874ec69cc8240572f40eea 100644 (file)
@@ -36,16 +36,6 @@ class StorageServer(service.MultiService, Referenceable):
     implements(RIStorageServer, IStatsProducer)
     name = 'storage'
     LeaseCheckerClass = LeaseCheckingCrawler
-    windows = False
-
-    try:
-        import win32api, win32con
-        windows = True
-        # <http://msdn.microsoft.com/en-us/library/ms680621%28VS.85%29.aspx>
-        win32api.SetErrorMode(win32con.SEM_FAILCRITICALERRORS |
-                              win32con.SEM_NOOPENFILEERRORBOX)
-    except ImportError:
-        pass
 
     def __init__(self, storedir, nodeid, reserved_space=0,
                  discard_storage=False, readonly_storage=False,
@@ -160,57 +150,6 @@ class StorageServer(service.MultiService, Referenceable):
     def _clean_incomplete(self):
         fileutil.rm_dir(self.incomingdir)
 
-    def get_disk_stats(self):
-        """Return disk statistics for the storage disk, in the form of a dict
-        with the following fields.
-          total:            total bytes on disk
-          free_for_root:    bytes actually free on disk
-          free_for_nonroot: bytes free for "a non-privileged user" [Unix] or
-                              the current user [Windows]; might take into
-                              account quotas depending on platform
-          used:             bytes used on disk
-          avail:            bytes available excluding reserved space
-        An AttributeError can occur if the OS has no API to get disk information.
-        An EnvironmentError can occur if the OS call fails."""
-
-        if self.windows:
-            # For Windows systems, where os.statvfs is not available, use GetDiskFreeSpaceEx.
-            # <http://docs.activestate.com/activepython/2.5/pywin32/win32api__GetDiskFreeSpaceEx_meth.html>
-            #
-            # Although the docs say that the argument should be the root directory
-            # of a disk, GetDiskFreeSpaceEx actually accepts any path on that disk
-            # (like its Win32 equivalent).
-
-            (free_for_nonroot, total, free_for_root) = self.win32api.GetDiskFreeSpaceEx(self.storedir)
-        else:
-            # For Unix-like systems.
-            # <http://docs.python.org/library/os.html#os.statvfs>
-            # <http://opengroup.org/onlinepubs/7990989799/xsh/fstatvfs.html>
-            # <http://opengroup.org/onlinepubs/7990989799/xsh/sysstatvfs.h.html>
-            s = os.statvfs(self.storedir)
-
-            # on my mac laptop:
-            #  statvfs(2) is a wrapper around statfs(2).
-            #    statvfs.f_frsize = statfs.f_bsize :
-            #     "minimum unit of allocation" (statvfs)
-            #     "fundamental file system block size" (statfs)
-            #    statvfs.f_bsize = statfs.f_iosize = stat.st_blocks : preferred IO size
-            # on an encrypted home directory ("FileVault"), it gets f_blocks
-            # wrong, and s.f_blocks*s.f_frsize is twice the size of my disk,
-            # but s.f_bavail*s.f_frsize is correct
-
-            total = s.f_frsize * s.f_blocks
-            free_for_root = s.f_frsize * s.f_bfree
-            free_for_nonroot = s.f_frsize * s.f_bavail
-
-        # valid for all platforms:
-        used = total - free_for_root
-        avail = max(free_for_nonroot - self.reserved_space, 0)
-
-        return { 'total': total, 'free_for_root': free_for_root,
-                 'free_for_nonroot': free_for_nonroot,
-                 'used': used, 'avail': avail, }
-
     def get_stats(self):
         # remember: RIStatsProvider requires that our return dict
         # contains numeric values.
@@ -221,7 +160,7 @@ class StorageServer(service.MultiService, Referenceable):
                 stats['storage_server.latencies.%s.%s' % (category, name)] = v
 
         try:
-            disk = self.get_disk_stats()
+            disk = fileutil.get_disk_stats(self.storedir, self.reserved_space)
             writeable = disk['avail'] > 0
 
             # spacetime predictors should use disk_avail / (d(disk_used)/dt)
@@ -253,13 +192,7 @@ class StorageServer(service.MultiService, Referenceable):
 
         if self.readonly_storage:
             return 0
-        try:
-            return self.get_disk_stats()['avail']
-        except AttributeError:
-            return None
-        except EnvironmentError:
-            log.msg("OS call to get disk statistics failed", level=log.UNUSUAL)
-            return 0
+        return fileutil.get_available_space(self.storedir, self.reserved_space)
 
     def allocated_size(self):
         space = 0
index c302bd1e66a6deb23e3ebb5bec17a004cfe036b3..1c07fccfc39f2000a081b50aafcc284f18f88ef5 100644 (file)
@@ -1,5 +1,8 @@
+import time, os.path, platform, stat, re, simplejson, struct
 
-import time, os.path, stat, re, simplejson, struct
+from allmydata.util import log
+
+import mock
 
 from twisted.trial import unittest
 
@@ -227,11 +230,6 @@ class BucketProxy(unittest.TestCase):
         return self._do_test_readwrite("test_readwrite_v2",
                                        0x44, WriteBucketProxy_v2, ReadBucketProxy)
 
-class FakeDiskStorageServer(StorageServer):
-    DISKAVAIL = 0
-    def get_disk_stats(self):
-        return { 'free_for_nonroot': self.DISKAVAIL, 'avail': max(self.DISKAVAIL - self.reserved_space, 0), }
-
 class Server(unittest.TestCase):
 
     def setUp(self):
@@ -265,6 +263,14 @@ class Server(unittest.TestCase):
                                           sharenums, size, canary)
 
     def test_large_share(self):
+        syslow = platform.system().lower()
+        if 'cygwin' in syslow or 'windows' in syslow or 'darwin' in syslow:
+            raise unittest.SkipTest("If your filesystem doesn't support efficient sparse files then it is very expensive (Mac OS X and Windows don't support efficient sparse files).")
+
+        avail = fileutil.get_available_space('.', 2**14)
+        if avail <= 4*2**30:
+            raise unittest.SkipTest("This test will spuriously fail if you have less than 4 GiB free on your filesystem.")
+
         ss = self.create("test_large_share")
 
         already,writers = self.allocate(ss, "allocate", [0], 2**32+2)
@@ -279,7 +285,6 @@ class Server(unittest.TestCase):
         readers = ss.remote_get_buckets("allocate")
         reader = readers[shnum]
         self.failUnlessEqual(reader.remote_read(2**32, 2), "ab")
-    test_large_share.skip = "This test can spuriously fail if you have less than 4 GiB free on your filesystem, and if your filesystem doesn't support efficient sparse files then it is very expensive (Mac OS X and Windows don't support efficient sparse files)."
 
     def test_dont_overfill_dirs(self):
         """
@@ -423,11 +428,15 @@ class Server(unittest.TestCase):
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
 
-    def test_reserved_space(self):
-        ss = self.create("test_reserved_space", reserved_space=10000,
-                         klass=FakeDiskStorageServer)
-        # the FakeDiskStorageServer doesn't do real calls to get_disk_stats
-        ss.DISKAVAIL = 15000
+    @mock.patch('allmydata.util.fileutil.get_disk_stats')
+    def test_reserved_space(self, mock_get_disk_stats):
+        reserved_space=10000
+        mock_get_disk_stats.return_value = {
+            'free_for_nonroot': 15000,
+            'avail': max(15000 - reserved_space, 0),
+            }
+
+        ss = self.create("test_reserved_space", reserved_space=reserved_space)
         # 15k available, 10k reserved, leaves 5k for shares
 
         # a newly created and filled share incurs this much overhead, beyond
@@ -466,9 +475,12 @@ class Server(unittest.TestCase):
 
         allocated = 1001 + OVERHEAD + LEASE_SIZE
 
-        # we have to manually increase DISKAVAIL, since we're not doing real
+        # we have to manually increase available, since we're not doing real
         # disk measurements
-        ss.DISKAVAIL -= allocated
+        mock_get_disk_stats.return_value = {
+            'free_for_nonroot': 15000 - allocated,
+            'avail': max(15000 - allocated - reserved_space, 0),
+            }
 
         # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and
         # 5000-1085=3915 free, therefore we can fit 39 100byte shares
@@ -482,23 +494,6 @@ class Server(unittest.TestCase):
         ss.disownServiceParent()
         del ss
 
-    def test_disk_stats(self):
-        # This will spuriously fail if there is zero disk space left (but so will other tests).
-        ss = self.create("test_disk_stats", reserved_space=0)
-
-        disk = ss.get_disk_stats()
-        self.failUnless(disk['total'] > 0, disk['total'])
-        self.failUnless(disk['used'] > 0, disk['used'])
-        self.failUnless(disk['free_for_root'] > 0, disk['free_for_root'])
-        self.failUnless(disk['free_for_nonroot'] > 0, disk['free_for_nonroot'])
-        self.failUnless(disk['avail'] > 0, disk['avail'])
-
-    def test_disk_stats_avail_nonnegative(self):
-        ss = self.create("test_disk_stats_avail_nonnegative", reserved_space=2**64)
-
-        disk = ss.get_disk_stats()
-        self.failUnlessEqual(disk['avail'], 0)
-
     def test_seek(self):
         basedir = self.workdir("test_seek_behavior")
         fileutil.make_dirs(basedir)
@@ -2461,14 +2456,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         d = self.render1(page, args={"t": ["json"]})
         return d
 
-class NoDiskStatsServer(StorageServer):
-    def get_disk_stats(self):
-        raise AttributeError
-
-class BadDiskStatsServer(StorageServer):
-    def get_disk_stats(self):
-        raise OSError
-
 class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
 
     def setUp(self):
@@ -2510,12 +2497,15 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         d = self.render1(page, args={"t": ["json"]})
         return d
 
-    def test_status_no_disk_stats(self):
+    @mock.patch('allmydata.util.fileutil.get_disk_stats')
+    def test_status_no_disk_stats(self, mock_get_disk_stats):
+        mock_get_disk_stats.side_effect = AttributeError()
+
         # Some platforms may have no disk stats API. Make sure the code can handle that
         # (test runs on all platforms).
         basedir = "storage/WebStatus/status_no_disk_stats"
         fileutil.make_dirs(basedir)
-        ss = NoDiskStatsServer(basedir, "\x00" * 20)
+        ss = StorageServer(basedir, "\x00" * 20)
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = w.renderSynchronously()
@@ -2526,12 +2516,15 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         self.failUnlessIn("Space Available to Tahoe: ?", s)
         self.failUnless(ss.get_available_space() is None)
 
-    def test_status_bad_disk_stats(self):
+    @mock.patch('allmydata.util.fileutil.get_disk_stats')
+    def test_status_bad_disk_stats(self, mock_get_disk_stats):
+        mock_get_disk_stats.side_effect = OSError()
+
         # If the API to get disk stats exists but a call to it fails, then the status should
         # show that no shares will be accepted, and get_available_space() should be 0.
         basedir = "storage/WebStatus/status_bad_disk_stats"
         fileutil.make_dirs(basedir)
-        ss = BadDiskStatsServer(basedir, "\x00" * 20)
+        ss = StorageServer(basedir, "\x00" * 20)
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = w.renderSynchronously()
@@ -2583,4 +2576,3 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         self.failUnlessEqual(w.render_abbrev_space(None, 10e6), "10.00 MB")
         self.failUnlessEqual(remove_prefix("foo.bar", "foo."), "bar")
         self.failUnlessEqual(remove_prefix("foo.bar", "baz."), None)
-
index f2770bba31a892338ea1bb799b4d929a2917f460..a8cf041dde15e7a1829fd23ef1aaa4c28caec8cb 100644 (file)
@@ -1808,4 +1808,3 @@ class SystemTest(SystemTestMixin, SkipMixin, unittest.TestCase):
             return d
         d.addCallback(_got_lit_filenode)
         return d
-