src/allmydata/test/test_drop_upload.py

   1
   2 import os, sys
   3
   4 from twisted.trial import unittest
   5 from twisted.python import filepath, runtime
   6 from twisted.internet import defer
   7
   8 from allmydata.interfaces import IDirectoryNode, NoSuchChildError
   9
  10 from allmydata.util import fake_inotify
  11 from allmydata.util.encodingutil import get_filesystem_encoding
  12 from allmydata.util.consumer import download_to_data
  13 from allmydata.test.no_network import GridTestMixin
  14 from allmydata.test.common_util import ReallyEqualMixin
  15 from allmydata.test.common import ShouldFailMixin
  16
  17 from allmydata.frontends.drop_upload import DropUploader
  18
  19
  20 class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin):
  21     """
  22     These tests will be run both with a mock notifier, and (on platforms that support it)
  23     with the real INotify.
  24     """
  25
  26     def _get_count(self, name):
  27         return self.stats_provider.get_stats()["counters"].get(name, 0)
  28
  29     def _test(self):
  30         self.uploader = None
  31         self.set_up_grid()
  32         dirname_u = u"loc\u0101l_dir"
  33         if sys.platform != "win32":
  34             try:
  35                 u"loc\u0101l_dir".encode(get_filesystem_encoding())
  36             except UnicodeEncodeError:
  37                 dirname_u = u"local_dir"
  38         self.local_dir = os.path.join(self.basedir, dirname_u)
  39         os.mkdir(self.local_dir)
  40
  41         self.client = self.g.clients[0]
  42         self.stats_provider = self.client.stats_provider
  43
  44         d = self.client.create_dirnode()
  45         def _made_upload_dir(n):
  46             self.failUnless(IDirectoryNode.providedBy(n))
  47             self.upload_dirnode = n
  48             self.upload_uri = n.get_uri()
  49             self.uploader = DropUploader(self.client, self.upload_uri, self.local_dir.encode('utf-8'),
  50                                          inotify=self.inotify)
  51             return self.uploader.start()
  52         d.addCallback(_made_upload_dir)
  53
  54         # Write something short enough for a LIT file.
  55         d.addCallback(lambda ign: self._test_file(u"short", "test"))
  56
  57         # Write to the same file again with different data.
  58         d.addCallback(lambda ign: self._test_file(u"short", "different"))
  59
  60         # Test that temporary files are not uploaded.
  61         d.addCallback(lambda ign: self._test_file(u"tempfile", "test", temporary=True))
  62
  63         # Test that we tolerate creation of a subdirectory.
  64         d.addCallback(lambda ign: os.mkdir(os.path.join(self.local_dir, u"directory")))
  65
  66         # Write something longer, and also try to test a Unicode name if the fs can represent it.
  67         name_u = u"l\u00F8ng"
  68         if sys.platform != "win32":
  69             try:
  70                 u"l\u00F8ng".encode(get_filesystem_encoding())
  71             except UnicodeEncodeError:
  72                 name_u = u"long"
  73         d.addCallback(lambda ign: self._test_file(name_u, "test"*100))
  74
  75         # TODO: test that causes an upload failure.
  76         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_failed'), 0))
  77
  78         # Prevent unclean reactor errors.
  79         def _cleanup(res):
  80             d = defer.succeed(None)
  81             if self.uploader is not None:
  82                 d.addCallback(lambda ign: self.uploader.finish(for_tests=True))
  83             d.addCallback(lambda ign: res)
  84             return d
  85         d.addBoth(_cleanup)
  86         return d
  87
  88     def _test_file(self, name_u, data, temporary=False):
  89         previously_uploaded = self._get_count('drop_upload.files_uploaded')
  90         previously_disappeared = self._get_count('drop_upload.files_disappeared')
  91
  92         d = defer.Deferred()
  93
  94         # Note: this relies on the fact that we only get one IN_CLOSE_WRITE notification per file
  95         # (otherwise we would get a defer.AlreadyCalledError). Should we be relying on that?
  96         self.uploader.set_uploaded_callback(d.callback)
  97
  98         path_u = os.path.join(self.local_dir, name_u)
  99         if sys.platform == "win32":
 100             path = filepath.FilePath(path_u)
 101         else:
 102             path = filepath.FilePath(path_u.encode(get_filesystem_encoding()))
 103
 104         f = open(path.path, "wb")
 105         try:
 106             if temporary and sys.platform != "win32":
 107                 os.unlink(path.path)
 108             f.write(data)
 109         finally:
 110             f.close()
 111         if temporary and sys.platform == "win32":
 112             os.unlink(path.path)
 113         self.notify_close_write(path)
 114
 115         if temporary:
 116             d.addCallback(lambda ign: self.shouldFail(NoSuchChildError, 'temp file not uploaded', None,
 117                                                       self.upload_dirnode.get, name_u))
 118             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_disappeared'),
 119                                                                  previously_disappeared + 1))
 120         else:
 121             d.addCallback(lambda ign: self.upload_dirnode.get(name_u))
 122             d.addCallback(download_to_data)
 123             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 124             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'),
 125                                                                  previously_uploaded + 1))
 126
 127         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_queued'), 0))
 128         return d
 129
 130
 131 class MockTest(DropUploadTestMixin, unittest.TestCase):
 132     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
 133
 134     def test_errors(self):
 135         self.basedir = "drop_upload.MockTest.test_errors"
 136         self.set_up_grid()
 137         errors_dir = os.path.join(self.basedir, "errors_dir")
 138         os.mkdir(errors_dir)
 139
 140         client = self.g.clients[0]
 141         d = client.create_dirnode()
 142         def _made_upload_dir(n):
 143             self.failUnless(IDirectoryNode.providedBy(n))
 144             upload_uri = n.get_uri()
 145             readonly_uri = n.get_readonly_uri()
 146
 147             self.shouldFail(AssertionError, 'invalid local dir', 'could not be represented',
 148                             DropUploader, client, upload_uri, '\xFF', inotify=fake_inotify)
 149             self.shouldFail(AssertionError, 'non-existant local dir', 'not an existing directory',
 150                             DropUploader, client, upload_uri, os.path.join(self.basedir, "Laputa"), inotify=fake_inotify)
 151
 152             self.shouldFail(AssertionError, 'bad URI', 'not a directory URI',
 153                             DropUploader, client, 'bad', errors_dir, inotify=fake_inotify)
 154             self.shouldFail(AssertionError, 'non-directory URI', 'not a directory URI',
 155                             DropUploader, client, 'URI:LIT:foo', errors_dir, inotify=fake_inotify)
 156             self.shouldFail(AssertionError, 'readonly directory URI', 'does not refer to a writeable directory',
 157                             DropUploader, client, readonly_uri, errors_dir, inotify=fake_inotify)
 158         d.addCallback(_made_upload_dir)
 159         return d
 160
 161     def test_drop_upload(self):
 162         self.inotify = fake_inotify
 163         self.basedir = "drop_upload.MockTest.test_drop_upload"
 164         return self._test()
 165
 166     def notify_close_write(self, path):
 167         self.uploader._notifier.event(path, self.inotify.IN_CLOSE_WRITE)
 168
 169
 170 class RealTest(DropUploadTestMixin, unittest.TestCase):
 171     """This is skipped unless both Twisted and the platform support inotify."""
 172
 173     def test_drop_upload(self):
 174         # We should always have runtime.platform.supportsINotify, because we're using
 175         # Twisted >= 10.1.
 176         if not runtime.platform.supportsINotify():
 177             raise unittest.SkipTest("Drop-upload support can only be tested for-real on an OS that supports inotify or equivalent.")
 178
 179         self.inotify = None  # use the appropriate inotify for the platform
 180         self.basedir = "drop_upload.RealTest.test_drop_upload"
 181         return self._test()
 182
 183     def notify_close_write(self, path):
 184         # Writing to the file causes the notification.
 185         pass