src/allmydata/test/test_drop_upload.py

   1
   2 import os, sys
   3
   4 from twisted.trial import unittest
   5 from twisted.python import filepath, runtime, log
   6 from twisted.internet import defer
   7
   8 from allmydata.interfaces import IDirectoryNode, NoSuchChildError
   9
  10 from allmydata.util import fileutil, fake_inotify
  11 from allmydata.util.encodingutil import get_filesystem_encoding
  12 from allmydata.util.consumer import download_to_data
  13 from allmydata.test.no_network import GridTestMixin
  14 from allmydata.test.common_util import ReallyEqualMixin
  15 from allmydata.test.common import ShouldFailMixin
  16
  17 from allmydata.frontends.drop_upload import DropUploader
  18
  19
  20 class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin):
  21     """
  22     These tests will be run both with a mock notifier, and (on platforms that support it)
  23     with the real INotify.
  24     """
  25
  26     def setUp(self):
  27         GridTestMixin.setUp(self)
  28         self.nonascii_dirs = []
  29
  30     def tearDown(self):
  31         try:
  32             GridTestMixin.tearDown(self)
  33         finally:
  34             # kludge to work around the fact that buildbot can't remove a directory tree that has any non-ASCII directory names
  35             if sys.platform == "win32":
  36                 for dirpath in self.nonascii_dirs:
  37                     try:
  38                         fileutil.rm_dir(dirpath)
  39                     finally:
  40                         log.err("We were unable to delete a non-ASCII directory %r created by the test. "
  41                                 "This is liable to cause failures on future builds." % (dirpath,))
  42
  43     def _mkdir_nonascii(self, dirpath):
  44         self.nonascii_dirs.append(dirpath)
  45         os.mkdir(dirpath)
  46
  47     def _get_count(self, name):
  48         return self.stats_provider.get_stats()["counters"].get(name, 0)
  49
  50     def _test(self):
  51         self.uploader = None
  52         self.set_up_grid()
  53         dirname_u = u"loc\u0101l_dir"
  54         if sys.platform != "win32":
  55             try:
  56                 u"loc\u0101l_dir".encode(get_filesystem_encoding())
  57             except UnicodeEncodeError:
  58                 dirname_u = u"local_dir"
  59         self.local_dir = os.path.join(self.basedir, dirname_u)
  60         self._mkdir_nonascii(self.local_dir)
  61
  62         self.client = self.g.clients[0]
  63         self.stats_provider = self.client.stats_provider
  64
  65         d = self.client.create_dirnode()
  66         def _made_upload_dir(n):
  67             self.failUnless(IDirectoryNode.providedBy(n))
  68             self.upload_dirnode = n
  69             self.upload_dircap = n.get_uri()
  70             self.uploader = DropUploader(self.client, self.upload_dircap, self.local_dir.encode('utf-8'),
  71                                          inotify=self.inotify)
  72             return self.uploader.start()
  73         d.addCallback(_made_upload_dir)
  74
  75         # Write something short enough for a LIT file.
  76         d.addCallback(lambda ign: self._test_file(u"short", "test"))
  77
  78         # Write to the same file again with different data.
  79         d.addCallback(lambda ign: self._test_file(u"short", "different"))
  80
  81         # Test that temporary files are not uploaded.
  82         d.addCallback(lambda ign: self._test_file(u"tempfile", "test", temporary=True))
  83
  84         # Test that we tolerate creation of a subdirectory.
  85         d.addCallback(lambda ign: os.mkdir(os.path.join(self.local_dir, u"directory")))
  86
  87         # Write something longer, and also try to test a Unicode name if the fs can represent it.
  88         name_u = u"l\u00F8ng"
  89         if sys.platform != "win32":
  90             try:
  91                 u"l\u00F8ng".encode(get_filesystem_encoding())
  92             except UnicodeEncodeError:
  93                 name_u = u"long"
  94         d.addCallback(lambda ign: self._test_file(name_u, "test"*100))
  95
  96         # TODO: test that causes an upload failure.
  97         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_failed'), 0))
  98
  99         # Prevent unclean reactor errors.
 100         def _cleanup(res):
 101             d = defer.succeed(None)
 102             if self.uploader is not None:
 103                 d.addCallback(lambda ign: self.uploader.finish(for_tests=True))
 104             d.addCallback(lambda ign: res)
 105             return d
 106         d.addBoth(_cleanup)
 107         return d
 108
 109     def _test_file(self, name_u, data, temporary=False):
 110         previously_uploaded = self._get_count('drop_upload.files_uploaded')
 111         previously_disappeared = self._get_count('drop_upload.files_disappeared')
 112
 113         d = defer.Deferred()
 114
 115         # Note: this relies on the fact that we only get one IN_CLOSE_WRITE notification per file
 116         # (otherwise we would get a defer.AlreadyCalledError). Should we be relying on that?
 117         self.uploader.set_uploaded_callback(d.callback)
 118
 119         path_u = os.path.join(self.local_dir, name_u)
 120         if sys.platform == "win32":
 121             path = filepath.FilePath(path_u)
 122         else:
 123             path = filepath.FilePath(path_u.encode(get_filesystem_encoding()))
 124
 125         # We don't use FilePath.setContent() here because it creates a temporary file that
 126         # is renamed into place, which causes events that the test is not expecting.
 127         f = open(path.path, "wb")
 128         try:
 129             if temporary and sys.platform != "win32":
 130                 os.unlink(path.path)
 131             f.write(data)
 132         finally:
 133             f.close()
 134         if temporary and sys.platform == "win32":
 135             os.unlink(path.path)
 136         self.notify_close_write(path)
 137
 138         if temporary:
 139             d.addCallback(lambda ign: self.shouldFail(NoSuchChildError, 'temp file not uploaded', None,
 140                                                       self.upload_dirnode.get, name_u))
 141             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_disappeared'),
 142                                                                  previously_disappeared + 1))
 143         else:
 144             d.addCallback(lambda ign: self.upload_dirnode.get(name_u))
 145             d.addCallback(download_to_data)
 146             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 147             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'),
 148                                                                  previously_uploaded + 1))
 149
 150         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_queued'), 0))
 151         return d
 152
 153
 154 class MockTest(DropUploadTestMixin, unittest.TestCase):
 155     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
 156
 157     def test_errors(self):
 158         self.basedir = "drop_upload.MockTest.test_errors"
 159         self.set_up_grid()
 160         errors_dir = os.path.join(self.basedir, "errors_dir")
 161         os.mkdir(errors_dir)
 162
 163         client = self.g.clients[0]
 164         d = client.create_dirnode()
 165         def _made_upload_dir(n):
 166             self.failUnless(IDirectoryNode.providedBy(n))
 167             upload_dircap = n.get_uri()
 168             readonly_dircap = n.get_readonly_uri()
 169
 170             self.shouldFail(AssertionError, 'invalid local.directory', 'could not be represented',
 171                             DropUploader, client, upload_dircap, '\xFF', inotify=fake_inotify)
 172             self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
 173                             DropUploader, client, upload_dircap, os.path.join(self.basedir, "Laputa"), inotify=fake_inotify)
 174
 175             fp = filepath.FilePath(self.basedir).child('NOT_A_DIR')
 176             fp.touch()
 177             self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
 178                             DropUploader, client, upload_dircap, fp.path, inotify=fake_inotify)
 179
 180             self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
 181                             DropUploader, client, 'bad', errors_dir, inotify=fake_inotify)
 182             self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
 183                             DropUploader, client, 'URI:LIT:foo', errors_dir, inotify=fake_inotify)
 184             self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
 185                             DropUploader, client, readonly_dircap, errors_dir, inotify=fake_inotify)
 186         d.addCallback(_made_upload_dir)
 187         return d
 188
 189     def test_drop_upload(self):
 190         self.inotify = fake_inotify
 191         self.basedir = "drop_upload.MockTest.test_drop_upload"
 192         return self._test()
 193
 194     def notify_close_write(self, path):
 195         self.uploader._notifier.event(path, self.inotify.IN_CLOSE_WRITE)
 196
 197
 198 class RealTest(DropUploadTestMixin, unittest.TestCase):
 199     """This is skipped unless both Twisted and the platform support inotify."""
 200
 201     def test_drop_upload(self):
 202         # We should always have runtime.platform.supportsINotify, because we're using
 203         # Twisted >= 10.1.
 204         if not runtime.platform.supportsINotify():
 205             raise unittest.SkipTest("Drop-upload support can only be tested for-real on an OS that supports inotify or equivalent.")
 206
 207         self.inotify = None  # use the appropriate inotify for the platform
 208         self.basedir = "drop_upload.RealTest.test_drop_upload"
 209         return self._test()
 210
 211     def notify_close_write(self, path):
 212         # Writing to the file causes the notification.
 213         pass