src/allmydata/test/test_drop_upload.py

   1
   2 import os, sys
   3
   4 from twisted.trial import unittest
   5 from twisted.python import filepath, runtime, log
   6 from twisted.internet import defer
   7
   8 from allmydata.interfaces import IDirectoryNode, NoSuchChildError
   9
  10 from allmydata.util import fileutil, fake_inotify
  11 from allmydata.util.encodingutil import get_filesystem_encoding
  12 from allmydata.util.consumer import download_to_data
  13 from allmydata.test.no_network import GridTestMixin
  14 from allmydata.test.common_util import ReallyEqualMixin
  15 from allmydata.test.common import ShouldFailMixin
  16
  17 from allmydata.frontends.drop_upload import DropUploader
  18
  19
  20 class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin):
  21     """
  22     These tests will be run both with a mock notifier, and (on platforms that support it)
  23     with the real INotify.
  24     """
  25
  26     def setUp(self):
  27         GridTestMixin.setUp(self)
  28         self.nonascii_dirs = []
  29
  30     def tearDown(self):
  31         try:
  32             GridTestMixin.tearDown(self)
  33         finally:
  34             # kludge to work around the fact that buildbot can't remove a directory tree that has any non-ASCII directory names
  35             if sys.platform == "win32":
  36                 for dirpath in self.nonascii_dirs:
  37                     try:
  38                         fileutil.rm_dir(dirpath)
  39                     finally:
  40                         log.err("We were unable to delete a non-ASCII directory %r created by the test. "
  41                                 "This is liable to cause failures on future builds." % (dirpath,))
  42
  43     def _mkdir_nonascii(self, dirpath):
  44         self.nonascii_dirs.append(dirpath)
  45         os.mkdir(dirpath)
  46
  47     def _get_count(self, name):
  48         return self.stats_provider.get_stats()["counters"].get(name, 0)
  49
  50     def _test(self):
  51         self.uploader = None
  52         self.set_up_grid()
  53         dirname_u = u"loc\u0101l_dir"
  54         if sys.platform != "win32":
  55             try:
  56                 u"loc\u0101l_dir".encode(get_filesystem_encoding())
  57             except UnicodeEncodeError:
  58                 dirname_u = u"local_dir"
  59         self.local_dir = os.path.join(self.basedir, dirname_u)
  60         self._mkdir_nonascii(self.local_dir)
  61
  62         self.client = self.g.clients[0]
  63         self.stats_provider = self.client.stats_provider
  64
  65         d = self.client.create_dirnode()
  66         def _made_upload_dir(n):
  67             self.failUnless(IDirectoryNode.providedBy(n))
  68             self.upload_dirnode = n
  69             self.upload_dircap = n.get_uri()
  70             self.uploader = DropUploader(self.client, self.upload_dircap, self.local_dir.encode('utf-8'),
  71                                          inotify=self.inotify)
  72             return self.uploader.start()
  73         d.addCallback(_made_upload_dir)
  74
  75         # Write something short enough for a LIT file.
  76         d.addCallback(lambda ign: self._test_file(u"short", "test"))
  77
  78         # Write to the same file again with different data.
  79         d.addCallback(lambda ign: self._test_file(u"short", "different"))
  80
  81         # Test that temporary files are not uploaded.
  82         d.addCallback(lambda ign: self._test_file(u"tempfile", "test", temporary=True))
  83
  84         # Test that we tolerate creation of a subdirectory.
  85         d.addCallback(lambda ign: os.mkdir(os.path.join(self.local_dir, u"directory")))
  86
  87         # Write something longer, and also try to test a Unicode name if the fs can represent it.
  88         name_u = u"l\u00F8ng"
  89         if sys.platform != "win32":
  90             try:
  91                 u"l\u00F8ng".encode(get_filesystem_encoding())
  92             except UnicodeEncodeError:
  93                 name_u = u"long"
  94         d.addCallback(lambda ign: self._test_file(name_u, "test"*100))
  95
  96         # TODO: test that causes an upload failure.
  97         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_failed'), 0))
  98
  99         # Prevent unclean reactor errors.
 100         def _cleanup(res):
 101             d = defer.succeed(None)
 102             if self.uploader is not None:
 103                 d.addCallback(lambda ign: self.uploader.finish(for_tests=True))
 104             d.addCallback(lambda ign: res)
 105             return d
 106         d.addBoth(_cleanup)
 107         return d
 108
 109     def _test_file(self, name_u, data, temporary=False):
 110         previously_uploaded = self._get_count('drop_upload.files_uploaded')
 111         previously_disappeared = self._get_count('drop_upload.files_disappeared')
 112
 113         d = defer.Deferred()
 114
 115         # Note: this relies on the fact that we only get one IN_CLOSE_WRITE notification per file
 116         # (otherwise we would get a defer.AlreadyCalledError). Should we be relying on that?
 117         self.uploader.set_uploaded_callback(d.callback)
 118
 119         path_u = os.path.join(self.local_dir, name_u)
 120         if sys.platform == "win32":
 121             path = filepath.FilePath(path_u)
 122         else:
 123             path = filepath.FilePath(path_u.encode(get_filesystem_encoding()))
 124
 125         f = open(path.path, "wb")
 126         try:
 127             if temporary and sys.platform != "win32":
 128                 os.unlink(path.path)
 129             f.write(data)
 130         finally:
 131             f.close()
 132         if temporary and sys.platform == "win32":
 133             os.unlink(path.path)
 134         self.notify_close_write(path)
 135
 136         if temporary:
 137             d.addCallback(lambda ign: self.shouldFail(NoSuchChildError, 'temp file not uploaded', None,
 138                                                       self.upload_dirnode.get, name_u))
 139             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_disappeared'),
 140                                                                  previously_disappeared + 1))
 141         else:
 142             d.addCallback(lambda ign: self.upload_dirnode.get(name_u))
 143             d.addCallback(download_to_data)
 144             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 145             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'),
 146                                                                  previously_uploaded + 1))
 147
 148         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_queued'), 0))
 149         return d
 150
 151
 152 class MockTest(DropUploadTestMixin, unittest.TestCase):
 153     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
 154
 155     def test_errors(self):
 156         self.basedir = "drop_upload.MockTest.test_errors"
 157         self.set_up_grid()
 158         errors_dir = os.path.join(self.basedir, "errors_dir")
 159         os.mkdir(errors_dir)
 160
 161         client = self.g.clients[0]
 162         d = client.create_dirnode()
 163         def _made_upload_dir(n):
 164             self.failUnless(IDirectoryNode.providedBy(n))
 165             upload_dircap = n.get_uri()
 166             readonly_dircap = n.get_readonly_uri()
 167
 168             self.shouldFail(AssertionError, 'invalid local dir', 'could not be represented',
 169                             DropUploader, client, upload_dircap, '\xFF', inotify=fake_inotify)
 170             self.shouldFail(AssertionError, 'non-existant local dir', 'there is no directory',
 171                             DropUploader, client, upload_dircap, os.path.join(self.basedir, "Laputa"), inotify=fake_inotify)
 172
 173             fp = filepath.FilePath(self.basedir).child('IM_NOT_A_DIR')
 174             fp.touch()
 175             self.shouldFail(AssertionError, 'non-existant local dir', 'is not a directory',
 176                             DropUploader, client, upload_dircap, fp.path, inotify=fake_inotify)
 177
 178             self.shouldFail(AssertionError, 'bad cap', 'does not refer to a directory',
 179                             DropUploader, client, 'bad', errors_dir, inotify=fake_inotify)
 180             self.shouldFail(AssertionError, 'non-directory cap', 'does not refer to a directory',
 181                             DropUploader, client, 'URI:LIT:foo', errors_dir, inotify=fake_inotify)
 182             self.shouldFail(AssertionError, 'readonly dircap', 'is not a writecap to a directory',
 183                             DropUploader, client, readonly_dircap, errors_dir, inotify=fake_inotify)
 184         d.addCallback(_made_upload_dir)
 185         return d
 186
 187     def test_drop_upload(self):
 188         self.inotify = fake_inotify
 189         self.basedir = "drop_upload.MockTest.test_drop_upload"
 190         return self._test()
 191
 192     def notify_close_write(self, path):
 193         self.uploader._notifier.event(path, self.inotify.IN_CLOSE_WRITE)
 194
 195
 196 class RealTest(DropUploadTestMixin, unittest.TestCase):
 197     """This is skipped unless both Twisted and the platform support inotify."""
 198
 199     def test_drop_upload(self):
 200         # We should always have runtime.platform.supportsINotify, because we're using
 201         # Twisted >= 10.1.
 202         if not runtime.platform.supportsINotify():
 203             raise unittest.SkipTest("Drop-upload support can only be tested for-real on an OS that supports inotify or equivalent.")
 204
 205         self.inotify = None  # use the appropriate inotify for the platform
 206         self.basedir = "drop_upload.RealTest.test_drop_upload"
 207         return self._test()
 208
 209     def notify_close_write(self, path):
 210         # Writing to the file causes the notification.
 211         pass