src/allmydata/test/test_drop_upload.py

   1
   2 import os, sys
   3
   4 from twisted.trial import unittest
   5 from twisted.python import filepath, runtime
   6 from twisted.internet import defer
   7
   8 from allmydata.interfaces import IDirectoryNode, NoSuchChildError
   9
  10 from allmydata.util import fake_inotify
  11 from allmydata.util.encodingutil import get_filesystem_encoding
  12 from allmydata.util.consumer import download_to_data
  13 from allmydata.test.no_network import GridTestMixin
  14 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
  15 from allmydata.test.common import ShouldFailMixin
  16
  17 from allmydata.frontends.drop_upload import DropUploader
  18
  19
  20 class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
  21     """
  22     These tests will be run both with a mock notifier, and (on platforms that support it)
  23     with the real INotify.
  24     """
  25
  26     def _get_count(self, name):
  27         return self.stats_provider.get_stats()["counters"].get(name, 0)
  28
  29     def _test(self):
  30         self.uploader = None
  31         self.set_up_grid()
  32         self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
  33         self.mkdir_nonascii(self.local_dir)
  34
  35         self.client = self.g.clients[0]
  36         self.stats_provider = self.client.stats_provider
  37
  38         d = self.client.create_dirnode()
  39         def _made_upload_dir(n):
  40             self.failUnless(IDirectoryNode.providedBy(n))
  41             self.upload_dirnode = n
  42             self.upload_dircap = n.get_uri()
  43             self.uploader = DropUploader(self.client, self.upload_dircap, self.local_dir.encode('utf-8'),
  44                                          inotify=self.inotify)
  45             return self.uploader.startService()
  46         d.addCallback(_made_upload_dir)
  47
  48         # Write something short enough for a LIT file.
  49         d.addCallback(lambda ign: self._test_file(u"short", "test"))
  50
  51         # Write to the same file again with different data.
  52         d.addCallback(lambda ign: self._test_file(u"short", "different"))
  53
  54         # Test that temporary files are not uploaded.
  55         d.addCallback(lambda ign: self._test_file(u"tempfile", "test", temporary=True))
  56
  57         # Test that we tolerate creation of a subdirectory.
  58         d.addCallback(lambda ign: os.mkdir(os.path.join(self.local_dir, u"directory")))
  59
  60         # Write something longer, and also try to test a Unicode name if the fs can represent it.
  61         name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
  62         d.addCallback(lambda ign: self._test_file(name_u, "test"*100))
  63
  64         # TODO: test that causes an upload failure.
  65         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_failed'), 0))
  66
  67         # Prevent unclean reactor errors.
  68         def _cleanup(res):
  69             d = defer.succeed(None)
  70             if self.uploader is not None:
  71                 d.addCallback(lambda ign: self.uploader.finish(for_tests=True))
  72             d.addCallback(lambda ign: res)
  73             return d
  74         d.addBoth(_cleanup)
  75         return d
  76
  77     def _test_file(self, name_u, data, temporary=False):
  78         previously_uploaded = self._get_count('drop_upload.files_uploaded')
  79         previously_disappeared = self._get_count('drop_upload.files_disappeared')
  80
  81         d = defer.Deferred()
  82
  83         # Note: this relies on the fact that we only get one IN_CLOSE_WRITE notification per file
  84         # (otherwise we would get a defer.AlreadyCalledError). Should we be relying on that?
  85         self.uploader.set_uploaded_callback(d.callback)
  86
  87         path_u = os.path.join(self.local_dir, name_u)
  88         if sys.platform == "win32":
  89             path = filepath.FilePath(path_u)
  90         else:
  91             path = filepath.FilePath(path_u.encode(get_filesystem_encoding()))
  92
  93         # We don't use FilePath.setContent() here because it creates a temporary file that
  94         # is renamed into place, which causes events that the test is not expecting.
  95         f = open(path.path, "wb")
  96         try:
  97             if temporary and sys.platform != "win32":
  98                 os.unlink(path.path)
  99             f.write(data)
 100         finally:
 101             f.close()
 102         if temporary and sys.platform == "win32":
 103             os.unlink(path.path)
 104         self.notify_close_write(path)
 105
 106         if temporary:
 107             d.addCallback(lambda ign: self.shouldFail(NoSuchChildError, 'temp file not uploaded', None,
 108                                                       self.upload_dirnode.get, name_u))
 109             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_disappeared'),
 110                                                                  previously_disappeared + 1))
 111         else:
 112             d.addCallback(lambda ign: self.upload_dirnode.get(name_u))
 113             d.addCallback(download_to_data)
 114             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 115             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'),
 116                                                                  previously_uploaded + 1))
 117
 118         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_queued'), 0))
 119         return d
 120
 121
 122 class MockTest(DropUploadTestMixin, unittest.TestCase):
 123     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
 124
 125     def test_errors(self):
 126         self.basedir = "drop_upload.MockTest.test_errors"
 127         self.set_up_grid()
 128         errors_dir = os.path.join(self.basedir, "errors_dir")
 129         os.mkdir(errors_dir)
 130
 131         client = self.g.clients[0]
 132         d = client.create_dirnode()
 133         def _made_upload_dir(n):
 134             self.failUnless(IDirectoryNode.providedBy(n))
 135             upload_dircap = n.get_uri()
 136             readonly_dircap = n.get_readonly_uri()
 137
 138             self.shouldFail(AssertionError, 'invalid local.directory', 'could not be represented',
 139                             DropUploader, client, upload_dircap, '\xFF', inotify=fake_inotify)
 140             self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
 141                             DropUploader, client, upload_dircap, os.path.join(self.basedir, "Laputa"), inotify=fake_inotify)
 142
 143             fp = filepath.FilePath(self.basedir).child('NOT_A_DIR')
 144             fp.touch()
 145             self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
 146                             DropUploader, client, upload_dircap, fp.path, inotify=fake_inotify)
 147
 148             self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
 149                             DropUploader, client, 'bad', errors_dir, inotify=fake_inotify)
 150             self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
 151                             DropUploader, client, 'URI:LIT:foo', errors_dir, inotify=fake_inotify)
 152             self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
 153                             DropUploader, client, readonly_dircap, errors_dir, inotify=fake_inotify)
 154         d.addCallback(_made_upload_dir)
 155         return d
 156
 157     def test_drop_upload(self):
 158         self.inotify = fake_inotify
 159         self.basedir = "drop_upload.MockTest.test_drop_upload"
 160         return self._test()
 161
 162     def notify_close_write(self, path):
 163         self.uploader._notifier.event(path, self.inotify.IN_CLOSE_WRITE)
 164
 165
 166 class RealTest(DropUploadTestMixin, unittest.TestCase):
 167     """This is skipped unless both Twisted and the platform support inotify."""
 168
 169     def test_drop_upload(self):
 170         # We should always have runtime.platform.supportsINotify, because we're using
 171         # Twisted >= 10.1.
 172         if not runtime.platform.supportsINotify():
 173             raise unittest.SkipTest("Drop-upload support can only be tested for-real on an OS that supports inotify or equivalent.")
 174
 175         self.inotify = None  # use the appropriate inotify for the platform
 176         self.basedir = "drop_upload.RealTest.test_drop_upload"
 177         return self._test()
 178
 179     def notify_close_write(self, path):
 180         # Writing to the file causes the notification.
 181         pass