4 from twisted.trial import unittest
5 from twisted.python import filepath, runtime, log
6 from twisted.internet import defer
8 from allmydata.interfaces import IDirectoryNode, NoSuchChildError
10 from allmydata.util import fileutil, fake_inotify
11 from allmydata.util.encodingutil import get_filesystem_encoding
12 from allmydata.util.consumer import download_to_data
13 from allmydata.test.no_network import GridTestMixin
14 from allmydata.test.common_util import ReallyEqualMixin
15 from allmydata.test.common import ShouldFailMixin
17 from allmydata.frontends.drop_upload import DropUploader
20 class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin):
22 These tests will be run both with a mock notifier, and (on platforms that support it)
23 with the real INotify.
27 GridTestMixin.setUp(self)
28 self.nonascii_dirs = []
32 GridTestMixin.tearDown(self)
34 # kludge to work around the fact that buildbot can't remove a directory tree that has any non-ASCII directory names
35 if sys.platform == "win32":
36 for dirpath in self.nonascii_dirs:
38 fileutil.rm_dir(dirpath)
40 log.err("We were unable to delete a non-ASCII directory %r created by the test. "
41 "This is liable to cause failures on future builds." % (dirpath,))
43 def _mkdir_nonascii(self, dirpath):
44 self.nonascii_dirs.append(dirpath)
47 def _get_count(self, name):
48 return self.stats_provider.get_stats()["counters"].get(name, 0)
53 dirname_u = u"loc\u0101l_dir"
54 if sys.platform != "win32":
56 u"loc\u0101l_dir".encode(get_filesystem_encoding())
57 except UnicodeEncodeError:
58 dirname_u = u"local_dir"
59 self.local_dir = os.path.join(self.basedir, dirname_u)
60 self._mkdir_nonascii(self.local_dir)
62 self.client = self.g.clients[0]
63 self.stats_provider = self.client.stats_provider
65 d = self.client.create_dirnode()
66 def _made_upload_dir(n):
67 self.failUnless(IDirectoryNode.providedBy(n))
68 self.upload_dirnode = n
69 self.upload_dircap = n.get_uri()
70 self.uploader = DropUploader(self.client, self.upload_dircap, self.local_dir.encode('utf-8'),
72 return self.uploader.start()
73 d.addCallback(_made_upload_dir)
75 # Write something short enough for a LIT file.
76 d.addCallback(lambda ign: self._test_file(u"short", "test"))
78 # Write to the same file again with different data.
79 d.addCallback(lambda ign: self._test_file(u"short", "different"))
81 # Test that temporary files are not uploaded.
82 d.addCallback(lambda ign: self._test_file(u"tempfile", "test", temporary=True))
84 # Test that we tolerate creation of a subdirectory.
85 d.addCallback(lambda ign: os.mkdir(os.path.join(self.local_dir, u"directory")))
87 # Write something longer, and also try to test a Unicode name if the fs can represent it.
89 if sys.platform != "win32":
91 u"l\u00F8ng".encode(get_filesystem_encoding())
92 except UnicodeEncodeError:
94 d.addCallback(lambda ign: self._test_file(name_u, "test"*100))
96 # TODO: test that causes an upload failure.
97 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_failed'), 0))
99 # Prevent unclean reactor errors.
101 d = defer.succeed(None)
102 if self.uploader is not None:
103 d.addCallback(lambda ign: self.uploader.finish(for_tests=True))
104 d.addCallback(lambda ign: res)
109 def _test_file(self, name_u, data, temporary=False):
110 previously_uploaded = self._get_count('drop_upload.files_uploaded')
111 previously_disappeared = self._get_count('drop_upload.files_disappeared')
115 # Note: this relies on the fact that we only get one IN_CLOSE_WRITE notification per file
116 # (otherwise we would get a defer.AlreadyCalledError). Should we be relying on that?
117 self.uploader.set_uploaded_callback(d.callback)
119 path_u = os.path.join(self.local_dir, name_u)
120 if sys.platform == "win32":
121 path = filepath.FilePath(path_u)
123 path = filepath.FilePath(path_u.encode(get_filesystem_encoding()))
125 # We don't use FilePath.setContent() here because it creates a temporary file that
126 # is renamed into place, which causes events that the test is not expecting.
127 f = open(path.path, "wb")
129 if temporary and sys.platform != "win32":
134 if temporary and sys.platform == "win32":
136 self.notify_close_write(path)
139 d.addCallback(lambda ign: self.shouldFail(NoSuchChildError, 'temp file not uploaded', None,
140 self.upload_dirnode.get, name_u))
141 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_disappeared'),
142 previously_disappeared + 1))
144 d.addCallback(lambda ign: self.upload_dirnode.get(name_u))
145 d.addCallback(download_to_data)
146 d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
147 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'),
148 previously_uploaded + 1))
150 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_queued'), 0))
154 class MockTest(DropUploadTestMixin, unittest.TestCase):
155 """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
157 def test_errors(self):
158 self.basedir = "drop_upload.MockTest.test_errors"
160 errors_dir = os.path.join(self.basedir, "errors_dir")
163 client = self.g.clients[0]
164 d = client.create_dirnode()
165 def _made_upload_dir(n):
166 self.failUnless(IDirectoryNode.providedBy(n))
167 upload_dircap = n.get_uri()
168 readonly_dircap = n.get_readonly_uri()
170 self.shouldFail(AssertionError, 'invalid local.directory', 'could not be represented',
171 DropUploader, client, upload_dircap, '\xFF', inotify=fake_inotify)
172 self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
173 DropUploader, client, upload_dircap, os.path.join(self.basedir, "Laputa"), inotify=fake_inotify)
175 fp = filepath.FilePath(self.basedir).child('NOT_A_DIR')
177 self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
178 DropUploader, client, upload_dircap, fp.path, inotify=fake_inotify)
180 self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
181 DropUploader, client, 'bad', errors_dir, inotify=fake_inotify)
182 self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
183 DropUploader, client, 'URI:LIT:foo', errors_dir, inotify=fake_inotify)
184 self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
185 DropUploader, client, readonly_dircap, errors_dir, inotify=fake_inotify)
186 d.addCallback(_made_upload_dir)
189 def test_drop_upload(self):
190 self.inotify = fake_inotify
191 self.basedir = "drop_upload.MockTest.test_drop_upload"
194 def notify_close_write(self, path):
195 self.uploader._notifier.event(path, self.inotify.IN_CLOSE_WRITE)
198 class RealTest(DropUploadTestMixin, unittest.TestCase):
199 """This is skipped unless both Twisted and the platform support inotify."""
201 def test_drop_upload(self):
202 # We should always have runtime.platform.supportsINotify, because we're using
204 if not runtime.platform.supportsINotify():
205 raise unittest.SkipTest("Drop-upload support can only be tested for-real on an OS that supports inotify or equivalent.")
207 self.inotify = None # use the appropriate inotify for the platform
208 self.basedir = "drop_upload.RealTest.test_drop_upload"
211 def notify_close_write(self, path):
212 # Writing to the file causes the notification.