]> git.rkrishnan.org Git - tahoe-lafs/tahoe-lafs.git/blobdiff - src/allmydata/test/test_system.py
remove "announcement_distinct_hosts" from introweb JSON
[tahoe-lafs/tahoe-lafs.git] / src / allmydata / test / test_system.py
index f3d618d6656066de80bf33012de4d4beccbbec39..1887c7e28b2e4fa4652a9c64c50e8e10c2a0494f 100644 (file)
@@ -1,6 +1,7 @@
-from base64 import b32encode
-import os, sys, time, simplejson
+
+import os, re, sys, time, simplejson
 from cStringIO import StringIO
+
 from twisted.trial import unittest
 from twisted.internet import defer
 from twisted.internet import threads # CLI tests use deferToThread
@@ -14,7 +15,8 @@ from allmydata.immutable.literal import LiteralFileNode
 from allmydata.immutable.filenode import ImmutableFileNode
 from allmydata.util import idlib, mathutil
 from allmydata.util import log, base32
-from allmydata.util.encodingutil import quote_output, unicode_to_argv, get_filesystem_encoding
+from allmydata.util.verlib import NormalizedVersion
+from allmydata.util.encodingutil import quote_output, unicode_to_argv
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.util.consumer import MemoryConsumer, download_to_data
 from allmydata.scripts import runner
@@ -24,6 +26,8 @@ from allmydata.monitor import Monitor
 from allmydata.mutable.common import NotWriteableError
 from allmydata.mutable import layout as mutable_layout
 from allmydata.mutable.publish import MutableData
+
+import foolscap
 from foolscap.api import DeadReferenceError, fireEventually
 from twisted.python.failure import Failure
 from twisted.web.client import getPage
@@ -98,7 +102,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d = self.set_up_nodes()
         def _check_connections(res):
             for c in self.clients:
-                c.DEFAULT_ENCODING_PARAMETERS['happy'] = 5
+                c.encoding_params['happy'] = 5
                 all_peerids = c.get_storage_broker().get_all_serverids()
                 self.failUnlessEqual(len(all_peerids), self.numclients)
                 sb = c.storage_broker
@@ -210,7 +214,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                                                       add_to_sparent=True))
         def _added(extra_node):
             self.extra_node = extra_node
-            self.extra_node.DEFAULT_ENCODING_PARAMETERS['happy'] = 5
+            self.extra_node.encoding_params['happy'] = 5
         d.addCallback(_added)
 
         def _has_helper():
@@ -720,7 +724,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d = self.set_up_nodes(use_stats_gatherer=True)
         def _new_happy_semantics(ign):
             for c in self.clients:
-                c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+                c.encoding_params['happy'] = 1
         d.addCallback(_new_happy_semantics)
         d.addCallback(self._test_introweb)
         d.addCallback(self.log, "starting publish")
@@ -808,8 +812,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                                      {"storage": 5})
                 self.failUnlessEqual(data["announcement_summary"],
                                      {"storage": 5})
-                self.failUnlessEqual(data["announcement_distinct_hosts"],
-                                     {"storage": 1})
             except unittest.FailTest:
                 print
                 print "GET %s?t=json output was:" % self.introweb_url
@@ -1093,16 +1095,15 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         public = "uri/" + self._root_directory_uri
         d = getPage(base)
         def _got_welcome(page):
-            # XXX This test is oversensitive to formatting
-            expected = "Connected to <span>%d</span>\n     of <span>%d</span> known storage servers:" % (self.numclients, self.numclients)
-            self.failUnless(expected in page,
-                            "I didn't see the right 'connected storage servers'"
-                            " message in: %s" % page
-                            )
-            expected = "<th>My nodeid:</th> <td class=\"nodeid mine data-chars\">%s</td>" % (b32encode(self.clients[0].nodeid).lower(),)
-            self.failUnless(expected in page,
-                            "I didn't see the right 'My nodeid' message "
-                            "in: %s" % page)
+            html = page.replace('\n', ' ')
+            connected_re = r'Connected to <span>%d</span>\s*of <span>%d</span> known storage servers' % (self.numclients, self.numclients)
+            self.failUnless(re.search(connected_re, html),
+                            "I didn't see the right '%s' message in:\n%s" % (connected_re, page))
+            # nodeids/tubids don't have any regexp-special characters
+            nodeid_re = r'<th>Node ID:</th>\s*<td title="TubID: %s">%s</td>' % (
+                self.clients[0].get_long_tubid(), self.clients[0].get_long_nodeid())
+            self.failUnless(re.search(nodeid_re, html),
+                            "I didn't see the right '%s' message in:\n%s" % (nodeid_re, page))
             self.failUnless("Helper: 0 active uploads" in page)
         d.addCallback(_got_welcome)
         d.addCallback(self.log, "done with _got_welcome")
@@ -1110,9 +1111,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # get the welcome page from the node that uses the helper too
         d.addCallback(lambda res: getPage(self.helper_webish_url))
         def _got_welcome_helper(page):
-            self.failUnless("Connected to helper?: <span>yes</span>" in page,
-                            page)
-            self.failUnless("Not running helper" in page)
+            html = page.replace('\n', ' ')
+            self.failUnless(re.search('<img src="img/connected-yes.png" alt="Connected" />', html), page)
+            self.failUnlessIn("Not running helper", page)
         d.addCallback(_got_welcome_helper)
 
         d.addCallback(lambda res: getPage(base + public))
@@ -1168,7 +1169,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         def _new_happy_semantics(ign):
             for c in self.clients:
                 # these get reset somewhere? Whatever.
-                c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+                c.encoding_params['happy'] = 1
         d.addCallback(_new_happy_semantics)
         d.addCallback(lambda res: self.PUT(public + "/subdir3/big.txt",
                                            "big" * 500000)) # 1.5MB
@@ -1270,7 +1271,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # itself) doesn't explode when you ask for its status
         d.addCallback(lambda res: getPage(self.helper_webish_url + "status/"))
         def _got_non_helper_status(res):
-            self.failUnless("Upload and Download Status" in res)
+            self.failUnlessIn("Recent and Active Operations", res)
         d.addCallback(_got_non_helper_status)
 
         # or for helper status with t=json
@@ -1284,8 +1285,8 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # see if the statistics page exists
         d.addCallback(lambda res: self.GET("statistics"))
         def _got_stats(res):
-            self.failUnless("Node Statistics" in res)
-            self.failUnless("  'downloader.files_downloaded': 5," in res, res)
+            self.failUnlessIn("Operational Statistics", res)
+            self.failUnlessIn("  'downloader.files_downloaded': 5,", res)
         d.addCallback(_got_stats)
         d.addCallback(lambda res: self.GET("statistics?t=json"))
         def _got_stats_json(res):
@@ -1396,18 +1397,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(self._test_control2, control_furl_file)
         return d
     def _test_control2(self, rref, filename):
-        d = rref.callRemote("upload_from_file_to_uri",
-                            filename.encode(get_filesystem_encoding()), convergence=None)
-        downfile = os.path.join(self.basedir, "control.downfile").encode(get_filesystem_encoding())
-        d.addCallback(lambda uri:
-                      rref.callRemote("download_from_uri_to_file",
-                                      uri, downfile))
-        def _check(res):
-            self.failUnlessEqual(res, downfile)
-            data = open(downfile, "r").read()
-            expected_data = open(filename, "r").read()
-            self.failUnlessEqual(data, expected_data)
-        d.addCallback(_check)
+        d = defer.succeed(None)
         d.addCallback(lambda res: rref.callRemote("speed_test", 1, 200, False))
         if sys.platform in ("linux2", "linux3"):
             d.addCallback(lambda res: rref.callRemote("get_memory_usage"))
@@ -1438,7 +1428,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
         def run(ignored, verb, *args, **kwargs):
             stdin = kwargs.get("stdin", "")
-            newargs = [verb] + nodeargs + list(args)
+            newargs = nodeargs + [verb] + list(args)
             return self._run_cli(newargs, stdin=stdin)
 
         def _check_ls((out,err), expected_children, unexpected_children=[]):
@@ -1682,7 +1672,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         open(os.path.join(sdn2, "rfile5"), "wb").write("rfile5")
 
         # from disk into tahoe
-        d.addCallback(run, "cp", "-r", dn, "tahoe:dir1")
+        d.addCallback(run, "cp", "-r", dn, "tahoe:")
         d.addCallback(run, "ls")
         d.addCallback(_check_ls, ["dir1"])
         d.addCallback(run, "ls", "dir1")
@@ -1700,7 +1690,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         def _check_cp_r_out((out,err)):
             def _cmp(name):
                 old = open(os.path.join(dn, name), "rb").read()
-                newfn = os.path.join(dn_copy, name)
+                newfn = os.path.join(dn_copy, "dir1", name)
                 self.failUnless(os.path.exists(newfn))
                 new = open(newfn, "rb").read()
                 self.failUnlessEqual(old, new)
@@ -1719,7 +1709,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(run, "cp", "-r", "--caps-only", "tahoe:dir1", dn_copy2)
         def _check_capsonly((out,err)):
             # these should all be LITs
-            x = open(os.path.join(dn_copy2, "subdir2", "rfile4")).read()
+            x = open(os.path.join(dn_copy2, "dir1", "subdir2", "rfile4")).read()
             y = uri.from_string_filenode(x)
             self.failUnlessEqual(y.data, "rfile4")
         d.addCallback(_check_capsonly)
@@ -1728,13 +1718,13 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(run, "cp", "-r", "tahoe:dir1", "tahoe:dir1-copy")
         d.addCallback(run, "ls")
         d.addCallback(_check_ls, ["dir1", "dir1-copy"])
-        d.addCallback(run, "ls", "dir1-copy")
+        d.addCallback(run, "ls", "dir1-copy/dir1")
         d.addCallback(_check_ls, ["rfile1", "rfile2", "rfile3", "subdir2"],
                       ["rfile4", "rfile5"])
-        d.addCallback(run, "ls", "tahoe:dir1-copy/subdir2")
+        d.addCallback(run, "ls", "tahoe:dir1-copy/dir1/subdir2")
         d.addCallback(_check_ls, ["rfile4", "rfile5"],
                       ["rfile1", "rfile2", "rfile3"])
-        d.addCallback(run, "get", "dir1-copy/subdir2/rfile4")
+        d.addCallback(run, "get", "dir1-copy/dir1/subdir2/rfile4")
         d.addCallback(_check_stdout_against, data="rfile4")
 
         # and copy it a second time, which ought to overwrite the same files
@@ -1743,7 +1733,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # tahoe_ls doesn't currently handle the error correctly: it tries to
         # JSON-parse a traceback.
 ##         def _ls_missing(res):
-##             argv = ["ls"] + nodeargs + ["bogus"]
+##             argv = nodeargs + ["ls", "bogus"]
 ##             return self._run_cli(argv)
 ##         d.addCallback(_ls_missing)
 ##         def _check_ls_missing((out,err)):
@@ -1761,13 +1751,13 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d = self.set_up_nodes()
         def _new_happy_semantics(ign):
             for c in self.clients:
-                c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+                c.encoding_params['happy'] = 1
         d.addCallback(_new_happy_semantics)
 
         def _run_in_subprocess(ignored, verb, *args, **kwargs):
             stdin = kwargs.get("stdin")
             env = kwargs.get("env")
-            newargs = [verb, "--node-directory", self.getdir("client0")] + list(args)
+            newargs = ["--node-directory", self.getdir("client0"), verb] + list(args)
             return self.run_bintahoe(newargs, stdin=stdin, env=env)
 
         def _check_succeeded(res, check_stderr=True):
@@ -1883,3 +1873,39 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             return d
         d.addCallback(_got_lit_filenode)
         return d
+
+
+class Connections(SystemTestMixin, unittest.TestCase):
+    def test_rref(self):
+        if NormalizedVersion(foolscap.__version__) < NormalizedVersion('0.6.4'):
+            raise unittest.SkipTest("skipped due to http://foolscap.lothar.com/trac/ticket/196 "
+                                    "(which does not affect normal usage of Tahoe-LAFS)")
+
+        self.basedir = "system/Connections/rref"
+        d = self.set_up_nodes(2)
+        def _start(ign):
+            self.c0 = self.clients[0]
+            nonclients = [s for s in self.c0.storage_broker.get_connected_servers()
+                          if s.get_serverid() != self.c0.nodeid]
+            self.failUnlessEqual(len(nonclients), 1)
+
+            self.s1 = nonclients[0]  # s1 is the server, not c0
+            self.s1_rref = self.s1.get_rref()
+            self.failIfEqual(self.s1_rref, None)
+            self.failUnless(self.s1.is_connected())
+        d.addCallback(_start)
+
+        # now shut down the server
+        d.addCallback(lambda ign: self.clients[1].disownServiceParent())
+        # and wait for the client to notice
+        def _poll():
+            return len(self.c0.storage_broker.get_connected_servers()) < 2
+        d.addCallback(lambda ign: self.poll(_poll))
+
+        def _down(ign):
+            self.failIf(self.s1.is_connected())
+            rref = self.s1.get_rref()
+            self.failUnless(rref)
+            self.failUnlessIdentical(rref, self.s1_rref)
+        d.addCallback(_down)
+        return d