-#! /usr/bin/env python
-
-import os, shutil, sys, urllib, time, stat
+import os, shutil, sys, urllib, time, stat, urlparse
from cStringIO import StringIO
from twisted.internet import defer, reactor, protocol, error
from twisted.application import service, internet
from allmydata.immutable import upload
from allmydata.scripts import create_node
from allmydata.util import fileutil, pollmixin
-import foolscap
-from foolscap import eventual
+from allmydata.util.fileutil import abspath_expanduser_unicode
+from allmydata.util.encodingutil import get_filesystem_encoding
+from foolscap.api import Tub, fireEventually, flushEventualQueue
from twisted.python import log
class StallableHTTPGetterDiscarder(tw_client.HTTPPageGetter):
# adapted from twisted.web.client.getPage . We can't just wrap or
# subclass because it provides no way to override the HTTPClientFactory
# that it creates.
- scheme, host, port, path = tw_client._parse(url)
+ scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
+ assert scheme == 'http'
+ host, port = netloc, 80
+ if ":" in host:
+ host, port = host.split(":")
+ port = int(port)
factory = StallableDiscardingHTTPClientFactory(url, *args, **kwargs)
factory.do_stall = stall
- assert scheme == 'http'
reactor.connectTCP(host, port, factory)
return factory.deferred
pass
class SystemFramework(pollmixin.PollMixin):
- numnodes = 5
+ numnodes = 7
def __init__(self, basedir, mode):
- self.basedir = basedir = os.path.abspath(basedir)
- if not basedir.startswith(os.path.abspath(".")):
+ self.basedir = basedir = abspath_expanduser_unicode(unicode(basedir))
+ if not (basedir + os.path.sep).startswith(abspath_expanduser_unicode(u".") + os.path.sep):
raise AssertionError("safety issue: basedir must be a subdir")
self.testdir = testdir = os.path.join(basedir, "test")
if os.path.exists(testdir):
self.sparent = service.MultiService()
self.sparent.startService()
self.proc = None
- self.tub = foolscap.Tub()
+ self.tub = Tub()
+ self.tub.setOption("expose-remote-exception-types", False)
self.tub.setServiceParent(self.sparent)
self.mode = mode
self.failed = False
#logfile = open(os.path.join(self.testdir, "log"), "w")
#flo = log.FileLogObserver(logfile)
#log.startLoggingWithObserver(flo.emit, setStdout=False)
- d = eventual.fireEventually()
+ d = fireEventually()
d.addCallback(lambda res: self.setUp())
d.addCallback(lambda res: self.record_initial_memusage())
d.addCallback(lambda res: self.make_nodes())
def tearDown(self, passthrough):
# the client node will shut down in a few seconds
- #os.remove(os.path.join(self.clientdir, "suicide_prevention_hotline"))
+ #os.remove(os.path.join(self.clientdir, client.Client.EXIT_TRIGGER_FILE))
log.msg("shutting down SystemTest services")
if self.keepalive_file and os.path.exists(self.keepalive_file):
age = time.time() - os.stat(self.keepalive_file)[stat.ST_MTIME]
if self.proc:
d.addCallback(lambda res: self.kill_client())
d.addCallback(lambda res: self.sparent.stopService())
- d.addCallback(lambda res: eventual.flushEventualQueue())
+ d.addCallback(lambda res: flushEventualQueue())
def _close_statsfile(res):
self.statsfile.close()
d.addCallback(_close_statsfile)
for i in range(self.numnodes):
nodedir = os.path.join(self.testdir, "node%d" % i)
os.mkdir(nodedir)
- f = open(os.path.join(nodedir, "introducer.furl"), "w")
- f.write(self.introducer_furl)
- f.close()
+ f = open(os.path.join(nodedir, "tahoe.cfg"), "w")
+ f.write("[client]\n"
+ "introducer.furl = %s\n"
+ "shares.happy = 1\n"
+ "[storage]\n"
+ % (self.introducer_furl,))
# the only tests for which we want the internal nodes to actually
# retain shares are the ones where somebody's going to download
# them.
# for these tests, we tell the storage servers to pretend to
# accept shares, but really just throw them out, since we're
# only testing upload and not download.
- f = open(os.path.join(nodedir, "debug_no_storage"), "w")
- f.write("no_storage\n")
- f.close()
+ f.write("debug_discard = true\n")
if self.mode in ("receive",):
# for this mode, the client-under-test gets all the shares,
# so our internal nodes can refuse requests
- f = open(os.path.join(nodedir, "readonly_storage"), "w")
- f.write("\n")
- f.close()
+ f.write("readonly = true\n")
+ f.close()
c = self.add_service(client.Client(basedir=nodedir))
self.nodes.append(c)
# the peers will start running, eventually they will connect to each
def start_client(self):
# this returns a Deferred that fires with the client's control.furl
log.msg("MAKING CLIENT")
- clientdir = self.clientdir = os.path.join(self.testdir, "client")
+ # self.testdir is an absolute Unicode path
+ clientdir = self.clientdir = os.path.join(self.testdir, u"client")
+ clientdir_str = clientdir.encode(get_filesystem_encoding())
quiet = StringIO()
- create_node.create_client(clientdir, {}, out=quiet)
+ create_node.create_node({'basedir': clientdir}, out=quiet)
log.msg("DONE MAKING CLIENT")
- f = open(os.path.join(clientdir, "introducer.furl"), "w")
- f.write(self.introducer_furl + "\n")
- f.close()
-
+ # now replace tahoe.cfg
# set webport=0 and then ask the node what port it picked.
- f = open(os.path.join(clientdir, "webport"), "w")
- f.write("tcp:0:interface=127.0.0.1\n")
- f.close()
+ f = open(os.path.join(clientdir, "tahoe.cfg"), "w")
+ f.write("[node]\n"
+ "web.port = tcp:0:interface=127.0.0.1\n"
+ "[client]\n"
+ "introducer.furl = %s\n"
+ "shares.happy = 1\n"
+ "[storage]\n"
+ % (self.introducer_furl,))
if self.mode in ("upload-self", "receive"):
# accept and store shares, to trigger the memory consumption bugs
pass
else:
# don't accept any shares
- f = open(os.path.join(clientdir, "readonly_storage"), "w")
- f.write("true\n")
- f.close()
+ f.write("readonly = true\n")
## also, if we do receive any shares, throw them away
- #f = open(os.path.join(clientdir, "debug_no_storage"), "w")
- #f.write("no_storage\n")
- #f.close()
+ #f.write("debug_discard = true")
if self.mode == "upload-self":
pass
+ f.close()
self.keepalive_file = os.path.join(clientdir,
- "suicide_prevention_hotline")
+ client.Client.EXIT_TRIGGER_FILE)
# now start updating the mtime.
self.touch_keepalive()
ts = internet.TimerService(1.0, self.touch_keepalive)
logfile = os.path.join(self.basedir, "client.log")
cmd = ["twistd", "-n", "-y", "tahoe-client.tac", "-l", logfile]
env = os.environ.copy()
- self.proc = reactor.spawnProcess(pp, cmd[0], cmd, env, path=clientdir)
+ self.proc = reactor.spawnProcess(pp, cmd[0], cmd, env, path=clientdir_str)
log.msg("CLIENT STARTED")
# now we wait for the client to get started. we're looking for the
print
print "uploading %s" % name
if self.mode in ("upload", "upload-self"):
- files[name] = self.create_data(name, size)
- d = self.control_rref.callRemote("upload_from_file_to_uri",
- files[name], convergence="check-memory convergence string")
- def _done(uri):
- os.remove(files[name])
- del files[name]
- return uri
- d.addCallback(_done)
+ d = self.control_rref.callRemote("upload_random_data_from_file",
+ size,
+ convergence="check-memory")
elif self.mode == "upload-POST":
data = "a" * size
url = "/uri"
files[name] = self.create_data(name, size)
u = self.nodes[0].getServiceNamed("uploader")
d = self.nodes[0].debug_wait_for_client_connections(self.numnodes+1)
- d.addCallback(lambda res: u.upload(upload.FileName(files[name], convergence="check-memory convergence string")))
- d.addCallback(lambda results: results.uri)
+ d.addCallback(lambda res:
+ u.upload(upload.FileName(files[name],
+ convergence="check-memory")))
+ d.addCallback(lambda results: results.get_uri())
else:
raise ValueError("unknown mode=%s" % self.mode)
def _complete(uri):
uri = uris[name]
if self.mode == "download":
- d = self.control_rref.callRemote("download_from_uri_to_file",
- uri, "dummy.out")
+ d = self.control_rref.callRemote("download_to_tempfile_and_delete",
+ uri)
elif self.mode == "download-GET":
url = "/uri/%s" % uri
d = self.GET_discard(urllib.quote(url), stall=False)
mode = "upload"
if len(sys.argv) > 1:
mode = sys.argv[1]
+ if sys.maxint == 2147483647:
+ bits = "32"
+ elif sys.maxint == 9223372036854775807:
+ bits = "64"
+ else:
+ bits = "?"
+ print "%s-bit system (sys.maxint=%d)" % (bits, sys.maxint)
# put the logfile and stats.out in _test_memory/ . These stick around.
# put the nodes and other files in _test_memory/test/ . These are
# removed each time we run.