From: Brian Warner Date: Thu, 20 Sep 2007 23:55:33 +0000 (-0700) Subject: check_speed: upload multiple files, measure Ax+B X-Git-Tag: allmydata-tahoe-0.6.0~36 X-Git-Url: https://git.rkrishnan.org/pf/content/en/seg/class-simplejson.JSONDecoder-index.html?a=commitdiff_plain;h=2632c0ad8c589e5797030365bcc10aa5022535df;p=tahoe-lafs%2Ftahoe-lafs.git check_speed: upload multiple files, measure Ax+B --- diff --git a/Makefile b/Makefile index 8f8b700c..17cc4ef2 100644 --- a/Makefile +++ b/Makefile @@ -212,6 +212,7 @@ check-memory-once: .built check-speed: .built if [ -z '$(TESTCLIENTDIR)' ]; then exit 1; fi $(PYTHON) bin/allmydata-tahoe restart -f $(TESTCLIENTDIR) + sleep 1 $(PYTHON) src/allmydata/test/check_speed.py $(TESTCLIENTDIR) test-darcs-boringfile: diff --git a/src/allmydata/control.py b/src/allmydata/control.py index 40d3a3d8..39289638 100644 --- a/src/allmydata/control.py +++ b/src/allmydata/control.py @@ -2,9 +2,10 @@ import os, time from zope.interface import implements from twisted.application import service +from twisted.internet import defer from foolscap import Referenceable from allmydata.interfaces import RIControlClient -from allmydata.util import testutil, idlib +from allmydata.util import testutil, fileutil from twisted.python import log def get_memory_usage(): @@ -46,26 +47,43 @@ class ControlServer(Referenceable, service.Service, testutil.PollMixin): d.addCallback(lambda res: filename) return d - def remote_upload_speed_test(self, size): + def remote_upload_speed_test(self, count, size): assert size > 8 - fn = os.path.join(self.parent.basedir, idlib.b2a(os.urandom(8))) - f = open(fn, "w") - f.write(os.urandom(8)) - size -= 8 - while size > 0: - chunk = min(size, 4096) - f.write("\x00" * chunk) - size -= chunk - f.close() + basedir = os.path.join(self.parent.basedir, "_speed_test_data") + log.msg("speed_test: count=%d, size=%d" % (count, size)) + fileutil.make_dirs(basedir) + for i in range(count): + s = size + fn = os.path.join(basedir, str(i)) + if os.path.exists(fn): + os.unlink(fn) + f = open(fn, "w") + f.write(os.urandom(8)) + s -= 8 + while s > 0: + chunk = min(s, 4096) + f.write("\x00" * chunk) + s -= chunk + f.close() uploader = self.parent.getServiceNamed("uploader") start = time.time() - d = uploader.upload_filename(fn) - def _done(uri): + d = defer.succeed(None) + def _do_one_file(uri, i): + if i >= count: + return + fn = os.path.join(basedir, str(i)) + d1 = uploader.upload_filename(fn) + d1.addCallback(_do_one_file, i+1) + return d1 + d.addCallback(_do_one_file, 0) + def _done(ignored): stop = time.time() return stop - start d.addCallback(_done) def _cleanup(res): - os.unlink(fn) + for i in range(count): + fn = os.path.join(basedir, str(i)) + os.unlink(fn) return res d.addBoth(_cleanup) return d diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 2e035808..47de6857 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1054,8 +1054,8 @@ class RIControlClient(RemoteInterface): measuring memory consupmtion in bytes.""" return DictOf(str, int) - def upload_speed_test(size=int): - """Write a tempfile to disk of the given size. Measure how long - it takes to upload it to the servers. + def upload_speed_test(count=int, size=int): + """Write 'count' tempfiles to disk, all of the given size. Measure + how long (in seconds) it takes to upload them all to the servers. """ return float diff --git a/src/allmydata/test/check_speed.py b/src/allmydata/test/check_speed.py index 4002237e..fe162349 100644 --- a/src/allmydata/test/check_speed.py +++ b/src/allmydata/test/check_speed.py @@ -6,6 +6,8 @@ from twisted.python import log from twisted.application import service from foolscap import Tub, eventual +MB = 1000000 + class SpeedTest: def __init__(self, test_client_dir): #self.real_stderr = sys.stderr @@ -59,6 +61,11 @@ class SpeedTest: print "TIME (%s): %s" % (key, time) self.times[key] = time + def one_test(self, res, name, count, size): + d = self.client_rref.callRemote("upload_speed_test", count, size) + d.addCallback(self.record_time, name) + return d + def do_test(self): print "doing test" rr = self.client_rref @@ -66,14 +73,48 @@ class SpeedTest: def _got(res): print "MEMORY USAGE:", res d.addCallback(_got) - d.addCallback(lambda res: rr.callRemote("upload_speed_test", 1000)) - d.addCallback(self.record_time, "startup") - d.addCallback(lambda res: rr.callRemote("upload_speed_test", int(1e6))) - d.addCallback(self.record_time, "1MB.1") - d.addCallback(lambda res: rr.callRemote("upload_speed_test", int(1e6))) - d.addCallback(self.record_time, "1MB.2") + d.addCallback(self.one_test, "startup", 1, 1000) # ignore this one + d.addCallback(self.one_test, "1x 200B", 1, 200) + d.addCallback(self.one_test, "10x 200B", 10, 200) + #d.addCallback(self.one_test, "100x 200B", 100, 200) + d.addCallback(self.one_test, "1MB", 1, 1*MB) + d.addCallback(self.one_test, "10MB", 1, 10*MB) + d.addCallback(self.calculate_speed) return d + def calculate_speed(self, res): + perfile = self.times["1x 200B"] + # time = A*size+B + # we assume that A*200bytes is negligible + B = self.times["10x 200B"] / 10 + print "per-file time: %.3fs" % B + A1 = 1*MB / (self.times["1MB"] - B) # in bytes per second + print "speed (1MB):", self.number(A1, "Bps") + A2 = 10*MB / (self.times["10MB"] - B) + print "speed (10MB):", self.number(A2, "Bps") + + def number(self, value, suffix=""): + scaling = 1 + if value < 1: + fmt = "%1.2g%s" + elif value < 100: + fmt = "%.1f%s" + elif value < 1000: + fmt = "%d%s" + elif value < 1e6: + fmt = "%.2fk%s"; scaling = 1e3 + elif value < 1e9: + fmt = "%.2fM%s"; scaling = 1e6 + elif value < 1e12: + fmt = "%.2fG%s"; scaling = 1e9 + elif value < 1e15: + fmt = "%.2fT%s"; scaling = 1e12 + elif value < 1e18: + fmt = "%.2fP%s"; scaling = 1e15 + else: + fmt = "huge! %g%s" + return fmt % (value / scaling, suffix) + def tearDown(self, res): d = self.base_service.stopService() d.addCallback(lambda ignored: res)