From f6b2072af112ba6fef8c01a560c456147554cc31 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 14 Dec 2007 02:05:31 -0700 Subject: [PATCH] check-speed: test SSK upload/download speed too. SDMF imposes a limit on the file sizes, no 10MB or 100MB test --- src/allmydata/control.py | 24 +++++++++++------ src/allmydata/interfaces.py | 5 ++-- src/allmydata/test/check_speed.py | 44 +++++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/src/allmydata/control.py b/src/allmydata/control.py index f465b3de..5e57f229 100644 --- a/src/allmydata/control.py +++ b/src/allmydata/control.py @@ -6,6 +6,7 @@ from twisted.internet import defer from foolscap import Referenceable from allmydata.interfaces import RIControlClient from allmydata.util import testutil, fileutil, mathutil +from allmydata import upload, download from twisted.python import log def get_memory_usage(): @@ -52,10 +53,11 @@ class ControlServer(Referenceable, service.Service, testutil.PollMixin): d.addCallback(lambda res: filename) return d - def remote_speed_test(self, count, size): + def remote_speed_test(self, count, size, mutable): assert size > 8 - log.msg("speed_test: count=%d, size=%d" % (count, size)) - st = SpeedTest(self.parent, count, size) + log.msg("speed_test: count=%d, size=%d, mutable=%d" % (count, size, + mutable)) + st = SpeedTest(self.parent, count, size, mutable) return st.run() def remote_get_memory_usage(self): @@ -95,10 +97,11 @@ class ControlServer(Referenceable, service.Service, testutil.PollMixin): return d class SpeedTest: - def __init__(self, parent, count, size): + def __init__(self, parent, count, size, mutable): self.parent = parent self.count = count self.size = size + self.mutable = mutable self.uris = {} self.basedir = os.path.join(self.parent.basedir, "_speed_test_data") @@ -127,7 +130,6 @@ class SpeedTest: f.close() def do_upload(self): - uploader = self.parent.getServiceNamed("uploader") start = time.time() d = defer.succeed(None) def _record_uri(uri, i): @@ -136,7 +138,13 @@ class SpeedTest: if i >= self.count: return fn = os.path.join(self.basedir, str(i)) - d1 = uploader.upload_filename(fn) + if self.mutable: + data = open(fn,"rb").read() + d1 = self.parent.create_mutable_file(data) + d1.addCallback(lambda n: n.get_uri()) + else: + up = upload.FileName(fn) + d1 = self.parent.upload(up) d1.addCallback(_record_uri, i) d1.addCallback(_upload_one_file, i+1) return d1 @@ -148,13 +156,13 @@ class SpeedTest: return d def do_download(self): - downloader = self.parent.getServiceNamed("downloader") start = time.time() d = defer.succeed(None) def _download_one_file(ignored, i): if i >= self.count: return - d1 = downloader.download_to_filehandle(self.uris[i], Discard()) + n = self.parent.create_node_from_uri(self.uris[i]) + d1 = n.download(download.FileHandle(Discard())) d1.addCallback(_download_one_file, i+1) return d1 d.addCallback(_download_one_file, 0) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 46f720bc..9239dabf 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1219,10 +1219,11 @@ class RIControlClient(RemoteInterface): measuring memory consupmtion in bytes.""" return DictOf(str, int) - def speed_test(count=int, size=int): + def speed_test(count=int, size=int, mutable=bool): """Write 'count' tempfiles to disk, all of the given size. Measure how long (in seconds) it takes to upload them all to the servers. - Then measure how long it takes to download all of them. + Then measure how long it takes to download all of them. If 'mutable' + is True, use mutable files instead of immutable ones. Returns a tuple of (upload_time, download_time). """ diff --git a/src/allmydata/test/check_speed.py b/src/allmydata/test/check_speed.py index 586a3633..c9196cd4 100644 --- a/src/allmydata/test/check_speed.py +++ b/src/allmydata/test/check_speed.py @@ -62,8 +62,8 @@ class SpeedTest: print "TIME (%s): %s up, %s down" % (key, times[0], times[1]) self.upload_times[key], self.download_times[key] = times - def one_test(self, res, name, count, size): - d = self.client_rref.callRemote("speed_test", count, size) + def one_test(self, res, name, count, size, mutable): + d = self.client_rref.callRemote("speed_test", count, size, mutable) d.addCallback(self.record_times, name) return d @@ -87,24 +87,27 @@ class SpeedTest: print "doing test" rr = self.client_rref d = defer.succeed(None) - d.addCallback(self.one_test, "startup", 1, 1000) # ignore this one + d.addCallback(self.one_test, "startup", 1, 1000, False) #ignore this one d.addCallback(self.measure_rtt) - d.addCallback(self.one_test, "1x 200B", 1, 200) - d.addCallback(self.one_test, "10x 200B", 10, 200) + d.addCallback(self.one_test, "1x 200B", 1, 200, False) + d.addCallback(self.one_test, "10x 200B", 10, 200, False) def _maybe_do_100x_200B(res): if self.upload_times["10x 200B"] < 5: print "10x 200B test went too fast, doing 100x 200B test" - return self.one_test(None, "100x 200B", 100, 200) + return self.one_test(None, "100x 200B", 100, 200, False) return - d.addCallback(_maybe_do_100x_200B) - d.addCallback(self.one_test, "1MB", 1, 1*MB) - d.addCallback(self.one_test, "10MB", 1, 10*MB) + #d.addCallback(_maybe_do_100x_200B) + d.addCallback(self.one_test, "1MB", 1, 1*MB, False) + d.addCallback(self.one_test, "10MB", 1, 10*MB, False) def _maybe_do_100MB(res): if self.upload_times["10MB"] > 30: print "10MB test took too long, skipping 100MB test" return - return self.one_test(None, "100MB", 1, 100*MB) - d.addCallback(_maybe_do_100MB) + return self.one_test(None, "100MB", 1, 100*MB, False) + #d.addCallback(_maybe_do_100MB) + d.addCallback(self.one_test, "1x 200B SSK", 1, 200, True) + d.addCallback(self.one_test, "10x 200B SSK", 10, 200, True) + d.addCallback(self.one_test, "1MB SSK", 1, 1*MB, True) d.addCallback(self.calculate_speeds) return d @@ -144,6 +147,25 @@ class SpeedTest: A3 = 100*MB / (self.download_times["100MB"] - B) print "download speed (100MB):", self.number(A3, "Bps") + + # upload SSK + if "100x 200B SSK" in self.upload_times: + B = self.upload_times["100x 200B SSK"] / 100 + else: + B = self.upload_times["10x 200B SSK"] / 10 + print "upload per-file time SSK: %.3fs" % B + A1 = 1*MB / (self.upload_times["1MB SSK"] - B) # in bytes per second + print "upload speed SSK (1MB):", self.number(A1, "Bps") + + # download SSK + if "100x 200B SSK" in self.download_times: + B = self.download_times["100x 200B SSK"] / 100 + else: + B = self.download_times["10x 200B SSK"] / 10 + print "download per-file time SSK: %.3fs" % B + A1 = 1*MB / (self.download_times["1MB SSK"] - B) # in bytes per second + print "download speed SSK (1MB):", self.number(A1, "Bps") + def number(self, value, suffix=""): scaling = 1 if value < 1: -- 2.45.2