From: Brian Warner Date: Wed, 30 Jan 2008 03:44:32 +0000 (-0700) Subject: speedcheck: track SSK creation time separately X-Git-Tag: allmydata-tahoe-0.8.0~204 X-Git-Url: https://git.rkrishnan.org/components/architecture.txt?a=commitdiff_plain;h=492cb92dc8f5c78853749fd4851332d154401fd5;p=tahoe-lafs%2Ftahoe-lafs.git speedcheck: track SSK creation time separately --- diff --git a/src/allmydata/control.py b/src/allmydata/control.py index 5e57f229..2fc3afc3 100644 --- a/src/allmydata/control.py +++ b/src/allmydata/control.py @@ -55,7 +55,7 @@ class ControlServer(Referenceable, service.Service, testutil.PollMixin): def remote_speed_test(self, count, size, mutable): assert size > 8 - log.msg("speed_test: count=%d, size=%d, mutable=%d" % (count, size, + log.msg("speed_test: count=%d, size=%d, mutable=%s" % (count, size, mutable)) st = SpeedTest(self.parent, count, size, mutable) return st.run() @@ -101,7 +101,7 @@ class SpeedTest: self.parent = parent self.count = count self.size = size - self.mutable = mutable + self.mutable_mode = mutable self.uris = {} self.basedir = os.path.join(self.parent.basedir, "_speed_test_data") @@ -130,18 +130,33 @@ class SpeedTest: f.close() def do_upload(self): - start = time.time() d = defer.succeed(None) + def _create_slot(res): + d1 = self.parent.create_mutable_file("") + def _created(n): + self._n = n + d1.addCallback(_created) + return d1 + if self.mutable_mode == "upload": + d.addCallback(_create_slot) + def _start(res): + self._start = time.time() + d.addCallback(_start) + def _record_uri(uri, i): self.uris[i] = uri def _upload_one_file(ignored, i): if i >= self.count: return fn = os.path.join(self.basedir, str(i)) - if self.mutable: + if self.mutable_mode == "create": data = open(fn,"rb").read() d1 = self.parent.create_mutable_file(data) d1.addCallback(lambda n: n.get_uri()) + elif self.mutable_mode == "upload": + data = open(fn,"rb").read() + d1 = self._n.replace(data) + d1.addCallback(lambda res: self._n.get_uri()) else: up = upload.FileName(fn) d1 = self.parent.upload(up) @@ -151,7 +166,7 @@ class SpeedTest: d.addCallback(_upload_one_file, 0) def _upload_done(ignored): stop = time.time() - self.upload_time = stop - start + self.upload_time = stop - self._start d.addCallback(_upload_done) return d diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 54a16d72..bbc354ee 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1245,11 +1245,13 @@ class RIControlClient(RemoteInterface): measuring memory consupmtion in bytes.""" return DictOf(str, int) - def speed_test(count=int, size=int, mutable=bool): + def speed_test(count=int, size=int, mutable=Any()): """Write 'count' tempfiles to disk, all of the given size. Measure how long (in seconds) it takes to upload them all to the servers. Then measure how long it takes to download all of them. If 'mutable' - is True, use mutable files instead of immutable ones. + is 'create', time creation of mutable files. If 'mutable' is + 'upload', then time access to the same mutable file instead of + creating one. Returns a tuple of (upload_time, download_time). """ diff --git a/src/allmydata/test/check_speed.py b/src/allmydata/test/check_speed.py index 7d4449a8..a006f4b8 100644 --- a/src/allmydata/test/check_speed.py +++ b/src/allmydata/test/check_speed.py @@ -63,6 +63,11 @@ class SpeedTest: self.upload_times[key], self.download_times[key] = times def one_test(self, res, name, count, size, mutable): + # values for 'mutable': + # False (upload a different CHK file for each 'count') + # "create" (upload different contents into a new SSK file) + # "upload" (upload different contents into the same SSK file. The + # time consumed does not include the creation of the file) d = self.client_rref.callRemote("speed_test", count, size, mutable) d.addCallback(self.record_times, name) return d @@ -89,6 +94,8 @@ class SpeedTest: d = defer.succeed(None) d.addCallback(self.one_test, "startup", 1, 1000, False) #ignore this one d.addCallback(self.measure_rtt) + + # immutable files d.addCallback(self.one_test, "1x 200B", 1, 200, False) d.addCallback(self.one_test, "10x 200B", 10, 200, False) def _maybe_do_100x_200B(res): @@ -105,9 +112,19 @@ class SpeedTest: return return self.one_test(None, "100MB", 1, 100*MB, False) d.addCallback(_maybe_do_100MB) - d.addCallback(self.one_test, "1x 200B SSK", 1, 200, True) - d.addCallback(self.one_test, "10x 200B SSK", 10, 200, True) - d.addCallback(self.one_test, "1MB SSK", 1, 1*MB, True) + + # mutable file creation + d.addCallback(self.one_test, "10x 200B SSK creation", 10, 200, "create") + + # mutable file upload/download + d.addCallback(self.one_test, "10x 200B SSK", 10, 200, "upload") + def _maybe_do_100x_200B_SSK(res): + if self.upload_times["10x 200B SSK"] < 5: + print "10x 200B SSK test went too fast, doing 100x 200B SSK" + return self.one_test(None, "100x 200B SSK", 100, 200, "upload") + return + d.addCallback(_maybe_do_100x_200B_SSK) + d.addCallback(self.one_test, "1MB SSK", 1, 1*MB, "upload") d.addCallback(self.calculate_speeds) return d @@ -147,6 +164,9 @@ class SpeedTest: A3 = 100*MB / (self.download_times["100MB"] - B) print "download speed (100MB):", self.number(A3, "Bps") + # SSK creation + B = self.upload_times["10x 200B SSK creation"] / 10 + print "create per-file time SSK: %.3fs" % B # upload SSK if "100x 200B SSK" in self.upload_times: