From a2dd4d14a68af6ddf93ce487c691e8e89adcad61 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 14 Apr 2008 18:36:27 -0700 Subject: [PATCH] helper status: include percentage fetched+pushed, add helper-uploads to the upload/download list --- src/allmydata/client.py | 8 ++++++++ src/allmydata/offloaded.py | 30 +++++++++++++++++++++++++++--- src/allmydata/test/test_web.py | 5 ++++- src/allmydata/web/status.py | 26 ++++++++++++++++++-------- 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 90abef3c..bcab4602 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -369,3 +369,11 @@ class Client(node.Node, testutil.PollMixin): def list_recent_retrieve(self): watcher = self.getServiceNamed("mutable-watcher") return watcher.list_recent_retrieve() + + def list_active_helper_statuses(self): + helper = self.getServiceNamed("helper") + return helper.get_active_upload_statuses() + def list_recent_helper_statuses(self): + helper = self.getServiceNamed("helper") + return helper.get_recent_upload_statuses() + diff --git a/src/allmydata/offloaded.py b/src/allmydata/offloaded.py index 03dcfeea..81c61156 100644 --- a/src/allmydata/offloaded.py +++ b/src/allmydata/offloaded.py @@ -143,6 +143,9 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): self._results = results self._upload_status = upload.UploadStatus() self._upload_status.set_helper(False) + self._upload_status.set_storage_index(storage_index) + self._upload_status.set_status("fetching ciphertext") + self._upload_status.set_progress(0, 1.0) self._helper.log("CHKUploadHelper starting for SI %s" % upload_id, parent=log_number) @@ -209,6 +212,10 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): del self._reader def _failed(self, f): + self.log(format="CHKUploadHelper(%(si)s) failed", + si=storage.si_b2a(self._storage_index)[:5], + failure=f, + level=log.UNUSUAL) self._finished_observers.fire(f) self._helper.upload_finished(self._storage_index, 0) del self._reader @@ -306,6 +313,7 @@ class CHKCiphertextFetcher(AskUntilSuccessMixin): def _got_size(self, size): self.log("total size is %d bytes" % size, level=log.NOISY) + self._upload_helper._upload_status.set_size(size) self._expected_size = size def _start_reading(self, res): @@ -365,16 +373,16 @@ class CHKCiphertextFetcher(AskUntilSuccessMixin): needed = self._expected_size - self._have fetch_size = min(needed, self.CHUNK_SIZE) if fetch_size == 0: + self._upload_helper._upload_status.set_progress(1, 1.0) return True # all done - percent = 0 + percent = 0.0 if self._expected_size: percent = 1.0 * (self._have+fetch_size) / self._expected_size - percent = int(100*percent) self.log(format="fetching %(start)d-%(end)d of %(total)d (%(percent)d%%)", start=self._have, end=self._have+fetch_size, total=self._expected_size, - percent=percent, + percent=int(100.0*percent), level=log.NOISY) d = self.call("read_encrypted", self._have, fetch_size) def _got_data(ciphertext_v): @@ -383,6 +391,7 @@ class CHKCiphertextFetcher(AskUntilSuccessMixin): self._have += len(data) self._ciphertext_fetched += len(data) self._upload_helper._helper.count("chk_upload_helper.fetched_bytes", len(data)) + self._upload_helper._upload_status.set_progress(1, percent) return False # not done d.addCallback(_got_data) return d @@ -432,6 +441,7 @@ class LocalCiphertextReader(AskUntilSuccessMixin): self._status = interfaces.IUploadStatus(upload_status) def start(self): + self._upload_helper._upload_status.set_status("pushing") self._size = os.stat(self._encoding_file)[stat.ST_SIZE] self.f = open(self._encoding_file, "rb") @@ -471,6 +481,7 @@ class Helper(Referenceable, service.MultiService): name = "helper" chk_upload_helper_class = CHKUploadHelper + MAX_UPLOAD_STATUSES = 10 def __init__(self, basedir, stats_provider=None): self._basedir = basedir @@ -479,6 +490,7 @@ class Helper(Referenceable, service.MultiService): fileutil.make_dirs(self._chk_incoming) fileutil.make_dirs(self._chk_encoding) self._active_uploads = {} + self._recent_upload_statuses = [] self.stats_provider = stats_provider if stats_provider: stats_provider.register_producer(self) @@ -612,4 +624,16 @@ class Helper(Referenceable, service.MultiService): def upload_finished(self, storage_index, size): self.count("chk_upload_helper.encoded_bytes", size) + uh = self._active_uploads[storage_index] del self._active_uploads[storage_index] + s = uh.get_upload_status() + s.set_active(False) + self._recent_upload_statuses.append(s) + while len(self._recent_upload_statuses) > self.MAX_UPLOAD_STATUSES: + self._recent_upload_statuses.pop(0) + + def get_active_upload_statuses(self): + return [u.get_upload_status() for u in self._active_uploads.values()] + + def get_recent_upload_statuses(self): + return self._recent_upload_statuses diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index c53b49ae..73a149a9 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -94,7 +94,10 @@ class FakeClient(service.MultiService): return [] def list_recent_retrieve(self): return [] - + def list_active_helper_statuses(self): + return [] + def list_recent_helper_statuses(self): + return [] class WebMixin(object): def setUp(self): diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 1b07045d..917c265f 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -659,17 +659,21 @@ class Status(rend.Page): addSlash = True def data_active_operations(self, ctx, data): - active = (IClient(ctx).list_active_uploads() + - IClient(ctx).list_active_downloads() + - IClient(ctx).list_active_publish() + - IClient(ctx).list_active_retrieve()) + client = IClient(ctx) + active = (client.list_active_uploads() + + client.list_active_downloads() + + client.list_active_publish() + + client.list_active_retrieve() + + client.list_active_helper_statuses()) return active def data_recent_operations(self, ctx, data): - recent = [o for o in (IClient(ctx).list_recent_uploads() + - IClient(ctx).list_recent_downloads() + - IClient(ctx).list_recent_publish() + - IClient(ctx).list_recent_retrieve()) + client = IClient(ctx) + recent = [o for o in (client.list_recent_uploads() + + client.list_recent_downloads() + + client.list_recent_publish() + + client.list_recent_retrieve() + + client.list_recent_helper_statuses()) if not o.get_active()] recent.sort(lambda a,b: cmp(a.get_started(), b.get_started())) recent.reverse() @@ -735,6 +739,12 @@ class Status(rend.Page): s = u.get_upload_status() if s.get_counter() == count: return UploadStatusPage(s) + for s in (client.list_active_helper_statuses() + + client.list_recent_helper_statuses()): + if s.get_counter() == count: + # immutable-upload helpers use the same status object as + # a regular immutable-upload + return UploadStatusPage(s) if stype == "down": for s in client.list_recent_downloads(): if s.get_counter() == count: -- 2.37.2