From ef0f523fe0477d7ddb7ca2e7b488084a84eb3423 Mon Sep 17 00:00:00 2001 From: robk-tahoe Date: Wed, 9 Apr 2008 18:08:59 -0700 Subject: [PATCH] stats gathering: added counters to upload,download,mutablewatcher counting number of operations, and for immutable files, bytes transferred --- src/allmydata/client.py | 6 +++--- src/allmydata/download.py | 8 +++++++- src/allmydata/mutable.py | 9 ++++++++- src/allmydata/upload.py | 8 +++++++- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 9a9f0960..6cc0c76f 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -159,10 +159,10 @@ class Client(node.Node, testutil.PollMixin): helper_furl = self.get_config("helper.furl") convergence_s = self.get_or_create_private_config('convergence', _make_secret) self.convergence = base32.a2b(convergence_s) - self.add_service(Uploader(helper_furl)) - self.add_service(Downloader()) + self.add_service(Uploader(helper_furl, self.stats_provider)) + self.add_service(Downloader(self.stats_provider)) self.add_service(Checker()) - self.add_service(MutableWatcher()) + self.add_service(MutableWatcher(self.stats_provider)) def _publish(res): # we publish an empty object so that the introducer can count how # many clients are connected and see what versions they're diff --git a/src/allmydata/download.py b/src/allmydata/download.py index 2485ec10..853e946a 100644 --- a/src/allmydata/download.py +++ b/src/allmydata/download.py @@ -1038,9 +1038,10 @@ class Downloader(service.MultiService): name = "downloader" MAX_DOWNLOAD_STATUSES = 10 - def __init__(self): + def __init__(self, stats_provider=None): service.MultiService.__init__(self) self._all_downloads = weakref.WeakKeyDictionary() + self.stats_provider = stats_provider self._recent_download_status = [] def download(self, u, t): @@ -1050,6 +1051,11 @@ class Downloader(service.MultiService): t = IDownloadTarget(t) assert t.write assert t.close + + if self.stats_provider: + self.stats_provider.count('downloader.files_downloaded', 1) + self.stats_provider.count('downloader.bytes_downloaded', u.get_size()) + if isinstance(u, uri.LiteralFileURI): dl = LiteralDownloader(self.parent, u, t) elif isinstance(u, uri.CHKFileURI): diff --git a/src/allmydata/mutable.py b/src/allmydata/mutable.py index 3555eb5d..eb60ff54 100644 --- a/src/allmydata/mutable.py +++ b/src/allmydata/mutable.py @@ -1830,8 +1830,9 @@ class MutableWatcher(service.MultiService): MAX_RETRIEVE_STATUSES = 20 name = "mutable-watcher" - def __init__(self): + def __init__(self, stats_provider=None): service.MultiService.__init__(self) + self.stats_provider = stats_provider self._all_publish = weakref.WeakKeyDictionary() self._recent_publish_status = [] self._all_retrieve = weakref.WeakKeyDictionary() @@ -1840,6 +1841,9 @@ class MutableWatcher(service.MultiService): def notify_publish(self, p): self._all_publish[p] = None self._recent_publish_status.append(p.get_status()) + if self.stats_provider: + self.stats_provider.count('mutable.files_published', 1) + #self.stats_provider.count('mutable.bytes_published', p._node.get_size()) while len(self._recent_publish_status) > self.MAX_PUBLISH_STATUSES: self._recent_publish_status.pop(0) @@ -1855,6 +1859,9 @@ class MutableWatcher(service.MultiService): def notify_retrieve(self, r): self._all_retrieve[r] = None self._recent_retrieve_status.append(r.get_status()) + if self.stats_provider: + self.stats_provider.count('mutable.files_retrieved', 1) + #self.stats_provider.count('mutable.bytes_retrieved', r._node.get_size()) while len(self._recent_retrieve_status) > self.MAX_RETRIEVE_STATUSES: self._recent_retrieve_status.pop(0) diff --git a/src/allmydata/upload.py b/src/allmydata/upload.py index eed5ee70..3e71ff89 100644 --- a/src/allmydata/upload.py +++ b/src/allmydata/upload.py @@ -1194,8 +1194,9 @@ class Uploader(service.MultiService): URI_LIT_SIZE_THRESHOLD = 55 MAX_UPLOAD_STATUSES = 10 - def __init__(self, helper_furl=None): + def __init__(self, helper_furl=None, stats_provider=None): self._helper_furl = helper_furl + self.stats_provider = stats_provider self._helper = None self._all_uploads = weakref.WeakKeyDictionary() self._recent_upload_status = [] @@ -1229,6 +1230,11 @@ class Uploader(service.MultiService): precondition(isinstance(default_params, dict), default_params) precondition("max_segment_size" in default_params, default_params) uploadable.set_default_encoding_parameters(default_params) + + if self.stats_provider: + self.stats_provider.count('uploader.files_uploaded', 1) + self.stats_provider.count('uploader.bytes_uploaded', size) + if size <= self.URI_LIT_SIZE_THRESHOLD: uploader = LiteralUploader(self.parent) elif self._helper: -- 2.45.2