From 5446ea5d67d7246ed5cf27283825d1fb298e2e8a Mon Sep 17 00:00:00 2001 From: robk-tahoe Date: Tue, 25 Mar 2008 18:19:08 -0700 Subject: [PATCH] stats: added stats reporting to the upload helper adds a stats_producer for the upload helper, which provides a series of counters to the stats gatherer, under the name 'chk_upload_helper'. it examines both the 'incoming' directory, and the 'encoding' dir, providing inc_count inc_size inc_size_old enc_count enc_size enc_size_old, respectively the number of files in each dir, the total size thereof, and the aggregate size of all files older than 48hrs --- src/allmydata/client.py | 2 +- src/allmydata/offloaded.py | 41 ++++++++++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 6cc0c76f..90abef3c 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -188,7 +188,7 @@ class Client(node.Node, testutil.PollMixin): def init_helper(self): d = self.when_tub_ready() def _publish(self): - h = Helper(os.path.join(self.basedir, "helper")) + h = Helper(os.path.join(self.basedir, "helper"), self.stats_provider) h.setServiceParent(self) # TODO: this is confusing. BASEDIR/private/helper.furl is created # by the helper. BASEDIR/helper.furl is consumed by the client diff --git a/src/allmydata/offloaded.py b/src/allmydata/offloaded.py index d3766f49..00b2ee9c 100644 --- a/src/allmydata/offloaded.py +++ b/src/allmydata/offloaded.py @@ -1,5 +1,5 @@ -import os.path, stat, time +import os, stat, time from zope.interface import implements from twisted.application import service from twisted.internet import defer @@ -467,19 +467,13 @@ class Helper(Referenceable, service.MultiService): name = "helper" chk_upload_helper_class = CHKUploadHelper - def __init__(self, basedir): + def __init__(self, basedir, stats_provider=None): self._basedir = basedir self._chk_incoming = os.path.join(basedir, "CHK_incoming") self._chk_encoding = os.path.join(basedir, "CHK_encoding") fileutil.make_dirs(self._chk_incoming) fileutil.make_dirs(self._chk_encoding) self._active_uploads = {} - self._stats = {"CHK_upload_requests": 0, - "CHK_upload_already_present": 0, - "CHK_upload_need_upload": 0, - "CHK_fetched_bytes": 0, - "CHK_encoded_bytes": 0, - } service.MultiService.__init__(self) def setServiceParent(self, parent): @@ -513,6 +507,37 @@ class Helper(Referenceable, service.MultiService): kwargs['facility'] = "tahoe.helper" return self.parent.log(*args, **kwargs) + def get_stats(self): + OLD = 86400*2 # 48hours + now = time.time() + inc_count = inc_size = inc_size_old = 0 + enc_count = enc_size = enc_size_old = 0 + inc = os.listdir(self._chk_incoming) + enc = os.listdir(self._chk_encoding) + for f in inc: + s = os.stat(os.path.join(self._chk_incoming, f)) + size = s[stat.ST_SIZE] + mtime = s[stat.ST_MTIME] + inc_count += 1 + inc_size += size + if now - mtime > OLD: + inc_size_old += size + for f in enc: + s = os.stat(os.path.join(self._chk_encoding, f)) + size = s[stat.ST_SIZE] + mtime = s[stat.ST_MTIME] + enc_count += 1 + enc_size += size + if now - mtime > OLD: + enc_size_old += size + return { 'chk_upload_helper.inc_count': inc_count, + 'chk_upload_helper.inc_size': inc_size, + 'chk_upload_helper.inc_size_old': inc_size_old, + 'chk_upload_helper.enc_count': enc_count, + 'chk_upload_helper.enc_size': enc_size, + 'chk_upload_helper.enc_size_old': enc_size_old, + } + def remote_upload_chk(self, storage_index): self._stats["CHK_upload_requests"] += 1 r = upload.UploadResults() -- 2.45.2