From 2c96a32633d50e64ea84f111e00b41f8e6303ad1 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Thu, 27 Mar 2008 16:46:08 -0700 Subject: [PATCH] helper: add more stats to webapi, at /helper_status --- misc/munin/tahoe-helperstats.py | 25 ++++++++++++++++ src/allmydata/offloaded.py | 14 +++++---- src/allmydata/web/helper.xhtml | 32 ++++++++++++++++++++ src/allmydata/web/status.py | 52 +++++++++++++++++++++++++++++++-- src/allmydata/webish.py | 1 + 5 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 misc/munin/tahoe-helperstats.py create mode 100644 src/allmydata/web/helper.xhtml diff --git a/misc/munin/tahoe-helperstats.py b/misc/munin/tahoe-helperstats.py new file mode 100644 index 00000000..d34aaab0 --- /dev/null +++ b/misc/munin/tahoe-helperstats.py @@ -0,0 +1,25 @@ +#! /usr/bin/python + +import os, sys +import urllib +import simplejson + +configinfo = """\ +graph_title Tahoe Helper Stats - Bytes Fetched +graph_vlabel bytes +graph_info This graph shows the amount of data being fetched by the helper +fetched.label Bytes Fetched +fetched.type GAUGE +fetched.draw LINE1 +fetched.min 0 +""" + +if len(sys.argv) > 1: + if sys.argv[1] == "config": + print configinfo.rstrip() + sys.exit(0) + +url = os.environ["url"] + +data = simplejson.loads(urllib.urlopen(url).read()) +print "fetched.value %d" % data["CHK_fetched_bytes"] diff --git a/src/allmydata/offloaded.py b/src/allmydata/offloaded.py index a7bde89e..d3766f49 100644 --- a/src/allmydata/offloaded.py +++ b/src/allmydata/offloaded.py @@ -205,12 +205,12 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): self._reader.close() os.unlink(self._encoding_file) self._finished_observers.fire(r) - self._helper.upload_finished(self._storage_index) + self._helper.upload_finished(self._storage_index, size) del self._reader def _failed(self, f): self._finished_observers.fire(f) - self._helper.upload_finished(self._storage_index) + self._helper.upload_finished(self._storage_index, 0) del self._reader class AskUntilSuccessMixin: @@ -377,6 +377,7 @@ class CHKCiphertextFetcher(AskUntilSuccessMixin): self._f.write(data) self._have += len(data) self._ciphertext_fetched += len(data) + self._upload_helper._helper._stats["CHK_fetched_bytes"] += len(data) return False # not done d.addCallback(_got_data) return d @@ -476,6 +477,8 @@ class Helper(Referenceable, service.MultiService): self._stats = {"CHK_upload_requests": 0, "CHK_upload_already_present": 0, "CHK_upload_need_upload": 0, + "CHK_fetched_bytes": 0, + "CHK_encoded_bytes": 0, } service.MultiService.__init__(self) @@ -491,11 +494,11 @@ class Helper(Referenceable, service.MultiService): for fn in os.listdir(self._chk_incoming): size = os.stat(os.path.join(self._chk_incoming, fn))[stat.ST_SIZE] chk_incoming_files += 1 - chk_incoming_size += 1 + chk_incoming_size += size for fn in os.listdir(self._chk_encoding): size = os.stat(os.path.join(self._chk_encoding, fn))[stat.ST_SIZE] chk_encoding_files += 1 - chk_encoding_size += 1 + chk_encoding_size += size stats = {"CHK_active_uploads": len(self._active_uploads), "CHK_incoming_files": chk_incoming_files, "CHK_incoming_size": chk_incoming_size, @@ -583,5 +586,6 @@ class Helper(Referenceable, service.MultiService): d.addCallback(_checked) return d - def upload_finished(self, storage_index): + def upload_finished(self, storage_index, size): + self._stats["CHK_encoded_bytes"] += size del self._active_uploads[storage_index] diff --git a/src/allmydata/web/helper.xhtml b/src/allmydata/web/helper.xhtml new file mode 100644 index 00000000..bea58017 --- /dev/null +++ b/src/allmydata/web/helper.xhtml @@ -0,0 +1,32 @@ + + + Helper Status - AllMyData Tahoe + + + + + + +

Helper Status

+ +

Immutable Uploads

+ + +
Return to the Welcome Page
+ + + diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 64e69fa5..17810d9d 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1,10 +1,11 @@ import time +import simplejson from twisted.internet import defer -from nevow import rend, tags as T +from nevow import rend, inevow, tags as T from allmydata.util import base32, idlib from allmydata.web.common import IClient, getxmlfile, abbreviate_time, \ - abbreviate_rate + abbreviate_rate, get_arg from allmydata.interfaces import IUploadStatus, IDownloadStatus, \ IPublishStatus, IRetrieveStatus @@ -760,3 +761,50 @@ class Status(rend.Page): return RetrieveStatusPage(s) +class HelperStatus(rend.Page): + docFactory = getxmlfile("helper.xhtml") + + def renderHTTP(self, ctx): + t = get_arg(inevow.IRequest(ctx), "t") + if t == "json": + return self.render_JSON(ctx) + # is there a better way to provide 'data' to all rendering methods? + helper = IClient(ctx).getServiceNamed("helper") + self.original = helper.get_stats()["helper"] + return rend.Page.renderHTTP(self, ctx) + + def render_JSON(self, ctx): + try: + h = IClient(ctx).getServiceNamed("helper") + except KeyError: + return simplejson.dumps({}) + + stats = h.get_stats()["helper"] + return simplejson.dumps(stats, indent=1) + + def render_active_uploads(self, ctx, data): + return data["CHK_active_uploads"] + + def render_incoming(self, ctx, data): + return "%d bytes in %d files" % (data["CHK_incoming_size"], + data["CHK_incoming_files"]) + + def render_encoding(self, ctx, data): + return "%d bytes in %d files" % (data["CHK_encoding_size"], + data["CHK_encoding_files"]) + + def render_upload_requests(self, ctx, data): + return str(data["CHK_upload_requests"]) + + def render_upload_already_present(self, ctx, data): + return str(data["CHK_upload_already_present"]) + + def render_upload_need_upload(self, ctx, data): + return str(data["CHK_upload_need_upload"]) + + def render_upload_bytes_fetched(self, ctx, data): + return str(data["CHK_fetched_bytes"]) + + def render_upload_bytes_encoded(self, ctx, data): + return str(data["CHK_encoded_bytes"]) + diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 7748a749..5afc3fb9 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -1445,6 +1445,7 @@ class Root(rend.Page): child_provisioning = provisioning.ProvisioningTool() child_status = status.Status() + child_helper_status = status.HelperStatus() def data_version(self, ctx, data): return get_package_versions_string() -- 2.45.2