From 2c96a32633d50e64ea84f111e00b41f8e6303ad1 Mon Sep 17 00:00:00 2001
From: Brian Warner <warner@allmydata.com>
Date: Thu, 27 Mar 2008 16:46:08 -0700
Subject: [PATCH] helper: add more stats to webapi, at /helper_status
---
misc/munin/tahoe-helperstats.py | 25 ++++++++++++++++
src/allmydata/offloaded.py | 14 +++++----
src/allmydata/web/helper.xhtml | 32 ++++++++++++++++++++
src/allmydata/web/status.py | 52 +++++++++++++++++++++++++++++++--
src/allmydata/webish.py | 1 +
5 files changed, 117 insertions(+), 7 deletions(-)
create mode 100644 misc/munin/tahoe-helperstats.py
create mode 100644 src/allmydata/web/helper.xhtml
diff --git a/misc/munin/tahoe-helperstats.py b/misc/munin/tahoe-helperstats.py
new file mode 100644
index 00000000..d34aaab0
--- /dev/null
+++ b/misc/munin/tahoe-helperstats.py
@@ -0,0 +1,25 @@
+#! /usr/bin/python
+
+import os, sys
+import urllib
+import simplejson
+
+configinfo = """\
+graph_title Tahoe Helper Stats - Bytes Fetched
+graph_vlabel bytes
+graph_info This graph shows the amount of data being fetched by the helper
+fetched.label Bytes Fetched
+fetched.type GAUGE
+fetched.draw LINE1
+fetched.min 0
+"""
+
+if len(sys.argv) > 1:
+ if sys.argv[1] == "config":
+ print configinfo.rstrip()
+ sys.exit(0)
+
+url = os.environ["url"]
+
+data = simplejson.loads(urllib.urlopen(url).read())
+print "fetched.value %d" % data["CHK_fetched_bytes"]
diff --git a/src/allmydata/offloaded.py b/src/allmydata/offloaded.py
index a7bde89e..d3766f49 100644
--- a/src/allmydata/offloaded.py
+++ b/src/allmydata/offloaded.py
@@ -205,12 +205,12 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader):
self._reader.close()
os.unlink(self._encoding_file)
self._finished_observers.fire(r)
- self._helper.upload_finished(self._storage_index)
+ self._helper.upload_finished(self._storage_index, size)
del self._reader
def _failed(self, f):
self._finished_observers.fire(f)
- self._helper.upload_finished(self._storage_index)
+ self._helper.upload_finished(self._storage_index, 0)
del self._reader
class AskUntilSuccessMixin:
@@ -377,6 +377,7 @@ class CHKCiphertextFetcher(AskUntilSuccessMixin):
self._f.write(data)
self._have += len(data)
self._ciphertext_fetched += len(data)
+ self._upload_helper._helper._stats["CHK_fetched_bytes"] += len(data)
return False # not done
d.addCallback(_got_data)
return d
@@ -476,6 +477,8 @@ class Helper(Referenceable, service.MultiService):
self._stats = {"CHK_upload_requests": 0,
"CHK_upload_already_present": 0,
"CHK_upload_need_upload": 0,
+ "CHK_fetched_bytes": 0,
+ "CHK_encoded_bytes": 0,
}
service.MultiService.__init__(self)
@@ -491,11 +494,11 @@ class Helper(Referenceable, service.MultiService):
for fn in os.listdir(self._chk_incoming):
size = os.stat(os.path.join(self._chk_incoming, fn))[stat.ST_SIZE]
chk_incoming_files += 1
- chk_incoming_size += 1
+ chk_incoming_size += size
for fn in os.listdir(self._chk_encoding):
size = os.stat(os.path.join(self._chk_encoding, fn))[stat.ST_SIZE]
chk_encoding_files += 1
- chk_encoding_size += 1
+ chk_encoding_size += size
stats = {"CHK_active_uploads": len(self._active_uploads),
"CHK_incoming_files": chk_incoming_files,
"CHK_incoming_size": chk_incoming_size,
@@ -583,5 +586,6 @@ class Helper(Referenceable, service.MultiService):
d.addCallback(_checked)
return d
- def upload_finished(self, storage_index):
+ def upload_finished(self, storage_index, size):
+ self._stats["CHK_encoded_bytes"] += size
del self._active_uploads[storage_index]
diff --git a/src/allmydata/web/helper.xhtml b/src/allmydata/web/helper.xhtml
new file mode 100644
index 00000000..bea58017
--- /dev/null
+++ b/src/allmydata/web/helper.xhtml
@@ -0,0 +1,32 @@
+<html xmlns:n="http://nevow.com/ns/nevow/0.1">
+ <head>
+ <title>Helper Status - AllMyData Tahoe</title>
+ <!-- <link href="http://www.allmydata.com/common/css/styles.css"
+ rel="stylesheet" type="text/css"/> -->
+ <link href="/webform_css" rel="stylesheet" type="text/css"/>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+ </head>
+ <body>
+
+<h1>Helper Status</h1>
+
+<h2>Immutable Uploads</h2>
+<ul>
+ <li>Active: <span n:render="active_uploads" /></li>
+ <li>--</li>
+ <li>Bytes Fetched: <span n:render="upload_bytes_fetched" /></li>
+ <li>Incoming: <span n:render="incoming" /></li>
+ <li>Encoding: <span n:render="encoding" /></li>
+ <li>Bytes Encoded: <span n:render="upload_bytes_encoded" /></li>
+ <li>--</li>
+ <li>Total Requests: <span n:render="upload_requests" /></li>
+ <ul>
+ <li>Already Present: <span n:render="upload_already_present" /></li>
+ <li>Need Upload: <span n:render="upload_need_upload" /></li>
+ </ul>
+</ul>
+
+<div>Return to the <a href="/">Welcome Page</a></div>
+
+ </body>
+</html>
diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py
index 64e69fa5..17810d9d 100644
--- a/src/allmydata/web/status.py
+++ b/src/allmydata/web/status.py
@@ -1,10 +1,11 @@
import time
+import simplejson
from twisted.internet import defer
-from nevow import rend, tags as T
+from nevow import rend, inevow, tags as T
from allmydata.util import base32, idlib
from allmydata.web.common import IClient, getxmlfile, abbreviate_time, \
- abbreviate_rate
+ abbreviate_rate, get_arg
from allmydata.interfaces import IUploadStatus, IDownloadStatus, \
IPublishStatus, IRetrieveStatus
@@ -760,3 +761,50 @@ class Status(rend.Page):
return RetrieveStatusPage(s)
+class HelperStatus(rend.Page):
+ docFactory = getxmlfile("helper.xhtml")
+
+ def renderHTTP(self, ctx):
+ t = get_arg(inevow.IRequest(ctx), "t")
+ if t == "json":
+ return self.render_JSON(ctx)
+ # is there a better way to provide 'data' to all rendering methods?
+ helper = IClient(ctx).getServiceNamed("helper")
+ self.original = helper.get_stats()["helper"]
+ return rend.Page.renderHTTP(self, ctx)
+
+ def render_JSON(self, ctx):
+ try:
+ h = IClient(ctx).getServiceNamed("helper")
+ except KeyError:
+ return simplejson.dumps({})
+
+ stats = h.get_stats()["helper"]
+ return simplejson.dumps(stats, indent=1)
+
+ def render_active_uploads(self, ctx, data):
+ return data["CHK_active_uploads"]
+
+ def render_incoming(self, ctx, data):
+ return "%d bytes in %d files" % (data["CHK_incoming_size"],
+ data["CHK_incoming_files"])
+
+ def render_encoding(self, ctx, data):
+ return "%d bytes in %d files" % (data["CHK_encoding_size"],
+ data["CHK_encoding_files"])
+
+ def render_upload_requests(self, ctx, data):
+ return str(data["CHK_upload_requests"])
+
+ def render_upload_already_present(self, ctx, data):
+ return str(data["CHK_upload_already_present"])
+
+ def render_upload_need_upload(self, ctx, data):
+ return str(data["CHK_upload_need_upload"])
+
+ def render_upload_bytes_fetched(self, ctx, data):
+ return str(data["CHK_fetched_bytes"])
+
+ def render_upload_bytes_encoded(self, ctx, data):
+ return str(data["CHK_encoded_bytes"])
+
diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py
index 7748a749..5afc3fb9 100644
--- a/src/allmydata/webish.py
+++ b/src/allmydata/webish.py
@@ -1445,6 +1445,7 @@ class Root(rend.Page):
child_provisioning = provisioning.ProvisioningTool()
child_status = status.Status()
+ child_helper_status = status.HelperStatus()
def data_version(self, ctx, data):
return get_package_versions_string()
--
2.45.2