From 2c96a32633d50e64ea84f111e00b41f8e6303ad1 Mon Sep 17 00:00:00 2001
From: Brian Warner <warner@allmydata.com>
Date: Thu, 27 Mar 2008 16:46:08 -0700
Subject: [PATCH] helper: add more stats to webapi, at /helper_status

---
 misc/munin/tahoe-helperstats.py | 25 ++++++++++++++++
 src/allmydata/offloaded.py      | 14 +++++----
 src/allmydata/web/helper.xhtml  | 32 ++++++++++++++++++++
 src/allmydata/web/status.py     | 52 +++++++++++++++++++++++++++++++--
 src/allmydata/webish.py         |  1 +
 5 files changed, 117 insertions(+), 7 deletions(-)
 create mode 100644 misc/munin/tahoe-helperstats.py
 create mode 100644 src/allmydata/web/helper.xhtml

diff --git a/misc/munin/tahoe-helperstats.py b/misc/munin/tahoe-helperstats.py
new file mode 100644
index 00000000..d34aaab0
--- /dev/null
+++ b/misc/munin/tahoe-helperstats.py
@@ -0,0 +1,25 @@
+#! /usr/bin/python
+
+import os, sys
+import urllib
+import simplejson
+
+configinfo = """\
+graph_title Tahoe Helper Stats - Bytes Fetched
+graph_vlabel bytes
+graph_info This graph shows the amount of data being fetched by the helper
+fetched.label Bytes Fetched
+fetched.type GAUGE
+fetched.draw LINE1
+fetched.min 0
+"""
+
+if len(sys.argv) > 1:
+    if sys.argv[1] == "config":
+        print configinfo.rstrip()
+        sys.exit(0)
+
+url = os.environ["url"]
+
+data = simplejson.loads(urllib.urlopen(url).read())
+print "fetched.value %d" % data["CHK_fetched_bytes"]
diff --git a/src/allmydata/offloaded.py b/src/allmydata/offloaded.py
index a7bde89e..d3766f49 100644
--- a/src/allmydata/offloaded.py
+++ b/src/allmydata/offloaded.py
@@ -205,12 +205,12 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader):
         self._reader.close()
         os.unlink(self._encoding_file)
         self._finished_observers.fire(r)
-        self._helper.upload_finished(self._storage_index)
+        self._helper.upload_finished(self._storage_index, size)
         del self._reader
 
     def _failed(self, f):
         self._finished_observers.fire(f)
-        self._helper.upload_finished(self._storage_index)
+        self._helper.upload_finished(self._storage_index, 0)
         del self._reader
 
 class AskUntilSuccessMixin:
@@ -377,6 +377,7 @@ class CHKCiphertextFetcher(AskUntilSuccessMixin):
                 self._f.write(data)
                 self._have += len(data)
                 self._ciphertext_fetched += len(data)
+                self._upload_helper._helper._stats["CHK_fetched_bytes"] += len(data)
             return False # not done
         d.addCallback(_got_data)
         return d
@@ -476,6 +477,8 @@ class Helper(Referenceable, service.MultiService):
         self._stats = {"CHK_upload_requests": 0,
                        "CHK_upload_already_present": 0,
                        "CHK_upload_need_upload": 0,
+                       "CHK_fetched_bytes": 0,
+                       "CHK_encoded_bytes": 0,
                        }
         service.MultiService.__init__(self)
 
@@ -491,11 +494,11 @@ class Helper(Referenceable, service.MultiService):
         for fn in os.listdir(self._chk_incoming):
             size = os.stat(os.path.join(self._chk_incoming, fn))[stat.ST_SIZE]
             chk_incoming_files += 1
-            chk_incoming_size += 1
+            chk_incoming_size += size
         for fn in os.listdir(self._chk_encoding):
             size = os.stat(os.path.join(self._chk_encoding, fn))[stat.ST_SIZE]
             chk_encoding_files += 1
-            chk_encoding_size += 1
+            chk_encoding_size += size
         stats = {"CHK_active_uploads": len(self._active_uploads),
                  "CHK_incoming_files": chk_incoming_files,
                  "CHK_incoming_size": chk_incoming_size,
@@ -583,5 +586,6 @@ class Helper(Referenceable, service.MultiService):
         d.addCallback(_checked)
         return d
 
-    def upload_finished(self, storage_index):
+    def upload_finished(self, storage_index, size):
+        self._stats["CHK_encoded_bytes"] += size
         del self._active_uploads[storage_index]
diff --git a/src/allmydata/web/helper.xhtml b/src/allmydata/web/helper.xhtml
new file mode 100644
index 00000000..bea58017
--- /dev/null
+++ b/src/allmydata/web/helper.xhtml
@@ -0,0 +1,32 @@
+<html xmlns:n="http://nevow.com/ns/nevow/0.1">
+  <head>
+    <title>Helper Status - AllMyData Tahoe</title>
+    <!-- <link href="http://www.allmydata.com/common/css/styles.css"
+          rel="stylesheet" type="text/css"/> -->
+    <link href="/webform_css" rel="stylesheet" type="text/css"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  </head>
+  <body>
+
+<h1>Helper Status</h1>
+
+<h2>Immutable Uploads</h2>
+<ul>
+  <li>Active: <span n:render="active_uploads" /></li>
+  <li>--</li>
+  <li>Bytes Fetched: <span n:render="upload_bytes_fetched" /></li>
+  <li>Incoming: <span n:render="incoming" /></li>
+  <li>Encoding: <span n:render="encoding" /></li>
+  <li>Bytes Encoded: <span n:render="upload_bytes_encoded" /></li>
+  <li>--</li>
+  <li>Total Requests: <span n:render="upload_requests" /></li>
+  <ul>
+    <li>Already Present: <span n:render="upload_already_present" /></li>
+    <li>Need Upload: <span n:render="upload_need_upload" /></li>
+  </ul>
+</ul>
+
+<div>Return to the <a href="/">Welcome Page</a></div>
+
+  </body>
+</html>
diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py
index 64e69fa5..17810d9d 100644
--- a/src/allmydata/web/status.py
+++ b/src/allmydata/web/status.py
@@ -1,10 +1,11 @@
 
 import time
+import simplejson
 from twisted.internet import defer
-from nevow import rend, tags as T
+from nevow import rend, inevow, tags as T
 from allmydata.util import base32, idlib
 from allmydata.web.common import IClient, getxmlfile, abbreviate_time, \
-     abbreviate_rate
+     abbreviate_rate, get_arg
 from allmydata.interfaces import IUploadStatus, IDownloadStatus, \
      IPublishStatus, IRetrieveStatus
 
@@ -760,3 +761,50 @@ class Status(rend.Page):
                     return RetrieveStatusPage(s)
 
 
+class HelperStatus(rend.Page):
+    docFactory = getxmlfile("helper.xhtml")
+
+    def renderHTTP(self, ctx):
+        t = get_arg(inevow.IRequest(ctx), "t")
+        if t == "json":
+            return self.render_JSON(ctx)
+        # is there a better way to provide 'data' to all rendering methods?
+        helper = IClient(ctx).getServiceNamed("helper")
+        self.original = helper.get_stats()["helper"]
+        return rend.Page.renderHTTP(self, ctx)
+
+    def render_JSON(self, ctx):
+        try:
+            h = IClient(ctx).getServiceNamed("helper")
+        except KeyError:
+            return simplejson.dumps({})
+
+        stats = h.get_stats()["helper"]
+        return simplejson.dumps(stats, indent=1)
+
+    def render_active_uploads(self, ctx, data):
+        return data["CHK_active_uploads"]
+
+    def render_incoming(self, ctx, data):
+        return "%d bytes in %d files" % (data["CHK_incoming_size"],
+                                         data["CHK_incoming_files"])
+
+    def render_encoding(self, ctx, data):
+        return "%d bytes in %d files" % (data["CHK_encoding_size"],
+                                         data["CHK_encoding_files"])
+
+    def render_upload_requests(self, ctx, data):
+        return str(data["CHK_upload_requests"])
+
+    def render_upload_already_present(self, ctx, data):
+        return str(data["CHK_upload_already_present"])
+
+    def render_upload_need_upload(self, ctx, data):
+        return str(data["CHK_upload_need_upload"])
+
+    def render_upload_bytes_fetched(self, ctx, data):
+        return str(data["CHK_fetched_bytes"])
+
+    def render_upload_bytes_encoded(self, ctx, data):
+        return str(data["CHK_encoded_bytes"])
+
diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py
index 7748a749..5afc3fb9 100644
--- a/src/allmydata/webish.py
+++ b/src/allmydata/webish.py
@@ -1445,6 +1445,7 @@ class Root(rend.Page):
 
     child_provisioning = provisioning.ProvisioningTool()
     child_status = status.Status()
+    child_helper_status = status.HelperStatus()
 
     def data_version(self, ctx, data):
         return get_package_versions_string()
-- 
2.45.2