From: Brian Warner Date: Wed, 5 Mar 2008 21:59:56 +0000 (-0700) Subject: webish: this file is too big, start breaking it into pieces, beginning with status X-Git-Tag: allmydata-tahoe-0.9.0~65 X-Git-Url: https://git.rkrishnan.org/pf/content/en/service/nxhtml.html?a=commitdiff_plain;h=4fa622f9b1b3282aaeea70e3d710efc828468c64;p=tahoe-lafs%2Ftahoe-lafs.git webish: this file is too big, start breaking it into pieces, beginning with status --- diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index b3b403d5..4b597d1a 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -6,6 +6,7 @@ from twisted.internet import defer from twisted.web import client, error, http from twisted.python import failure, log from allmydata import interfaces, provisioning, uri, webish, upload, download +from allmydata.web import status from allmydata.util import fileutil from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, FakeMutableFileNode, create_chk_filenode from allmydata.interfaces import IURI, INewDirectoryURI, IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode @@ -415,7 +416,7 @@ class Web(WebMixin, unittest.TestCase): return d def test_status_numbers(self): - drrm = webish.DownloadResultsRendererMixin() + drrm = status.DownloadResultsRendererMixin() self.failUnlessEqual(drrm.render_time(None, None), "") self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s") self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms") @@ -426,7 +427,7 @@ class Web(WebMixin, unittest.TestCase): self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps") self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps") - urrm = webish.UploadResultsRendererMixin() + urrm = status.UploadResultsRendererMixin() self.failUnlessEqual(urrm.render_time(None, None), "") self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s") self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms") diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py new file mode 100644 index 00000000..7a72d6ac --- /dev/null +++ b/src/allmydata/web/common.py @@ -0,0 +1,6 @@ + +from zope.interface import Interface + +class IClient(Interface): + pass + diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py new file mode 100644 index 00000000..f7070939 --- /dev/null +++ b/src/allmydata/web/status.py @@ -0,0 +1,656 @@ + +import time +from twisted.internet import defer +from nevow import rend, loaders, tags as T +from nevow.util import resource_filename +from allmydata.util import base32, idlib +from allmydata.web.common import IClient +from allmydata.interfaces import IUploadStatus, IDownloadStatus, \ + IPublishStatus, IRetrieveStatus + +def getxmlfile(name): + return loaders.xmlfile(resource_filename('allmydata.web', '%s' % name)) + +def plural(sequence_or_length): + if isinstance(sequence_or_length, int): + length = sequence_or_length + else: + length = len(sequence_or_length) + if length == 1: + return "" + return "s" + +class RateAndTimeMixin: + + def render_time(self, ctx, data): + # 1.23s, 790ms, 132us + if data is None: + return "" + s = float(data) + if s >= 1.0: + return "%.2fs" % s + if s >= 0.01: + return "%dms" % (1000*s) + if s >= 0.001: + return "%.1fms" % (1000*s) + return "%dus" % (1000000*s) + + def render_rate(self, ctx, data): + # 21.8kBps, 554.4kBps 4.37MBps + if data is None: + return "" + r = float(data) + if r > 1000000: + return "%1.2fMBps" % (r/1000000) + if r > 1000: + return "%.1fkBps" % (r/1000) + return "%dBps" % r + +class UploadResultsRendererMixin(RateAndTimeMixin): + # this requires a method named 'upload_results' + + def render_sharemap(self, ctx, data): + d = self.upload_results() + d.addCallback(lambda res: res.sharemap) + def _render(sharemap): + if sharemap is None: + return "None" + l = T.ul() + for shnum in sorted(sharemap.keys()): + l[T.li["%d -> %s" % (shnum, sharemap[shnum])]] + return l + d.addCallback(_render) + return d + + def render_servermap(self, ctx, data): + d = self.upload_results() + d.addCallback(lambda res: res.servermap) + def _render(servermap): + if servermap is None: + return "None" + l = T.ul() + for peerid in sorted(servermap.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + shares_s = ",".join(["#%d" % shnum + for shnum in servermap[peerid]]) + l[T.li["[%s] got share%s: %s" % (peerid_s, + plural(servermap[peerid]), + shares_s)]] + return l + d.addCallback(_render) + return d + + def data_file_size(self, ctx, data): + d = self.upload_results() + d.addCallback(lambda res: res.file_size) + return d + + def _get_time(self, name): + d = self.upload_results() + d.addCallback(lambda res: res.timings.get(name)) + return d + + def data_time_total(self, ctx, data): + return self._get_time("total") + + def data_time_storage_index(self, ctx, data): + return self._get_time("storage_index") + + def data_time_contacting_helper(self, ctx, data): + return self._get_time("contacting_helper") + + def data_time_existence_check(self, ctx, data): + return self._get_time("existence_check") + + def data_time_cumulative_fetch(self, ctx, data): + return self._get_time("cumulative_fetch") + + def data_time_helper_total(self, ctx, data): + return self._get_time("helper_total") + + def data_time_peer_selection(self, ctx, data): + return self._get_time("peer_selection") + + def data_time_total_encode_and_push(self, ctx, data): + return self._get_time("total_encode_and_push") + + def data_time_cumulative_encoding(self, ctx, data): + return self._get_time("cumulative_encoding") + + def data_time_cumulative_sending(self, ctx, data): + return self._get_time("cumulative_sending") + + def data_time_hashes_and_close(self, ctx, data): + return self._get_time("hashes_and_close") + + def _get_rate(self, name): + d = self.upload_results() + def _convert(r): + file_size = r.file_size + time = r.timings.get(name) + if time is None: + return None + try: + return 1.0 * file_size / time + except ZeroDivisionError: + return None + d.addCallback(_convert) + return d + + def data_rate_total(self, ctx, data): + return self._get_rate("total") + + def data_rate_storage_index(self, ctx, data): + return self._get_rate("storage_index") + + def data_rate_encode(self, ctx, data): + return self._get_rate("cumulative_encoding") + + def data_rate_push(self, ctx, data): + return self._get_rate("cumulative_sending") + + def data_rate_encode_and_push(self, ctx, data): + d = self.upload_results() + def _convert(r): + file_size = r.file_size + if file_size is None: + return None + time1 = r.timings.get("cumulative_encoding") + if time1 is None: + return None + time2 = r.timings.get("cumulative_sending") + if time2 is None: + return None + try: + return 1.0 * file_size / (time1+time2) + except ZeroDivisionError: + return None + d.addCallback(_convert) + return d + + def data_rate_ciphertext_fetch(self, ctx, data): + d = self.upload_results() + def _convert(r): + fetch_size = r.ciphertext_fetched + if fetch_size is None: + return None + time = r.timings.get("cumulative_fetch") + if time is None: + return None + try: + return 1.0 * fetch_size / time + except ZeroDivisionError: + return None + d.addCallback(_convert) + return d + +class UploadStatusPage(UploadResultsRendererMixin, rend.Page): + docFactory = getxmlfile("upload-status.xhtml") + + def __init__(self, data): + rend.Page.__init__(self, data) + self.upload_status = data + + def upload_results(self): + return defer.maybeDeferred(self.upload_status.get_results) + + def render_results(self, ctx, data): + d = self.upload_results() + def _got_results(results): + if results: + return ctx.tag + return "" + d.addCallback(_got_results) + return d + + def render_started(self, ctx, data): + TIME_FORMAT = "%H:%M:%S %d-%b-%Y" + started_s = time.strftime(TIME_FORMAT, + time.localtime(data.get_started())) + return started_s + + def render_si(self, ctx, data): + si_s = base32.b2a_or_none(data.get_storage_index()) + if si_s is None: + si_s = "(None)" + return si_s + + def render_helper(self, ctx, data): + return {True: "Yes", + False: "No"}[data.using_helper()] + + def render_total_size(self, ctx, data): + size = data.get_size() + if size is None: + size = "(unknown)" + return size + + def render_progress_hash(self, ctx, data): + progress = data.get_progress()[0] + # TODO: make an ascii-art bar + return "%.1f%%" % (100.0 * progress) + + def render_progress_ciphertext(self, ctx, data): + progress = data.get_progress()[1] + # TODO: make an ascii-art bar + return "%.1f%%" % (100.0 * progress) + + def render_progress_encode_push(self, ctx, data): + progress = data.get_progress()[2] + # TODO: make an ascii-art bar + return "%.1f%%" % (100.0 * progress) + + def render_status(self, ctx, data): + return data.get_status() + +class DownloadResultsRendererMixin(RateAndTimeMixin): + # this requires a method named 'download_results' + + def render_servermap(self, ctx, data): + d = self.download_results() + d.addCallback(lambda res: res.servermap) + def _render(servermap): + if servermap is None: + return "None" + l = T.ul() + for peerid in sorted(servermap.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + shares_s = ",".join(["#%d" % shnum + for shnum in servermap[peerid]]) + l[T.li["[%s] has share%s: %s" % (peerid_s, + plural(servermap[peerid]), + shares_s)]] + return l + d.addCallback(_render) + return d + + def render_servers_used(self, ctx, data): + d = self.download_results() + d.addCallback(lambda res: res.servers_used) + def _got(servers_used): + if not servers_used: + return "" + peerids_s = ", ".join(["[%s]" % idlib.shortnodeid_b2a(peerid) + for peerid in servers_used]) + return T.li["Servers Used: ", peerids_s] + d.addCallback(_got) + return d + + def render_problems(self, ctx, data): + d = self.download_results() + d.addCallback(lambda res: res.server_problems) + def _got(server_problems): + if not server_problems: + return "" + l = T.ul() + for peerid in sorted(server_problems.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + l[T.li["[%s]: %s" % (peerid_s, server_problems[peerid])]] + return T.li["Server Problems:", l] + d.addCallback(_got) + return d + + def data_file_size(self, ctx, data): + d = self.download_results() + d.addCallback(lambda res: res.file_size) + return d + + def _get_time(self, name): + d = self.download_results() + d.addCallback(lambda res: res.timings.get(name)) + return d + + def data_time_total(self, ctx, data): + return self._get_time("total") + + def data_time_peer_selection(self, ctx, data): + return self._get_time("peer_selection") + + def data_time_uri_extension(self, ctx, data): + return self._get_time("uri_extension") + + def data_time_hashtrees(self, ctx, data): + return self._get_time("hashtrees") + + def data_time_segments(self, ctx, data): + return self._get_time("segments") + + def data_time_cumulative_fetch(self, ctx, data): + return self._get_time("cumulative_fetch") + + def data_time_cumulative_decode(self, ctx, data): + return self._get_time("cumulative_decode") + + def data_time_cumulative_decrypt(self, ctx, data): + return self._get_time("cumulative_decrypt") + + def _get_rate(self, name): + d = self.download_results() + def _convert(r): + file_size = r.file_size + time = r.timings.get(name) + if time is None: + return None + try: + return 1.0 * file_size / time + except ZeroDivisionError: + return None + d.addCallback(_convert) + return d + + def data_rate_total(self, ctx, data): + return self._get_rate("total") + + def data_rate_segments(self, ctx, data): + return self._get_rate("segments") + + def data_rate_fetch(self, ctx, data): + return self._get_rate("cumulative_fetch") + + def data_rate_decode(self, ctx, data): + return self._get_rate("cumulative_decode") + + def data_rate_decrypt(self, ctx, data): + return self._get_rate("cumulative_decrypt") + + def render_server_timings(self, ctx, data): + d = self.download_results() + d.addCallback(lambda res: res.timings.get("fetch_per_server")) + def _render(per_server): + if per_server is None: + return "" + l = T.ul() + for peerid in sorted(per_server.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + times_s = ", ".join([self.render_time(None, t) + for t in per_server[peerid]]) + l[T.li["[%s]: %s" % (peerid_s, times_s)]] + return T.li["Per-Server Segment Fetch Response Times: ", l] + d.addCallback(_render) + return d + +class DownloadStatusPage(DownloadResultsRendererMixin, rend.Page): + docFactory = getxmlfile("download-status.xhtml") + + def __init__(self, data): + rend.Page.__init__(self, data) + self.download_status = data + + def download_results(self): + return defer.maybeDeferred(self.download_status.get_results) + + def render_results(self, ctx, data): + d = self.download_results() + def _got_results(results): + if results: + return ctx.tag + return "" + d.addCallback(_got_results) + return d + + def render_started(self, ctx, data): + TIME_FORMAT = "%H:%M:%S %d-%b-%Y" + started_s = time.strftime(TIME_FORMAT, + time.localtime(data.get_started())) + return started_s + + def render_si(self, ctx, data): + si_s = base32.b2a_or_none(data.get_storage_index()) + if si_s is None: + si_s = "(None)" + return si_s + + def render_helper(self, ctx, data): + return {True: "Yes", + False: "No"}[data.using_helper()] + + def render_total_size(self, ctx, data): + size = data.get_size() + if size is None: + size = "(unknown)" + return size + + def render_progress(self, ctx, data): + progress = data.get_progress() + # TODO: make an ascii-art bar + return "%.1f%%" % (100.0 * progress) + + def render_status(self, ctx, data): + return data.get_status() + +class RetrieveStatusPage(rend.Page, RateAndTimeMixin): + docFactory = getxmlfile("retrieve-status.xhtml") + + def __init__(self, data): + rend.Page.__init__(self, data) + self.retrieve_status = data + + def render_started(self, ctx, data): + TIME_FORMAT = "%H:%M:%S %d-%b-%Y" + started_s = time.strftime(TIME_FORMAT, + time.localtime(data.get_started())) + return started_s + + def render_si(self, ctx, data): + si_s = base32.b2a_or_none(data.get_storage_index()) + if si_s is None: + si_s = "(None)" + return si_s + + def render_helper(self, ctx, data): + return {True: "Yes", + False: "No"}[data.using_helper()] + + def render_current_size(self, ctx, data): + size = data.get_size() + if size is None: + size = "(unknown)" + return size + + def render_progress(self, ctx, data): + progress = data.get_progress() + # TODO: make an ascii-art bar + return "%.1f%%" % (100.0 * progress) + + def render_status(self, ctx, data): + return data.get_status() + + def render_encoding(self, ctx, data): + k, n = data.get_encoding() + return ctx.tag["Encoding: %s of %s" % (k, n)] + + def render_search_distance(self, ctx, data): + d = data.get_search_distance() + return ctx.tag["Search Distance: %s peer%s" % (d, plural(d))] + + def render_problems(self, ctx, data): + problems = data.problems + if not problems: + return "" + l = T.ul() + for peerid in sorted(problems.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + l[T.li["[%s]: %s" % (peerid_s, problems[peerid])]] + return ctx.tag["Server Problems:", l] + + def _get_rate(self, data, name): + file_size = self.retrieve_status.get_size() + time = self.retrieve_status.timings.get(name) + if time is None: + return None + try: + return 1.0 * file_size / time + except ZeroDivisionError: + return None + + def data_time_total(self, ctx, data): + return self.retrieve_status.timings.get("total") + def data_rate_total(self, ctx, data): + return self._get_rate(data, "total") + + def data_time_peer_selection(self, ctx, data): + return self.retrieve_status.timings.get("peer_selection") + + def data_time_fetch(self, ctx, data): + return self.retrieve_status.timings.get("fetch") + def data_rate_fetch(self, ctx, data): + return self._get_rate(data, "fetch") + + def data_time_decode(self, ctx, data): + return self.retrieve_status.timings.get("decode") + def data_rate_decode(self, ctx, data): + return self._get_rate(data, "decode") + + def data_time_decrypt(self, ctx, data): + return self.retrieve_status.timings.get("decrypt") + def data_rate_decrypt(self, ctx, data): + return self._get_rate(data, "decrypt") + + def render_server_timings(self, ctx, data): + per_server = self.retrieve_status.timings.get("fetch_per_server") + if not per_server: + return "" + l = T.ul() + for peerid in sorted(per_server.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + times_s = ", ".join([self.render_time(None, t) + for t in per_server[peerid]]) + l[T.li["[%s]: %s" % (peerid_s, times_s)]] + return T.li["Per-Server Fetch Response Times: ", l] + + +class PublishStatusPage(rend.Page): + docFactory = getxmlfile("publish-status.xhtml") + + def render_started(self, ctx, data): + TIME_FORMAT = "%H:%M:%S %d-%b-%Y" + started_s = time.strftime(TIME_FORMAT, + time.localtime(data.get_started())) + return started_s + + def render_si(self, ctx, data): + si_s = base32.b2a_or_none(data.get_storage_index()) + if si_s is None: + si_s = "(None)" + return si_s + + def render_helper(self, ctx, data): + return {True: "Yes", + False: "No"}[data.using_helper()] + + def render_current_size(self, ctx, data): + size = data.get_size() + if size is None: + size = "(unknown)" + return size + + def render_progress(self, ctx, data): + progress = data.get_progress() + # TODO: make an ascii-art bar + return "%.1f%%" % (100.0 * progress) + + def render_status(self, ctx, data): + return data.get_status() + +class Status(rend.Page): + docFactory = getxmlfile("status.xhtml") + addSlash = True + + def data_active_operations(self, ctx, data): + active = (IClient(ctx).list_active_uploads() + + IClient(ctx).list_active_downloads() + + IClient(ctx).list_active_publish() + + IClient(ctx).list_active_retrieve()) + return active + + def data_recent_operations(self, ctx, data): + recent = [o for o in (IClient(ctx).list_recent_uploads() + + IClient(ctx).list_recent_downloads() + + IClient(ctx).list_recent_publish() + + IClient(ctx).list_recent_retrieve()) + if not o.get_active()] + recent.sort(lambda a,b: cmp(a.get_started(), b.get_started())) + recent.reverse() + return recent + + def render_row(self, ctx, data): + s = data + + TIME_FORMAT = "%H:%M:%S %d-%b-%Y" + started_s = time.strftime(TIME_FORMAT, + time.localtime(s.get_started())) + ctx.fillSlots("started", started_s) + + si_s = base32.b2a_or_none(s.get_storage_index()) + if si_s is None: + si_s = "(None)" + ctx.fillSlots("si", si_s) + ctx.fillSlots("helper", {True: "Yes", + False: "No"}[s.using_helper()]) + + size = s.get_size() + if size is None: + size = "(unknown)" + ctx.fillSlots("total_size", size) + + progress = data.get_progress() + if IUploadStatus.providedBy(data): + link = "up-%d" % data.get_counter() + ctx.fillSlots("type", "upload") + # TODO: make an ascii-art bar + (chk, ciphertext, encandpush) = progress + progress_s = ("hash: %.1f%%, ciphertext: %.1f%%, encode: %.1f%%" % + ( (100.0 * chk), + (100.0 * ciphertext), + (100.0 * encandpush) )) + ctx.fillSlots("progress", progress_s) + elif IDownloadStatus.providedBy(data): + link = "down-%d" % data.get_counter() + ctx.fillSlots("type", "download") + ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) + elif IPublishStatus.providedBy(data): + link = "publish-%d" % data.get_counter() + ctx.fillSlots("type", "publish") + ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) + else: + assert IRetrieveStatus.providedBy(data) + ctx.fillSlots("type", "retrieve") + link = "retrieve-%d" % data.get_counter() + ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) + ctx.fillSlots("status", T.a(href=link)[s.get_status()]) + return ctx.tag + + def childFactory(self, ctx, name): + client = IClient(ctx) + stype,count_s = name.split("-") + count = int(count_s) + if stype == "up": + for s in client.list_recent_uploads(): + if s.get_counter() == count: + return UploadStatusPage(s) + for s in client.list_all_uploads(): + if s.get_counter() == count: + return UploadStatusPage(s) + if stype == "down": + for s in client.list_recent_downloads(): + if s.get_counter() == count: + return DownloadStatusPage(s) + for s in client.list_all_downloads(): + if s.get_counter() == count: + return DownloadStatusPage(s) + if stype == "publish": + for s in client.list_recent_publish(): + if s.get_counter() == count: + return PublishStatusPage(s) + for s in client.list_all_publish(): + if s.get_counter() == count: + return PublishStatusPage(s) + if stype == "retrieve": + for s in client.list_recent_retrieve(): + if s.get_counter() == count: + return RetrieveStatusPage(s) + for s in client.list_all_retrieve(): + if s.get_counter() == count: + return RetrieveStatusPage(s) + + diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 1cab77a3..96e8c352 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -6,11 +6,10 @@ from twisted.internet import defer, address from twisted.internet.interfaces import IConsumer from nevow import inevow, rend, loaders, appserver, url, tags as T from nevow.static import File as nevow_File # TODO: merge with static.File? -from allmydata.util import base32, fileutil, idlib, observer, log +from allmydata.util import fileutil, idlib, observer, log import simplejson from allmydata.interfaces import IDownloadTarget, IDirectoryNode, IFileNode, \ - IMutableFileNode, IUploadStatus, IDownloadStatus, IPublishStatus, \ - IRetrieveStatus + IMutableFileNode import allmydata # to display import path from allmydata import download from allmydata.upload import FileHandle, FileName @@ -23,11 +22,12 @@ from foolscap.eventual import fireEventually from nevow.util import resource_filename +from allmydata.web import status +from allmydata.web.common import IClient + def getxmlfile(name): return loaders.xmlfile(resource_filename('allmydata.web', '%s' % name)) -class IClient(Interface): - pass class ILocalAccess(Interface): def local_access_is_allowed(): """Return True if t=upload&localdir= is allowed, giving anyone who @@ -1373,180 +1373,7 @@ class UnlinkedPUTCreateDirectory(rend.Page): # XXX add redirect_to_result return d -def plural(sequence_or_length): - if isinstance(sequence_or_length, int): - length = sequence_or_length - else: - length = len(sequence_or_length) - if length == 1: - return "" - return "s" - -class RateAndTimeMixin: - - def render_time(self, ctx, data): - # 1.23s, 790ms, 132us - if data is None: - return "" - s = float(data) - if s >= 1.0: - return "%.2fs" % s - if s >= 0.01: - return "%dms" % (1000*s) - if s >= 0.001: - return "%.1fms" % (1000*s) - return "%dus" % (1000000*s) - - def render_rate(self, ctx, data): - # 21.8kBps, 554.4kBps 4.37MBps - if data is None: - return "" - r = float(data) - if r > 1000000: - return "%1.2fMBps" % (r/1000000) - if r > 1000: - return "%.1fkBps" % (r/1000) - return "%dBps" % r - -class UploadResultsRendererMixin(RateAndTimeMixin): - # this requires a method named 'upload_results' - - def render_sharemap(self, ctx, data): - d = self.upload_results() - d.addCallback(lambda res: res.sharemap) - def _render(sharemap): - if sharemap is None: - return "None" - l = T.ul() - for shnum in sorted(sharemap.keys()): - l[T.li["%d -> %s" % (shnum, sharemap[shnum])]] - return l - d.addCallback(_render) - return d - - def render_servermap(self, ctx, data): - d = self.upload_results() - d.addCallback(lambda res: res.servermap) - def _render(servermap): - if servermap is None: - return "None" - l = T.ul() - for peerid in sorted(servermap.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - shares_s = ",".join(["#%d" % shnum - for shnum in servermap[peerid]]) - l[T.li["[%s] got share%s: %s" % (peerid_s, - plural(servermap[peerid]), - shares_s)]] - return l - d.addCallback(_render) - return d - - def data_file_size(self, ctx, data): - d = self.upload_results() - d.addCallback(lambda res: res.file_size) - return d - - def _get_time(self, name): - d = self.upload_results() - d.addCallback(lambda res: res.timings.get(name)) - return d - - def data_time_total(self, ctx, data): - return self._get_time("total") - - def data_time_storage_index(self, ctx, data): - return self._get_time("storage_index") - - def data_time_contacting_helper(self, ctx, data): - return self._get_time("contacting_helper") - - def data_time_existence_check(self, ctx, data): - return self._get_time("existence_check") - - def data_time_cumulative_fetch(self, ctx, data): - return self._get_time("cumulative_fetch") - - def data_time_helper_total(self, ctx, data): - return self._get_time("helper_total") - - def data_time_peer_selection(self, ctx, data): - return self._get_time("peer_selection") - - def data_time_total_encode_and_push(self, ctx, data): - return self._get_time("total_encode_and_push") - - def data_time_cumulative_encoding(self, ctx, data): - return self._get_time("cumulative_encoding") - - def data_time_cumulative_sending(self, ctx, data): - return self._get_time("cumulative_sending") - - def data_time_hashes_and_close(self, ctx, data): - return self._get_time("hashes_and_close") - - def _get_rate(self, name): - d = self.upload_results() - def _convert(r): - file_size = r.file_size - time = r.timings.get(name) - if time is None: - return None - try: - return 1.0 * file_size / time - except ZeroDivisionError: - return None - d.addCallback(_convert) - return d - - def data_rate_total(self, ctx, data): - return self._get_rate("total") - - def data_rate_storage_index(self, ctx, data): - return self._get_rate("storage_index") - - def data_rate_encode(self, ctx, data): - return self._get_rate("cumulative_encoding") - - def data_rate_push(self, ctx, data): - return self._get_rate("cumulative_sending") - - def data_rate_encode_and_push(self, ctx, data): - d = self.upload_results() - def _convert(r): - file_size = r.file_size - if file_size is None: - return None - time1 = r.timings.get("cumulative_encoding") - if time1 is None: - return None - time2 = r.timings.get("cumulative_sending") - if time2 is None: - return None - try: - return 1.0 * file_size / (time1+time2) - except ZeroDivisionError: - return None - d.addCallback(_convert) - return d - - def data_rate_ciphertext_fetch(self, ctx, data): - d = self.upload_results() - def _convert(r): - fetch_size = r.ciphertext_fetched - if fetch_size is None: - return None - time = r.timings.get("cumulative_fetch") - if time is None: - return None - try: - return 1.0 * fetch_size / time - except ZeroDivisionError: - return None - d.addCallback(_convert) - return d - -class UnlinkedPOSTCHKUploader(UploadResultsRendererMixin, rend.Page): +class UnlinkedPOSTCHKUploader(status.UploadResultsRendererMixin, rend.Page): """'POST /uri', to create an unlinked file.""" docFactory = getxmlfile("upload-results.xhtml") @@ -1625,476 +1452,6 @@ class UnlinkedPOSTCreateDirectory(rend.Page): d.addCallback(lambda dirnode: dirnode.get_uri()) return d -class UploadStatusPage(UploadResultsRendererMixin, rend.Page): - docFactory = getxmlfile("upload-status.xhtml") - - def __init__(self, data): - rend.Page.__init__(self, data) - self.upload_status = data - - def upload_results(self): - return defer.maybeDeferred(self.upload_status.get_results) - - def render_results(self, ctx, data): - d = self.upload_results() - def _got_results(results): - if results: - return ctx.tag - return "" - d.addCallback(_got_results) - return d - - def render_started(self, ctx, data): - TIME_FORMAT = "%H:%M:%S %d-%b-%Y" - started_s = time.strftime(TIME_FORMAT, - time.localtime(data.get_started())) - return started_s - - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) - if si_s is None: - si_s = "(None)" - return si_s - - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] - - def render_total_size(self, ctx, data): - size = data.get_size() - if size is None: - size = "(unknown)" - return size - - def render_progress_hash(self, ctx, data): - progress = data.get_progress()[0] - # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) - - def render_progress_ciphertext(self, ctx, data): - progress = data.get_progress()[1] - # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) - - def render_progress_encode_push(self, ctx, data): - progress = data.get_progress()[2] - # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) - - def render_status(self, ctx, data): - return data.get_status() - -class DownloadResultsRendererMixin(RateAndTimeMixin): - # this requires a method named 'download_results' - - def render_servermap(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.servermap) - def _render(servermap): - if servermap is None: - return "None" - l = T.ul() - for peerid in sorted(servermap.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - shares_s = ",".join(["#%d" % shnum - for shnum in servermap[peerid]]) - l[T.li["[%s] has share%s: %s" % (peerid_s, - plural(servermap[peerid]), - shares_s)]] - return l - d.addCallback(_render) - return d - - def render_servers_used(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.servers_used) - def _got(servers_used): - if not servers_used: - return "" - peerids_s = ", ".join(["[%s]" % idlib.shortnodeid_b2a(peerid) - for peerid in servers_used]) - return T.li["Servers Used: ", peerids_s] - d.addCallback(_got) - return d - - def render_problems(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.server_problems) - def _got(server_problems): - if not server_problems: - return "" - l = T.ul() - for peerid in sorted(server_problems.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - l[T.li["[%s]: %s" % (peerid_s, server_problems[peerid])]] - return T.li["Server Problems:", l] - d.addCallback(_got) - return d - - def data_file_size(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.file_size) - return d - - def _get_time(self, name): - d = self.download_results() - d.addCallback(lambda res: res.timings.get(name)) - return d - - def data_time_total(self, ctx, data): - return self._get_time("total") - - def data_time_peer_selection(self, ctx, data): - return self._get_time("peer_selection") - - def data_time_uri_extension(self, ctx, data): - return self._get_time("uri_extension") - - def data_time_hashtrees(self, ctx, data): - return self._get_time("hashtrees") - - def data_time_segments(self, ctx, data): - return self._get_time("segments") - - def data_time_cumulative_fetch(self, ctx, data): - return self._get_time("cumulative_fetch") - - def data_time_cumulative_decode(self, ctx, data): - return self._get_time("cumulative_decode") - - def data_time_cumulative_decrypt(self, ctx, data): - return self._get_time("cumulative_decrypt") - - def _get_rate(self, name): - d = self.download_results() - def _convert(r): - file_size = r.file_size - time = r.timings.get(name) - if time is None: - return None - try: - return 1.0 * file_size / time - except ZeroDivisionError: - return None - d.addCallback(_convert) - return d - - def data_rate_total(self, ctx, data): - return self._get_rate("total") - - def data_rate_segments(self, ctx, data): - return self._get_rate("segments") - - def data_rate_fetch(self, ctx, data): - return self._get_rate("cumulative_fetch") - - def data_rate_decode(self, ctx, data): - return self._get_rate("cumulative_decode") - - def data_rate_decrypt(self, ctx, data): - return self._get_rate("cumulative_decrypt") - - def render_server_timings(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.timings.get("fetch_per_server")) - def _render(per_server): - if per_server is None: - return "" - l = T.ul() - for peerid in sorted(per_server.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - times_s = ", ".join([self.render_time(None, t) - for t in per_server[peerid]]) - l[T.li["[%s]: %s" % (peerid_s, times_s)]] - return T.li["Per-Server Segment Fetch Response Times: ", l] - d.addCallback(_render) - return d - -class DownloadStatusPage(DownloadResultsRendererMixin, rend.Page): - docFactory = getxmlfile("download-status.xhtml") - - def __init__(self, data): - rend.Page.__init__(self, data) - self.download_status = data - - def download_results(self): - return defer.maybeDeferred(self.download_status.get_results) - - def render_results(self, ctx, data): - d = self.download_results() - def _got_results(results): - if results: - return ctx.tag - return "" - d.addCallback(_got_results) - return d - - def render_started(self, ctx, data): - TIME_FORMAT = "%H:%M:%S %d-%b-%Y" - started_s = time.strftime(TIME_FORMAT, - time.localtime(data.get_started())) - return started_s - - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) - if si_s is None: - si_s = "(None)" - return si_s - - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] - - def render_total_size(self, ctx, data): - size = data.get_size() - if size is None: - size = "(unknown)" - return size - - def render_progress(self, ctx, data): - progress = data.get_progress() - # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) - - def render_status(self, ctx, data): - return data.get_status() - -class RetrieveStatusPage(rend.Page, RateAndTimeMixin): - docFactory = getxmlfile("retrieve-status.xhtml") - - def __init__(self, data): - rend.Page.__init__(self, data) - self.retrieve_status = data - - def render_started(self, ctx, data): - TIME_FORMAT = "%H:%M:%S %d-%b-%Y" - started_s = time.strftime(TIME_FORMAT, - time.localtime(data.get_started())) - return started_s - - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) - if si_s is None: - si_s = "(None)" - return si_s - - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] - - def render_current_size(self, ctx, data): - size = data.get_size() - if size is None: - size = "(unknown)" - return size - - def render_progress(self, ctx, data): - progress = data.get_progress() - # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) - - def render_status(self, ctx, data): - return data.get_status() - - def render_encoding(self, ctx, data): - k, n = data.get_encoding() - return ctx.tag["Encoding: %s of %s" % (k, n)] - - def render_search_distance(self, ctx, data): - d = data.get_search_distance() - return ctx.tag["Search Distance: %s peer%s" % (d, plural(d))] - - def render_problems(self, ctx, data): - problems = data.problems - if not problems: - return "" - l = T.ul() - for peerid in sorted(problems.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - l[T.li["[%s]: %s" % (peerid_s, problems[peerid])]] - return ctx.tag["Server Problems:", l] - - def _get_rate(self, data, name): - file_size = self.retrieve_status.get_size() - time = self.retrieve_status.timings.get(name) - if time is None: - return None - try: - return 1.0 * file_size / time - except ZeroDivisionError: - return None - - def data_time_total(self, ctx, data): - return self.retrieve_status.timings.get("total") - def data_rate_total(self, ctx, data): - return self._get_rate(data, "total") - - def data_time_peer_selection(self, ctx, data): - return self.retrieve_status.timings.get("peer_selection") - - def data_time_fetch(self, ctx, data): - return self.retrieve_status.timings.get("fetch") - def data_rate_fetch(self, ctx, data): - return self._get_rate(data, "fetch") - - def data_time_decode(self, ctx, data): - return self.retrieve_status.timings.get("decode") - def data_rate_decode(self, ctx, data): - return self._get_rate(data, "decode") - - def data_time_decrypt(self, ctx, data): - return self.retrieve_status.timings.get("decrypt") - def data_rate_decrypt(self, ctx, data): - return self._get_rate(data, "decrypt") - - def render_server_timings(self, ctx, data): - per_server = self.retrieve_status.timings.get("fetch_per_server") - if not per_server: - return "" - l = T.ul() - for peerid in sorted(per_server.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - times_s = ", ".join([self.render_time(None, t) - for t in per_server[peerid]]) - l[T.li["[%s]: %s" % (peerid_s, times_s)]] - return T.li["Per-Server Fetch Response Times: ", l] - - -class PublishStatusPage(rend.Page): - docFactory = getxmlfile("publish-status.xhtml") - - def render_started(self, ctx, data): - TIME_FORMAT = "%H:%M:%S %d-%b-%Y" - started_s = time.strftime(TIME_FORMAT, - time.localtime(data.get_started())) - return started_s - - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) - if si_s is None: - si_s = "(None)" - return si_s - - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] - - def render_current_size(self, ctx, data): - size = data.get_size() - if size is None: - size = "(unknown)" - return size - - def render_progress(self, ctx, data): - progress = data.get_progress() - # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) - - def render_status(self, ctx, data): - return data.get_status() - -class Status(rend.Page): - docFactory = getxmlfile("status.xhtml") - addSlash = True - - def data_active_operations(self, ctx, data): - active = (IClient(ctx).list_active_uploads() + - IClient(ctx).list_active_downloads() + - IClient(ctx).list_active_publish() + - IClient(ctx).list_active_retrieve()) - return active - - def data_recent_operations(self, ctx, data): - recent = [o for o in (IClient(ctx).list_recent_uploads() + - IClient(ctx).list_recent_downloads() + - IClient(ctx).list_recent_publish() + - IClient(ctx).list_recent_retrieve()) - if not o.get_active()] - recent.sort(lambda a,b: cmp(a.get_started(), b.get_started())) - recent.reverse() - return recent - - def render_row(self, ctx, data): - s = data - - TIME_FORMAT = "%H:%M:%S %d-%b-%Y" - started_s = time.strftime(TIME_FORMAT, - time.localtime(s.get_started())) - ctx.fillSlots("started", started_s) - - si_s = base32.b2a_or_none(s.get_storage_index()) - if si_s is None: - si_s = "(None)" - ctx.fillSlots("si", si_s) - ctx.fillSlots("helper", {True: "Yes", - False: "No"}[s.using_helper()]) - - size = s.get_size() - if size is None: - size = "(unknown)" - ctx.fillSlots("total_size", size) - - progress = data.get_progress() - if IUploadStatus.providedBy(data): - link = "up-%d" % data.get_counter() - ctx.fillSlots("type", "upload") - # TODO: make an ascii-art bar - (chk, ciphertext, encandpush) = progress - progress_s = ("hash: %.1f%%, ciphertext: %.1f%%, encode: %.1f%%" % - ( (100.0 * chk), - (100.0 * ciphertext), - (100.0 * encandpush) )) - ctx.fillSlots("progress", progress_s) - elif IDownloadStatus.providedBy(data): - link = "down-%d" % data.get_counter() - ctx.fillSlots("type", "download") - ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) - elif IPublishStatus.providedBy(data): - link = "publish-%d" % data.get_counter() - ctx.fillSlots("type", "publish") - ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) - else: - assert IRetrieveStatus.providedBy(data) - ctx.fillSlots("type", "retrieve") - link = "retrieve-%d" % data.get_counter() - ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) - ctx.fillSlots("status", T.a(href=link)[s.get_status()]) - return ctx.tag - - def childFactory(self, ctx, name): - client = IClient(ctx) - stype,count_s = name.split("-") - count = int(count_s) - if stype == "up": - for s in client.list_recent_uploads(): - if s.get_counter() == count: - return UploadStatusPage(s) - for s in client.list_all_uploads(): - if s.get_counter() == count: - return UploadStatusPage(s) - if stype == "down": - for s in client.list_recent_downloads(): - if s.get_counter() == count: - return DownloadStatusPage(s) - for s in client.list_all_downloads(): - if s.get_counter() == count: - return DownloadStatusPage(s) - if stype == "publish": - for s in client.list_recent_publish(): - if s.get_counter() == count: - return PublishStatusPage(s) - for s in client.list_all_publish(): - if s.get_counter() == count: - return PublishStatusPage(s) - if stype == "retrieve": - for s in client.list_recent_retrieve(): - if s.get_counter() == count: - return RetrieveStatusPage(s) - for s in client.list_all_retrieve(): - if s.get_counter() == count: - return RetrieveStatusPage(s) - - class Root(rend.Page): addSlash = True @@ -2170,7 +1527,7 @@ class Root(rend.Page): child_tahoe_css = nevow_File(resource_filename('allmydata.web', 'tahoe.css')) child_provisioning = provisioning.ProvisioningTool() - child_status = Status() + child_status = status.Status() def data_version(self, ctx, data): return get_package_versions_string()