From: Brian Warner <warner@allmydata.com> Date: Sat, 1 Mar 2008 05:19:03 +0000 (-0700) Subject: retain 10 most recent upload/download status objects, show them in /status . Prep... X-Git-Tag: allmydata-tahoe-0.9.0~87 X-Git-Url: https://git.rkrishnan.org/%5B/%5D%20/file/URI:LIT:krugkidfnzsc4/(%5B%5E?a=commitdiff_plain;h=1a7651ce825030edfe5fc49c2558b2c9d333162f;p=tahoe-lafs%2Ftahoe-lafs.git retain 10 most recent upload/download status objects, show them in /status . Prep for showing individual status objects --- diff --git a/src/allmydata/client.py b/src/allmydata/client.py index c0ea63ae..c8c1b183 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -267,10 +267,18 @@ class Client(node.Node, testutil.PollMixin): uploader = self.getServiceNamed("uploader") return uploader.upload(uploadable) - def list_uploads(self): + def list_all_uploads(self): uploader = self.getServiceNamed("uploader") - return uploader.list_uploads() + return uploader.list_all_uploads() - def list_downloads(self): + def list_all_downloads(self): downloader = self.getServiceNamed("downloader") - return downloader.list_downloads() + return downloader.list_all_downloads() + + def list_recent_uploads(self): + uploader = self.getServiceNamed("uploader") + return uploader.list_recent_uploads() + + def list_recent_downloads(self): + downloader = self.getServiceNamed("downloader") + return downloader.list_recent_downloads() diff --git a/src/allmydata/download.py b/src/allmydata/download.py index b4a3261d..83bebb37 100644 --- a/src/allmydata/download.py +++ b/src/allmydata/download.py @@ -1,5 +1,5 @@ -import os, random, weakref +import os, random, weakref, itertools from zope.interface import implements from twisted.internet import defer from twisted.internet.interfaces import IPushProducer, IConsumer @@ -327,6 +327,7 @@ class SegmentDownloader: class DownloadStatus: implements(IDownloadStatus) + statusid_counter = itertools.count(0) def __init__(self): self.storage_index = None @@ -337,6 +338,7 @@ class DownloadStatus: self.paused = False self.stopped = False self.active = True + self.counter = self.statusid_counter.next() def get_storage_index(self): return self.storage_index @@ -355,6 +357,8 @@ class DownloadStatus: return self.progress def get_active(self): return self.active + def get_counter(self): + return self.counter def set_storage_index(self, si): self.storage_index = si @@ -907,10 +911,12 @@ class Downloader(service.MultiService): """ implements(IDownloader) name = "downloader" + MAX_DOWNLOAD_STATUSES = 10 def __init__(self): service.MultiService.__init__(self) self._all_downloads = weakref.WeakKeyDictionary() + self._recent_download_status = [] def download(self, u, t): assert self.parent @@ -925,7 +931,10 @@ class Downloader(service.MultiService): dl = FileDownloader(self.parent, u, t) else: raise RuntimeError("I don't know how to download a %s" % u) - self._all_downloads[dl.get_download_status()] = None + self._all_downloads[dl] = None + self._recent_download_status.append(dl.get_download_status()) + while len(self._recent_download_status) > self.MAX_DOWNLOAD_STATUSES: + self._recent_download_status.pop(0) d = dl.start() return d @@ -938,5 +947,7 @@ class Downloader(service.MultiService): return self.download(uri, FileHandle(filehandle)) - def list_downloads(self): + def list_all_downloads(self): return self._all_downloads.keys() + def list_recent_downloads(self): + return self._recent_download_status diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index d4e6cc7c..35bf0a34 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1388,12 +1388,23 @@ class IClient(Interface): """ class IClientStatus(Interface): - def list_uploads(): - """Return a list of IUploadStatus objects, one for each - upload which is currently running.""" - def list_downloads(): - """Return a list of IDownloadStatus objects, one for each - download which is currently running.""" + def list_all_uploads(): + """Return a list of IUploadStatus objects, one for each upload which + currently has an object available. This uses weakrefs to track the + objects, so it may report uploads which have already finished. Use + get_active() to filter these out.""" + def list_recent_uploads(): + """Return a list of IUploadStatus objects for the most recently + started uploads.""" + + def list_all_downloads(): + """Return a list of IDownloadStatus objects, one for each download + which currently has an object available. This uses weakrefs to track + the objects, so it may report downloadswhich have already finished. + Use get_active() to filter these out.""" + def list_recent_downloads(): + """Return a list of IDownloadStatus objects for the most recently + started downloads.""" class IUploadStatus(Interface): def get_storage_index(): @@ -1422,6 +1433,10 @@ class IUploadStatus(Interface): three numbers and report the sum to the user.""" def get_active(): """Return True if the upload is currently active, False if not.""" + def get_counter(): + """Each upload status gets a unique number: this method returns that + number. This provides a handle to this particular upload, so a web + page can generate a suitable hyperlink.""" class IDownloadStatus(Interface): def get_storage_index(): @@ -1443,6 +1458,10 @@ class IDownloadStatus(Interface): first byte of plaintext is pushed to the download target.""" def get_active(): """Return True if the download is currently active, False if not.""" + def get_counter(): + """Each download status gets a unique number: this method returns + that number. This provides a handle to this particular download, so a + web page can generate a suitable hyperlink.""" class NotCapableError(Exception): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 3e2ef1ca..0ce85d75 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1171,7 +1171,7 @@ class SystemTest(testutil.SignalMixin, testutil.PollMixin, unittest.TestCase): file=("foo.txt", "data2" * 10000))) # check that the status page exists - d.addCallback(lambda res: self.GET("status")) + d.addCallback(lambda res: self.GET("status", followRedirect=True)) # TODO: mangle the second segment of a file, to test errors that # occur after we've already sent some good data, which uses a diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index 122dbacc..30dcadcf 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -67,9 +67,14 @@ class FakeClient(service.MultiService): d.addCallback(_got_data) return d - def list_uploads(self): + def list_all_uploads(self): return [upload.UploadStatus()] - def list_downloads(self): + def list_all_downloads(self): + return [download.DownloadStatus()] + + def list_recent_uploads(self): + return [upload.UploadStatus()] + def list_recent_downloads(self): return [download.DownloadStatus()] @@ -375,7 +380,7 @@ class Web(WebMixin, unittest.TestCase): return d def test_status(self): - d = self.GET("/status") + d = self.GET("/status", followRedirect=True) def _check(res): self.failUnless('Upload and Download Status' in res) d.addCallback(_check) diff --git a/src/allmydata/upload.py b/src/allmydata/upload.py index 7b43fb38..04cc9317 100644 --- a/src/allmydata/upload.py +++ b/src/allmydata/upload.py @@ -1,5 +1,5 @@ -import os, time, weakref +import os, time, weakref, itertools from zope.interface import implements from twisted.python import failure from twisted.internet import defer @@ -564,6 +564,7 @@ class EncryptAnUploadable: class UploadStatus: implements(IUploadStatus) + statusid_counter = itertools.count(0) def __init__(self): self.storage_index = None @@ -572,6 +573,7 @@ class UploadStatus: self.status = "Not started" self.progress = [0.0, 0.0, 0.0] self.active = True + self.counter = self.statusid_counter.next() def get_storage_index(self): return self.storage_index @@ -585,6 +587,8 @@ class UploadStatus: return tuple(self.progress) def get_active(self): return self.active + def get_counter(self): + return self.counter def set_storage_index(self, si): self.storage_index = si @@ -1139,11 +1143,13 @@ class Uploader(service.MultiService): name = "uploader" uploader_class = CHKUploader URI_LIT_SIZE_THRESHOLD = 55 + MAX_UPLOAD_STATUSES = 10 def __init__(self, helper_furl=None): self._helper_furl = helper_furl self._helper = None self._all_uploads = weakref.WeakKeyDictionary() + self._recent_upload_status = [] service.MultiService.__init__(self) def startService(self): @@ -1180,7 +1186,10 @@ class Uploader(service.MultiService): uploader = AssistedUploader(self._helper) else: uploader = self.uploader_class(self.parent) - self._all_uploads[uploader.get_upload_status()] = None + self._all_uploads[uploader] = None + self._recent_upload_status.append(uploader.get_upload_status()) + while len(self._recent_upload_status) > self.MAX_UPLOAD_STATUSES: + self._recent_upload_status.pop(0) return uploader.start(uploadable) d.addCallback(_got_size) def _done(res): @@ -1189,5 +1198,7 @@ class Uploader(service.MultiService): d.addBoth(_done) return d - def list_uploads(self): + def list_all_uploads(self): return self._all_uploads.keys() + def list_recent_uploads(self): + return self._recent_upload_status diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index 485059a0..9bb24580 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -12,7 +12,7 @@ <div>Please visit the <a href="http://allmydata.org">Tahoe home page</a> for code updates and bug reporting. The <a href="provisioning">provisioning -tool</a> may also be useful. <a href="status">Current Uploads and +tool</a> may also be useful. <a href="status/">Current Uploads and Downloads</a></div> <h2>Grid Status</h2> diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index bb584503..4359d7f2 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -9,7 +9,7 @@ from nevow.static import File as nevow_File # TODO: merge with static.File? from allmydata.util import base32, fileutil, idlib, observer, log import simplejson from allmydata.interfaces import IDownloadTarget, IDirectoryNode, IFileNode, \ - IMutableFileNode + IMutableFileNode, IUploadStatus, IDownloadStatus import allmydata # to display import path from allmydata import download from allmydata.upload import FileHandle, FileName @@ -1610,15 +1610,20 @@ class UnlinkedPOSTCreateDirectory(rend.Page): class Status(rend.Page): docFactory = getxmlfile("status.xhtml") + addSlash = True def data_active_uploads(self, ctx, data): - return [u for u in IClient(ctx).list_uploads() if u.get_active()] + return [u for u in IClient(ctx).list_all_uploads() + if u.get_active()] def data_active_downloads(self, ctx, data): - return [d for d in IClient(ctx).list_downloads() if d.get_active()] + return [d for d in IClient(ctx).list_all_downloads() + if d.get_active()] def data_recent_uploads(self, ctx, data): - return [u for u in IClient(ctx).list_uploads() if not u.get_active()] + return [u for u in IClient(ctx).list_recent_uploads() + if not u.get_active()] def data_recent_downloads(self, ctx, data): - return [d for d in IClient(ctx).list_downloads() if not d.get_active()] + return [d for d in IClient(ctx).list_recent_downloads() + if not d.get_active()] def _render_common(self, ctx, data): s = data @@ -1632,6 +1637,12 @@ class Status(rend.Page): if size is None: size = "(unknown)" ctx.fillSlots("total_size", size) + if IUploadStatus.providedBy(data): + link = "up-%d" % data.get_counter() + else: + assert IDownloadStatus.providedBy(data) + link = "down-%d" % data.get_counter() + #ctx.fillSlots("status", T.a(href=link)[s.get_status()]) ctx.fillSlots("status", s.get_status()) def render_row_upload(self, ctx, data):