src/allmydata/control.py

   1
   2 import os, time
   3 from zope.interface import implements
   4 from twisted.application import service
   5 from twisted.internet import defer
   6 from foolscap import Referenceable
   7 from allmydata.interfaces import RIControlClient
   8 from allmydata.util import testutil, fileutil, mathutil
   9 from allmydata import upload, download
  10 from twisted.python import log
  11
  12 def get_memory_usage():
  13     # this is obviously linux-specific
  14     stat_names = ("VmPeak",
  15                   "VmSize",
  16                   #"VmHWM",
  17                   "VmData")
  18     stats = {}
  19     try:
  20         for line in open("/proc/self/status", "r").readlines():
  21             name, right = line.split(":",2)
  22             if name in stat_names:
  23                 assert right.endswith(" kB\n")
  24                 right = right[:-4]
  25                 stats[name] = int(right) * 1024
  26     except:
  27         # Probably not on (a compatible version of) Linux
  28         stats['VmSize'] = 0
  29         stats['VmPeak'] = 0
  30     return stats
  31
  32 def log_memory_usage(where=""):
  33     stats = get_memory_usage()
  34     log.msg("VmSize: %9d  VmPeak: %9d  %s" % (stats["VmSize"],
  35                                               stats["VmPeak"],
  36                                               where))
  37
  38
  39 class ControlServer(Referenceable, service.Service, testutil.PollMixin):
  40     implements(RIControlClient)
  41
  42     def remote_wait_for_client_connections(self, num_clients):
  43         return self.parent.debug_wait_for_client_connections(num_clients)
  44
  45     def remote_upload_from_file_to_uri(self, filename):
  46         uploader = self.parent.getServiceNamed("uploader")
  47         u = upload.FileName(filename)
  48         d = uploader.upload(u)
  49         return d
  50
  51     def remote_download_from_uri_to_file(self, uri, filename):
  52         downloader = self.parent.getServiceNamed("downloader")
  53         d = downloader.download_to_filename(uri, filename)
  54         d.addCallback(lambda res: filename)
  55         return d
  56
  57     def remote_speed_test(self, count, size, mutable):
  58         assert size > 8
  59         log.msg("speed_test: count=%d, size=%d, mutable=%s" % (count, size,
  60                                                                mutable))
  61         st = SpeedTest(self.parent, count, size, mutable)
  62         return st.run()
  63
  64     def remote_get_memory_usage(self):
  65         return get_memory_usage()
  66
  67     def remote_measure_peer_response_time(self):
  68         # I'd like to average together several pings, but I don't want this
  69         # phase to take more than 10 seconds. Expect worst-case latency to be
  70         # 300ms.
  71         results = {}
  72         everyone = list(self.parent.introducer_client.get_all_peers())
  73         num_pings = int(mathutil.div_ceil(10, (len(everyone) * 0.3)))
  74         everyone = everyone * num_pings
  75         d = self._do_one_ping(None, everyone, results)
  76         return d
  77     def _do_one_ping(self, res, everyone_left, results):
  78         if not everyone_left:
  79             return results
  80         peerid, connection = everyone_left.pop(0)
  81         start = time.time()
  82         d = connection.callRemote("get_nodeid")
  83         def _done(ignored):
  84             stop = time.time()
  85             elapsed = stop - start
  86             if peerid in results:
  87                 results[peerid].append(elapsed)
  88             else:
  89                 results[peerid] = [elapsed]
  90         d.addCallback(_done)
  91         d.addCallback(self._do_one_ping, everyone_left, results)
  92         def _average(res):
  93             averaged = {}
  94             for peerid,times in results.iteritems():
  95                 averaged[peerid] = sum(times) / len(times)
  96             return averaged
  97         d.addCallback(_average)
  98         return d
  99
 100 class SpeedTest:
 101     def __init__(self, parent, count, size, mutable):
 102         self.parent = parent
 103         self.count = count
 104         self.size = size
 105         self.mutable_mode = mutable
 106         self.uris = {}
 107         self.basedir = os.path.join(self.parent.basedir, "_speed_test_data")
 108
 109     def run(self):
 110         self.create_data()
 111         d = self.do_upload()
 112         d.addCallback(lambda res: self.do_download())
 113         d.addBoth(self.do_cleanup)
 114         d.addCallback(lambda res: (self.upload_time, self.download_time))
 115         return d
 116
 117     def create_data(self):
 118         fileutil.make_dirs(self.basedir)
 119         for i in range(self.count):
 120             s = self.size
 121             fn = os.path.join(self.basedir, str(i))
 122             if os.path.exists(fn):
 123                 os.unlink(fn)
 124             f = open(fn, "w")
 125             f.write(os.urandom(8))
 126             s -= 8
 127             while s > 0:
 128                 chunk = min(s, 4096)
 129                 f.write("\x00" * chunk)
 130                 s -= chunk
 131             f.close()
 132
 133     def do_upload(self):
 134         d = defer.succeed(None)
 135         def _create_slot(res):
 136             d1 = self.parent.create_mutable_file("")
 137             def _created(n):
 138                 self._n = n
 139             d1.addCallback(_created)
 140             return d1
 141         if self.mutable_mode == "upload":
 142             d.addCallback(_create_slot)
 143         def _start(res):
 144             self._start = time.time()
 145         d.addCallback(_start)
 146
 147         def _record_uri(uri, i):
 148             self.uris[i] = uri
 149         def _upload_one_file(ignored, i):
 150             if i >= self.count:
 151                 return
 152             fn = os.path.join(self.basedir, str(i))
 153             if self.mutable_mode == "create":
 154                 data = open(fn,"rb").read()
 155                 d1 = self.parent.create_mutable_file(data)
 156                 d1.addCallback(lambda n: n.get_uri())
 157             elif self.mutable_mode == "upload":
 158                 data = open(fn,"rb").read()
 159                 d1 = self._n.replace(data)
 160                 d1.addCallback(lambda res: self._n.get_uri())
 161             else:
 162                 up = upload.FileName(fn)
 163                 d1 = self.parent.upload(up)
 164             d1.addCallback(_record_uri, i)
 165             d1.addCallback(_upload_one_file, i+1)
 166             return d1
 167         d.addCallback(_upload_one_file, 0)
 168         def _upload_done(ignored):
 169             stop = time.time()
 170             self.upload_time = stop - self._start
 171         d.addCallback(_upload_done)
 172         return d
 173
 174     def do_download(self):
 175         start = time.time()
 176         d = defer.succeed(None)
 177         def _download_one_file(ignored, i):
 178             if i >= self.count:
 179                 return
 180             n = self.parent.create_node_from_uri(self.uris[i])
 181             d1 = n.download(download.FileHandle(Discard()))
 182             d1.addCallback(_download_one_file, i+1)
 183             return d1
 184         d.addCallback(_download_one_file, 0)
 185         def _download_done(ignored):
 186             stop = time.time()
 187             self.download_time = stop - start
 188         d.addCallback(_download_done)
 189         return d
 190
 191     def do_cleanup(self, res):
 192         for i in range(self.count):
 193             fn = os.path.join(self.basedir, str(i))
 194             os.unlink(fn)
 195         return res
 196
 197 class Discard:
 198     def write(self, data):
 199         pass
 200     # download_to_filehandle explicitly does not close the filehandle it was
 201     # given: that is reserved for the provider of the filehandle. Therefore
 202     # the lack of a close() method on this otherwise filehandle-like object
 203     # is a part of the test.