src/allmydata/test/check_load.py

   1 #! /usr/bin/python
   2
   3 """
   4 this is a load-generating client program. It does all of its work through a
   5 given tahoe node (specified by URL), and performs random reads and writes
   6 to the target.
   7
   8 Run this in a directory with the following files:
   9  server-URLs : a list of tahoe node URLs (one per line). Each operation
  10                will use a randomly-selected server.
  11  root.cap: (string) the top-level directory rwcap to use
  12  delay: (float) seconds to delay between operations
  13  operation-mix: "R/W": two ints, relative frequency of read and write ops
  14  #size:?
  15
  16 Set argv[1] to a per-client stats-NN.out file. This will will be updated with
  17 running totals of bytes-per-second and operations-per-second. The stats from
  18 multiple clients can be totalled together and averaged over time to compute
  19 the traffic being accepted by the grid.
  20
  21 Each time a 'read' operation is performed, the client will begin at the root
  22 and randomly choose a child. If the child is a directory, the client will
  23 recurse. If the child is a file, the client will read the contents of the
  24 file.
  25
  26 Each time a 'write' operation is performed, the client will generate a target
  27 filename (a random string). 90% of the time, the file will be written into
  28 the same directory that was used last time (starting at the root). 10% of the
  29 time, a new directory is created by assembling 1 to 5 pathnames chosen at
  30 random. The client then writes a certain number of zero bytes to this file.
  31 The filesize is determined with something like a power-law distribution, with
  32 a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)
  33
  34
  35 """
  36
  37 import os, sys, httplib, binascii
  38 import urllib, simplejson, random, time, urlparse
  39
  40 if sys.argv[1] == "--stats":
  41     statsfiles = sys.argv[2:]
  42     # gather stats every 10 seconds, do a moving-window average of the last
  43     # 60 seconds
  44     DELAY = 10
  45     MAXSAMPLES = 6
  46     totals = []
  47     last_stats = {}
  48     while True:
  49         stats = {}
  50         for sf in statsfiles:
  51             for line in open(sf, "r").readlines():
  52                 name, value = line.split(":")
  53                 value = int(value.strip())
  54                 if name not in stats:
  55                     stats[name] = 0
  56                 stats[name] += float(value)
  57         if last_stats:
  58             delta = dict( [ (name,stats[name]-last_stats[name])
  59                             for name in stats ] )
  60             print "THIS SAMPLE:"
  61             for name in sorted(delta.keys()):
  62                 avg = float(delta[name]) / float(DELAY)
  63                 print "%20s: %0.2f per second" % (name, avg)
  64             totals.append(delta)
  65             while len(totals) > MAXSAMPLES:
  66                 totals.pop(0)
  67
  68             # now compute average
  69             print
  70             print "MOVING WINDOW AVERAGE:"
  71             for name in sorted(delta.keys()):
  72                 avg = sum([ s[name] for s in totals]) / (DELAY*len(totals))
  73                 print "%20s %0.2f per second" % (name, avg)
  74
  75         last_stats = stats
  76         print
  77         print
  78         time.sleep(DELAY)
  79
  80 stats_out = sys.argv[1]
  81
  82 server_urls = []
  83 for url in open("server-URLs", "r").readlines():
  84     url = url.strip()
  85     if url:
  86         server_urls.append(url)
  87 root = open("root.cap", "r").read().strip()
  88 delay = float(open("delay", "r").read().strip())
  89 readfreq, writefreq = (
  90     [int(x) for x in open("operation-mix", "r").read().strip().split("/")])
  91
  92
  93 files_uploaded = 0
  94 files_downloaded = 0
  95 bytes_uploaded = 0
  96 bytes_downloaded = 0
  97 directories_read = 0
  98 directories_written = 0
  99
 100 def listdir(nodeurl, root, vdrive_pathname):
 101     if nodeurl[-1] != "/":
 102         nodeurl += "/"
 103     url = nodeurl + "uri/%s/" % urllib.quote(root)
 104     if vdrive_pathname:
 105         url += urllib.quote(vdrive_pathname)
 106     url += "?t=json"
 107     data = urllib.urlopen(url).read()
 108     try:
 109         parsed = simplejson.loads(data)
 110     except ValueError:
 111         print "URL was", url
 112         print "DATA was", data
 113         raise
 114     nodetype, d = parsed
 115     assert nodetype == "dirnode"
 116     global directories_read
 117     directories_read += 1
 118     return d['children']
 119
 120
 121 def choose_random_descendant(server_url, root, pathname=""):
 122     children = listdir(server_url, root, pathname)
 123     name = random.choice(children.keys())
 124     child = children[name]
 125     if pathname:
 126         new_pathname = pathname + "/" + name
 127     else:
 128         new_pathname = name
 129     if child[0] == "filenode":
 130         return new_pathname
 131     return choose_random_descendant(server_url, root, new_pathname)
 132
 133 def read_and_discard(nodeurl, root, pathname):
 134     if nodeurl[-1] != "/":
 135         nodeurl += "/"
 136     url = nodeurl + "uri/%s/" % urllib.quote(root)
 137     if pathname:
 138         url += urllib.quote(pathname)
 139     f = urllib.urlopen(url)
 140     global bytes_downloaded
 141     while True:
 142         data = f.read(4096)
 143         if not data:
 144             break
 145         bytes_downloaded += len(data)
 146
 147
 148 directories = [
 149     "dreamland/disengaging/hucksters",
 150     "dreamland/disengaging/klondikes",
 151     "dreamland/disengaging/neatly",
 152     "dreamland/cottages/richmond",
 153     "dreamland/cottages/perhaps",
 154     "dreamland/cottages/spies",
 155     "dreamland/finder/diversion",
 156     "dreamland/finder/cigarette",
 157     "dreamland/finder/album",
 158     "hazing/licences/comedian",
 159     "hazing/licences/goat",
 160     "hazing/licences/shopkeeper",
 161     "hazing/regiment/frigate",
 162     "hazing/regiment/quackery",
 163     "hazing/regiment/centerpiece",
 164     "hazing/disassociate/mob",
 165     "hazing/disassociate/nihilistic",
 166     "hazing/disassociate/bilbo",
 167     ]
 168
 169 def create_random_directory():
 170     d = random.choice(directories)
 171     pieces = d.split("/")
 172     numsegs = random.randint(1, len(pieces))
 173     return "/".join(pieces[0:numsegs])
 174
 175 def generate_filename():
 176     fn = binascii.hexlify(os.urandom(4))
 177     return fn
 178
 179 def choose_size():
 180     mean = 10e3
 181     size = random.expovariate(1.0 / mean)
 182     return int(min(size, 100e6))
 183
 184 # copied from twisted/web/client.py
 185 def parse_url(url, defaultPort=None):
 186     url = url.strip()
 187     parsed = urlparse.urlparse(url)
 188     scheme = parsed[0]
 189     path = urlparse.urlunparse(('','')+parsed[2:])
 190     if defaultPort is None:
 191         if scheme == 'https':
 192             defaultPort = 443
 193         else:
 194             defaultPort = 80
 195     host, port = parsed[1], defaultPort
 196     if ':' in host:
 197         host, port = host.split(':')
 198         port = int(port)
 199     if path == "":
 200         path = "/"
 201     return scheme, host, port, path
 202
 203 def generate_and_put(nodeurl, root, vdrive_fname, size):
 204     if nodeurl[-1] != "/":
 205         nodeurl += "/"
 206     url = nodeurl + "uri/%s/" % urllib.quote(root)
 207     url += urllib.quote(vdrive_fname)
 208
 209     scheme, host, port, path = parse_url(url)
 210     if scheme == "http":
 211         c = httplib.HTTPConnection(host, port)
 212     elif scheme == "https":
 213         c = httplib.HTTPSConnection(host, port)
 214     else:
 215         raise ValueError("unknown scheme '%s', need http or https" % scheme)
 216     c.putrequest("PUT", path)
 217     c.putheader("Hostname", host)
 218     c.putheader("User-Agent", "tahoe-check-load")
 219     c.putheader("Connection", "close")
 220     c.putheader("Content-Length", "%d" % size)
 221     c.endheaders()
 222     global bytes_uploaded
 223     while size:
 224         chunksize = min(size, 4096)
 225         size -= chunksize
 226         c.send("\x00" * chunksize)
 227         bytes_uploaded += chunksize
 228     return c.getresponse()
 229
 230
 231 current_writedir = ""
 232
 233 while True:
 234     time.sleep(delay)
 235     if random.uniform(0, readfreq+writefreq) < readfreq:
 236         op = "read"
 237     else:
 238         op = "write"
 239     print "OP:", op
 240     server = random.choice(server_urls)
 241     if op == "read":
 242         pathname = choose_random_descendant(server, root)
 243         print "  reading", pathname
 244         read_and_discard(server, root, pathname)
 245         files_downloaded += 1
 246     elif op == "write":
 247         if random.uniform(0, 100) < 10:
 248             current_writedir = create_random_directory()
 249         filename = generate_filename()
 250         if current_writedir:
 251             pathname = current_writedir + "/" + filename
 252         else:
 253             pathname = filename
 254         print "  writing", pathname
 255         size = choose_size()
 256         print "   size", size
 257         generate_and_put(server, root, pathname, size)
 258         files_uploaded += 1
 259
 260     f = open(stats_out+".tmp", "w")
 261     f.write("files-uploaded: %d\n" % files_uploaded)
 262     f.write("files-downloaded: %d\n" % files_downloaded)
 263     f.write("bytes-uploaded: %d\n" % bytes_uploaded)
 264     f.write("bytes-downloaded: %d\n" % bytes_downloaded)
 265     f.write("directories-read: %d\n" % directories_read)
 266     f.write("directories-written: %d\n" % directories_written)
 267     f.close()
 268     os.rename(stats_out+".tmp", stats_out)
 269