src/allmydata/test/check_load.py

   1 """
   2 this is a load-generating client program. It does all of its work through a
   3 given tahoe node (specified by URL), and performs random reads and writes
   4 to the target.
   5
   6 Run this in a directory with the following files:
   7  server-URLs : a list of tahoe node URLs (one per line). Each operation
   8                will use a randomly-selected server.
   9  root.cap: (string) the top-level directory rwcap to use
  10  delay: (float) seconds to delay between operations
  11  operation-mix: "R/W": two ints, relative frequency of read and write ops
  12  #size:?
  13
  14 Set argv[1] to a per-client stats-NN.out file. This will will be updated with
  15 running totals of bytes-per-second and operations-per-second. The stats from
  16 multiple clients can be totalled together and averaged over time to compute
  17 the traffic being accepted by the grid.
  18
  19 Each time a 'read' operation is performed, the client will begin at the root
  20 and randomly choose a child. If the child is a directory, the client will
  21 recurse. If the child is a file, the client will read the contents of the
  22 file.
  23
  24 Each time a 'write' operation is performed, the client will generate a target
  25 filename (a random string). 90% of the time, the file will be written into
  26 the same directory that was used last time (starting at the root). 10% of the
  27 time, a new directory is created by assembling 1 to 5 pathnames chosen at
  28 random. The client then writes a certain number of zero bytes to this file.
  29 The filesize is determined with something like a power-law distribution, with
  30 a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)
  31
  32
  33 """
  34
  35 import os, sys, httplib, binascii
  36 import urllib, simplejson, random, time, urlparse
  37
  38 if sys.argv[1] == "--stats":
  39     statsfiles = sys.argv[2:]
  40     # gather stats every 10 seconds, do a moving-window average of the last
  41     # 60 seconds
  42     DELAY = 10
  43     MAXSAMPLES = 6
  44     totals = []
  45     last_stats = {}
  46     while True:
  47         stats = {}
  48         for sf in statsfiles:
  49             for line in open(sf, "r").readlines():
  50                 name, value = line.split(":")
  51                 value = int(value.strip())
  52                 if name not in stats:
  53                     stats[name] = 0
  54                 stats[name] += float(value)
  55         if last_stats:
  56             delta = dict( [ (name,stats[name]-last_stats[name])
  57                             for name in stats ] )
  58             print "THIS SAMPLE:"
  59             for name in sorted(delta.keys()):
  60                 avg = float(delta[name]) / float(DELAY)
  61                 print "%20s: %0.2f per second" % (name, avg)
  62             totals.append(delta)
  63             while len(totals) > MAXSAMPLES:
  64                 totals.pop(0)
  65
  66             # now compute average
  67             print
  68             print "MOVING WINDOW AVERAGE:"
  69             for name in sorted(delta.keys()):
  70                 avg = sum([ s[name] for s in totals]) / (DELAY*len(totals))
  71                 print "%20s %0.2f per second" % (name, avg)
  72
  73         last_stats = stats
  74         print
  75         print
  76         time.sleep(DELAY)
  77
  78 stats_out = sys.argv[1]
  79
  80 server_urls = []
  81 for url in open("server-URLs", "r").readlines():
  82     url = url.strip()
  83     if url:
  84         server_urls.append(url)
  85 root = open("root.cap", "r").read().strip()
  86 delay = float(open("delay", "r").read().strip())
  87 readfreq, writefreq = (
  88     [int(x) for x in open("operation-mix", "r").read().strip().split("/")])
  89
  90
  91 files_uploaded = 0
  92 files_downloaded = 0
  93 bytes_uploaded = 0
  94 bytes_downloaded = 0
  95 directories_read = 0
  96 directories_written = 0
  97
  98 def listdir(nodeurl, root, remote_pathname):
  99     if nodeurl[-1] != "/":
 100         nodeurl += "/"
 101     url = nodeurl + "uri/%s/" % urllib.quote(root)
 102     if remote_pathname:
 103         url += urllib.quote(remote_pathname)
 104     url += "?t=json"
 105     data = urllib.urlopen(url).read()
 106     try:
 107         parsed = simplejson.loads(data)
 108     except ValueError:
 109         print "URL was", url
 110         print "DATA was", data
 111         raise
 112     nodetype, d = parsed
 113     assert nodetype == "dirnode"
 114     global directories_read
 115     directories_read += 1
 116     children = dict( [(unicode(name),value)
 117                       for (name,value)
 118                       in d["children"].iteritems()] )
 119     return children
 120
 121
 122 def choose_random_descendant(server_url, root, pathname=""):
 123     children = listdir(server_url, root, pathname)
 124     name = random.choice(children.keys())
 125     child = children[name]
 126     if pathname:
 127         new_pathname = pathname + "/" + name
 128     else:
 129         new_pathname = name
 130     if child[0] == "filenode":
 131         return new_pathname
 132     return choose_random_descendant(server_url, root, new_pathname)
 133
 134 def read_and_discard(nodeurl, root, pathname):
 135     if nodeurl[-1] != "/":
 136         nodeurl += "/"
 137     url = nodeurl + "uri/%s/" % urllib.quote(root)
 138     if pathname:
 139         url += urllib.quote(pathname)
 140     f = urllib.urlopen(url)
 141     global bytes_downloaded
 142     while True:
 143         data = f.read(4096)
 144         if not data:
 145             break
 146         bytes_downloaded += len(data)
 147
 148
 149 directories = [
 150     "dreamland/disengaging/hucksters",
 151     "dreamland/disengaging/klondikes",
 152     "dreamland/disengaging/neatly",
 153     "dreamland/cottages/richmond",
 154     "dreamland/cottages/perhaps",
 155     "dreamland/cottages/spies",
 156     "dreamland/finder/diversion",
 157     "dreamland/finder/cigarette",
 158     "dreamland/finder/album",
 159     "hazing/licences/comedian",
 160     "hazing/licences/goat",
 161     "hazing/licences/shopkeeper",
 162     "hazing/regiment/frigate",
 163     "hazing/regiment/quackery",
 164     "hazing/regiment/centerpiece",
 165     "hazing/disassociate/mob",
 166     "hazing/disassociate/nihilistic",
 167     "hazing/disassociate/bilbo",
 168     ]
 169
 170 def create_random_directory():
 171     d = random.choice(directories)
 172     pieces = d.split("/")
 173     numsegs = random.randint(1, len(pieces))
 174     return "/".join(pieces[0:numsegs])
 175
 176 def generate_filename():
 177     fn = binascii.hexlify(os.urandom(4))
 178     return fn
 179
 180 def choose_size():
 181     mean = 10e3
 182     size = random.expovariate(1.0 / mean)
 183     return int(min(size, 100e6))
 184
 185 # copied from twisted/web/client.py
 186 def parse_url(url, defaultPort=None):
 187     url = url.strip()
 188     parsed = urlparse.urlparse(url)
 189     scheme = parsed[0]
 190     path = urlparse.urlunparse(('','')+parsed[2:])
 191     if defaultPort is None:
 192         if scheme == 'https':
 193             defaultPort = 443
 194         else:
 195             defaultPort = 80
 196     host, port = parsed[1], defaultPort
 197     if ':' in host:
 198         host, port = host.split(':')
 199         port = int(port)
 200     if path == "":
 201         path = "/"
 202     return scheme, host, port, path
 203
 204 def generate_and_put(nodeurl, root, remote_filename, size):
 205     if nodeurl[-1] != "/":
 206         nodeurl += "/"
 207     url = nodeurl + "uri/%s/" % urllib.quote(root)
 208     url += urllib.quote(remote_filename)
 209
 210     scheme, host, port, path = parse_url(url)
 211     if scheme == "http":
 212         c = httplib.HTTPConnection(host, port)
 213     elif scheme == "https":
 214         c = httplib.HTTPSConnection(host, port)
 215     else:
 216         raise ValueError("unknown scheme '%s', need http or https" % scheme)
 217     c.putrequest("PUT", path)
 218     c.putheader("Hostname", host)
 219     c.putheader("User-Agent", "tahoe-check-load")
 220     c.putheader("Connection", "close")
 221     c.putheader("Content-Length", "%d" % size)
 222     c.endheaders()
 223     global bytes_uploaded
 224     while size:
 225         chunksize = min(size, 4096)
 226         size -= chunksize
 227         c.send("\x00" * chunksize)
 228         bytes_uploaded += chunksize
 229     return c.getresponse()
 230
 231
 232 current_writedir = ""
 233
 234 while True:
 235     time.sleep(delay)
 236     if random.uniform(0, readfreq+writefreq) < readfreq:
 237         op = "read"
 238     else:
 239         op = "write"
 240     print "OP:", op
 241     server = random.choice(server_urls)
 242     if op == "read":
 243         pathname = choose_random_descendant(server, root)
 244         print "  reading", pathname
 245         read_and_discard(server, root, pathname)
 246         files_downloaded += 1
 247     elif op == "write":
 248         if random.uniform(0, 100) < 10:
 249             current_writedir = create_random_directory()
 250         filename = generate_filename()
 251         if current_writedir:
 252             pathname = current_writedir + "/" + filename
 253         else:
 254             pathname = filename
 255         print "  writing", pathname
 256         size = choose_size()
 257         print "   size", size
 258         generate_and_put(server, root, pathname, size)
 259         files_uploaded += 1
 260
 261     f = open(stats_out+".tmp", "w")
 262     f.write("files-uploaded: %d\n" % files_uploaded)
 263     f.write("files-downloaded: %d\n" % files_downloaded)
 264     f.write("bytes-uploaded: %d\n" % bytes_uploaded)
 265     f.write("bytes-downloaded: %d\n" % bytes_downloaded)
 266     f.write("directories-read: %d\n" % directories_read)
 267     f.write("directories-written: %d\n" % directories_written)
 268     f.close()
 269     os.rename(stats_out+".tmp", stats_out)
 270