4 Test an existing Tahoe grid, both to see if the grid is still running and to
5 see if the client is still compatible with it. This script is suitable for
6 running from a periodic monitoring script, perhaps by an hourly cronjob.
8 This script uses a pre-established client node (configured to connect to the
9 grid being tested) and a pre-established directory (stored as the 'testgrid:'
10 alias in that client node's aliases file). It then performs a number of
11 uploads and downloads to exercise compatibility in various directions (new
12 client vs old data). All operations are performed by invoking various CLI
13 commands through bin/tahoe . The script must be given two arguments: the
14 client node directory, and the location of the bin/tahoe executable. Note
15 that this script does not import anything from tahoe directly, so it doesn't
16 matter what its PYTHONPATH is, as long as the bin/tahoe that it uses is
19 This script expects that the client node will be not running when the script
20 starts, but it will forcibly shut down the node just to be sure. It will shut
21 down the node after the test finishes.
23 To set up the client node, do the following:
25 tahoe create-client DIR
26 populate DIR/introducer.furl
28 tahoe add-alias -d DIR testgrid `tahoe mkdir -d DIR`
29 pick a 10kB-ish test file, compute its md5sum
30 tahoe put -d DIR FILE testgrid:old.MD5SUM
31 tahoe put -d DIR FILE testgrid:recent.MD5SUM
32 tahoe put -d DIR FILE testgrid:recentdir/recent.MD5SUM
33 echo "" | tahoe put -d DIR --mutable testgrid:log
34 echo "" | tahoe put -d DIR --mutable testgrid:recentlog
36 This script will perform the following steps (the kind of compatibility that
37 is being tested is in [brackets]):
39 read old.* and check the md5sums [confirm that new code can read old files]
40 read all recent.* files and check md5sums [read recent files]
41 delete all recent.* files and verify they're gone [modify an old directory]
42 read recentdir/recent.* files and check [read recent directory]
43 delete recentdir/recent.* and verify [modify recent directory]
44 delete recentdir and verify (keep the directory from growing unboundedly)
46 upload random 10kB file to recentdir/recent.MD5SUM (prepare for next time)
47 upload random 10kB file to recent.MD5SUM [new code can upload to old servers]
48 append one-line timestamp to log [read/write old mutable files]
49 append one-line timestamp to recentlog [read/write recent mutable files]
51 upload small header to new mutable recentlog [create mutable files]
53 This script will also keep track of speeds and latencies and will write them
54 in a machine-readable logfile.
58 import time, subprocess, md5, os.path, random
59 from twisted.python import usage
61 class GridTesterOptions(usage.Options):
64 ("no", "n", "Dry run: do not run any commands, just print them."),
67 def parseArgs(self, nodedir, tahoe):
68 # Note: does not support Unicode arguments.
69 self.nodedir = os.path.expanduser(nodedir)
70 self.tahoe = os.path.abspath(os.path.expanduser(tahoe))
72 class CommandFailed(Exception):
76 def __init__(self, config):
78 self.tahoe = config.tahoe
79 self.nodedir = config.nodedir
81 def command(self, *cmd, **kwargs):
82 expected_rc = kwargs.get("expected_rc", 0)
83 stdin = kwargs.get("stdin", None)
87 p = subprocess.Popen(cmd,
88 stdin=subprocess.PIPE,
89 stdout=subprocess.PIPE,
90 stderr=subprocess.PIPE)
91 (stdout,stderr) = p.communicate(stdin)
93 p = subprocess.Popen(cmd,
94 stdout=subprocess.PIPE,
95 stderr=subprocess.PIPE)
96 (stdout,stderr) = p.communicate()
98 if expected_rc != None and rc != expected_rc:
102 raise CommandFailed("command '%s' failed: rc=%d" % (cmd, rc))
103 return stdout, stderr
105 def cli(self, cmd, *args, **kwargs):
106 print "tahoe", cmd, " ".join(args)
107 stdout, stderr = self.command(self.tahoe, cmd, "-d", self.nodedir,
109 if not kwargs.get("ignore_stderr", False) and stderr != "":
110 raise CommandFailed("command '%s' had stderr: %s" % (" ".join(args),
114 def stop_old_node(self):
115 print "tahoe stop", self.nodedir, "(force)"
116 self.command(self.tahoe, "stop", self.nodedir, expected_rc=None)
118 def start_node(self):
119 print "tahoe start", self.nodedir
120 self.command(self.tahoe, "start", self.nodedir)
124 print "tahoe stop", self.nodedir
125 self.command(self.tahoe, "stop", self.nodedir)
127 def read_and_check(self, f):
128 expected_md5_s = f[f.find(".")+1:]
129 out = self.cli("get", "testgrid:" + f)
130 got_md5_s = md5.new(out).hexdigest()
131 if got_md5_s != expected_md5_s:
132 raise CommandFailed("%s had md5sum of %s" % (f, got_md5_s))
134 def delete_and_check(self, dirname, f):
135 oldfiles = self.listdir(dirname)
137 absfilename = "testgrid:" + dirname + "/" + f
139 absfilename = "testgrid:" + f
140 if f not in oldfiles:
141 raise CommandFailed("um, '%s' was supposed to already be in %s"
143 self.cli("rm", absfilename)
144 newfiles = self.listdir(dirname)
146 raise CommandFailed("failed to remove '%s' from %s" % (f, dirname))
148 def listdir(self, dirname):
149 out = self.cli("ls", "testgrid:"+dirname).strip().split("\n")
150 files = [f.strip() for f in out]
155 files = self.listdir("")
157 if f.startswith("old.") or f.startswith("recent."):
158 self.read_and_check("" + f)
160 if f.startswith("recent."):
161 self.delete_and_check("", f)
162 files = self.listdir("recentdir")
164 if f.startswith("old.") or f.startswith("recent."):
165 self.read_and_check("recentdir/" + f)
167 if f.startswith("recent."):
168 self.delete_and_check("recentdir", f)
169 self.delete_and_check("", "recentdir")
171 self.cli("mkdir", "testgrid:recentdir")
172 fn, data = self.makefile("recent")
173 self.put("recentdir/"+fn, data)
174 files = self.listdir("recentdir")
176 raise CommandFailed("failed to put %s in recentdir/" % fn)
177 fn, data = self.makefile("recent")
179 files = self.listdir("")
181 raise CommandFailed("failed to put %s in testgrid:" % fn)
184 self.update("recentlog")
185 self.delete_and_check("", "recentlog")
186 self.put_mutable("recentlog", "Recent Mutable Log Header\n\n")
188 def put(self, fn, data):
189 self.cli("put", "-", "testgrid:"+fn, stdin=data, ignore_stderr=True)
191 def put_mutable(self, fn, data):
192 self.cli("put", "--mutable", "-", "testgrid:"+fn,
193 stdin=data, ignore_stderr=True)
195 def update(self, fn):
196 old = self.cli("get", "testgrid:"+fn)
197 new = old + time.ctime() + "\n"
200 def makefile(self, prefix):
201 size = random.randint(10001, 10100)
202 data = os.urandom(size)
203 md5sum = md5.new(data).hexdigest()
204 fn = prefix + "." + md5sum
216 config = GridTesterOptions()
217 config.parseOptions()
218 gt = GridTester(config)
221 if __name__ == "__main__":