src/allmydata/scripts/runner.py

   1 #! /usr/bin/env python
   2
   3 import os, subprocess, sys, signal, time
   4 from cStringIO import StringIO
   5 from twisted.python import usage
   6
   7 from twisted.python.procutils import which
   8
   9 def testtwistd(loc):
  10     try:
  11         return subprocess.call(["python", loc,], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  12     except:
  13         return -1
  14
  15 twistd = None
  16 if not twistd:
  17     for maybetwistd in which("twistd"):
  18         ret = testtwistd(maybetwistd)
  19         if ret == 0:
  20             twistd = maybetwistd
  21             break
  22
  23 if not twistd:
  24     for maybetwistd in which("twistd.py"):
  25         ret = testtwistd(maybetwistd)
  26         if ret == 0:
  27             twistd = maybetwistd
  28             break
  29
  30 if not twistd:
  31     maybetwistd = os.path.join(sys.prefix, 'Scripts', 'twistd')
  32     ret = testtwistd(maybetwistd)
  33     if ret == 0:
  34         twistd = maybetwistd
  35
  36 if not twistd:
  37     maybetwistd = os.path.join(sys.prefix, 'Scripts', 'twistd.py')
  38     ret = testtwistd(maybetwistd)
  39     if ret == 0:
  40         twistd = maybetwistd
  41
  42 if not twistd:
  43     print "Can't find twistd (it comes with Twisted).  Aborting."
  44     sys.exit(1)
  45
  46 class BasedirMixin:
  47     optFlags = [
  48         ["multiple", "m", "allow multiple basedirs to be specified at once"],
  49         ]
  50
  51     def postOptions(self):
  52         if not self.basedirs:
  53             raise usage.UsageError("<basedir> parameter is required")
  54         if self['basedir']:
  55             del self['basedir']
  56         self['basedirs'] = [os.path.abspath(os.path.expanduser(b))
  57                             for b in self.basedirs]
  58
  59     def parseArgs(self, *args):
  60         self.basedirs = []
  61         if self['basedir']:
  62             self.basedirs.append(self['basedir'])
  63         if self['multiple']:
  64             self.basedirs.extend(args)
  65         else:
  66             if len(args) == 0 and not self.basedirs:
  67                 self.basedirs.append(".")
  68             if len(args) > 0:
  69                 self.basedirs.append(args[0])
  70             if len(args) > 1:
  71                 raise usage.UsageError("I wasn't expecting so many arguments")
  72
  73 class NoDefaultBasedirMixin(BasedirMixin):
  74     def parseArgs(self, *args):
  75         # create-client won't default to --basedir=.
  76         self.basedirs = []
  77         if self['basedir']:
  78             self.basedirs.append(self['basedir'])
  79         if self['multiple']:
  80             self.basedirs.extend(args)
  81         else:
  82             if len(args) > 0:
  83                 self.basedirs.append(args[0])
  84             if len(args) > 1:
  85                 raise usage.UsageError("I wasn't expecting so many arguments")
  86         if not self.basedirs:
  87             raise usage.UsageError("--basedir must be provided")
  88
  89 class StartOptions(BasedirMixin, usage.Options):
  90     optParameters = [
  91         ["basedir", "C", None, "which directory to start the node in"],
  92         ]
  93
  94 class StopOptions(BasedirMixin, usage.Options):
  95     optParameters = [
  96         ["basedir", "C", None, "which directory to stop the node in"],
  97         ]
  98
  99 class RestartOptions(BasedirMixin, usage.Options):
 100     optParameters = [
 101         ["basedir", "C", None, "which directory to restart the node in"],
 102         ]
 103
 104 class CreateClientOptions(NoDefaultBasedirMixin, usage.Options):
 105     optParameters = [
 106         ["basedir", "C", None, "which directory to create the client in"],
 107         ]
 108
 109 class CreateIntroducerOptions(NoDefaultBasedirMixin, usage.Options):
 110     optParameters = [
 111         ["basedir", "C", None, "which directory to create the introducer in"],
 112         ]
 113
 114 class DumpOptions(usage.Options):
 115     optParameters = [
 116         ["filename", "f", None, "which file to dump"],
 117         ]
 118
 119     def parseArgs(self, filename=None):
 120         if filename:
 121             self['filename'] = filename
 122
 123     def postOptions(self):
 124         if not self['filename']:
 125             raise usage.UsageError("<filename> parameter is required")
 126
 127 class DumpRootDirnodeOptions(BasedirMixin, usage.Options):
 128     optParameters = [
 129         ["basedir", "C", None, "the vdrive-server's base directory"],
 130         ]
 131
 132 class DumpDirnodeOptions(BasedirMixin, usage.Options):
 133     optParameters = [
 134         ["uri", "u", None, "the URI of the dirnode to dump."],
 135         ["basedir", "C", None, "which directory to create the introducer in"],
 136         ]
 137     optFlags = [
 138         ["verbose", "v", "be extra noisy (show encrypted data)"],
 139         ]
 140     def parseArgs(self, *args):
 141         if len(args) == 1:
 142             self['uri'] = args[-1]
 143             args = args[:-1]
 144         BasedirMixin.parseArgs(self, *args)
 145
 146     def postOptions(self):
 147         BasedirMixin.postOptions(self)
 148         if not self['uri']:
 149             raise usage.UsageError("<uri> parameter is required")
 150
 151 client_tac = """
 152 # -*- python -*-
 153
 154 from allmydata import client
 155 from twisted.application import service
 156
 157 c = client.Client()
 158
 159 application = service.Application("allmydata_client")
 160 c.setServiceParent(application)
 161 """
 162
 163 introducer_tac = """
 164 # -*- python -*-
 165
 166 from allmydata import introducer_and_vdrive
 167 from twisted.application import service
 168
 169 c = introducer_and_vdrive.IntroducerAndVdrive()
 170
 171 application = service.Application("allmydata_introducer")
 172 c.setServiceParent(application)
 173 """
 174
 175 class Options(usage.Options):
 176     synopsis = "Usage:  allmydata <command> [command options]"
 177
 178     optFlags = [
 179         ["quiet", "q", "operate silently"],
 180         ]
 181
 182     subCommands = [
 183         ["create-client", None, CreateClientOptions, "Create a client node."],
 184         ["create-introducer", None, CreateIntroducerOptions, "Create a introducer node."],
 185         ["start", None, StartOptions, "Start a node (of any type)."],
 186         ["stop", None, StopOptions, "Stop a node."],
 187         ["restart", None, RestartOptions, "Restart a node."],
 188         ["dump-uri-extension", None, DumpOptions,
 189          "Unpack and display the contents of a uri_extension file."],
 190         ["dump-root-dirnode", None, DumpRootDirnodeOptions,
 191          "Compute most of the URI for the vdrive server's root dirnode."],
 192         ["dump-dirnode", None, DumpDirnodeOptions,
 193          "Unpack and display the contents of a vdrive DirectoryNode."],
 194         ]
 195
 196     def postOptions(self):
 197         if not hasattr(self, 'subOptions'):
 198             raise usage.UsageError("must specify a command")
 199
 200 def runner(argv, run_by_human=True, stdout=sys.stdout, stderr=sys.stderr):
 201     config = Options()
 202     try:
 203         config.parseOptions(argv)
 204     except usage.error, e:
 205         if not run_by_human:
 206             raise
 207         print "%s:  %s" % (sys.argv[0], e)
 208         print
 209         c = getattr(config, 'subOptions', config)
 210         print str(c)
 211         return 1
 212
 213     command = config.subCommand
 214     so = config.subOptions
 215
 216     if config['quiet']:
 217         stdout = StringIO()
 218
 219     rc = 0
 220     if command == "create-client":
 221         for basedir in so.basedirs:
 222             rc = create_client(basedir, so, stdout, stderr) or rc
 223     elif command == "create-introducer":
 224         for basedir in so.basedirs:
 225             rc = create_introducer(basedir, so, stdout, stderr) or rc
 226     elif command == "start":
 227         for basedir in so.basedirs:
 228             rc = start(basedir, so, stdout, stderr) or rc
 229     elif command == "stop":
 230         for basedir in so.basedirs:
 231             rc = stop(basedir, so, stdout, stderr) or rc
 232     elif command == "restart":
 233         for basedir in so.basedirs:
 234             rc = stop(basedir, so, stdout, stderr) or rc
 235         if rc:
 236             print >>stderr, "not restarting"
 237             return rc
 238         for basedir in so.basedirs:
 239             rc = start(basedir, so, stdout, stderr) or rc
 240     elif command == "dump-uri-extension":
 241         rc = dump_uri_extension(so, stdout, stderr)
 242     elif command == "dump-root-dirnode":
 243         rc = dump_root_dirnode(so.basedirs[0], so, stdout, stderr)
 244     elif command == "dump-dirnode":
 245         rc = dump_directory_node(so.basedirs[0], so, stdout, stderr)
 246     return rc
 247
 248 def run():
 249     rc = runner(sys.argv[1:])
 250     sys.exit(rc)
 251
 252 def create_client(basedir, config, out=sys.stdout, err=sys.stderr):
 253     if os.path.exists(basedir):
 254         if os.listdir(basedir):
 255             print >>err, "The base directory already exists: %s" % basedir
 256             print >>err, "To avoid clobbering anything, I am going to quit now"
 257             print >>err, "Please use a different directory, or delete this one"
 258             return -1
 259         # we're willing to use an empty directory
 260     else:
 261         os.mkdir(basedir)
 262     f = open(os.path.join(basedir, "client.tac"), "w")
 263     f.write(client_tac)
 264     f.close()
 265     print >>out, "client created in %s" % basedir
 266     print >>out, " please copy introducer.furl and vdrive.furl into the directory"
 267
 268 def create_introducer(basedir, config, out=sys.stdout, err=sys.stderr):
 269     if os.path.exists(basedir):
 270         if os.listdir(basedir):
 271             print >>err, "The base directory already exists: %s" % basedir
 272             print >>err, "To avoid clobbering anything, I am going to quit now"
 273             print >>err, "Please use a different directory, or delete this one"
 274             return -1
 275         # we're willing to use an empty directory
 276     else:
 277         os.mkdir(basedir)
 278     f = open(os.path.join(basedir, "introducer.tac"), "w")
 279     f.write(introducer_tac)
 280     f.close()
 281     print >>out, "introducer created in %s" % basedir
 282
 283 def start(basedir, config, out=sys.stdout, err=sys.stderr):
 284     print >>out, "STARTING", basedir
 285     if os.path.exists(os.path.join(basedir, "client.tac")):
 286         tac = "client.tac"
 287         type = "client"
 288     elif os.path.exists(os.path.join(basedir, "introducer.tac")):
 289         tac = "introducer.tac"
 290         type = "introducer"
 291     else:
 292         print >>err, "%s does not look like a node directory" % basedir
 293         if not os.path.isdir(basedir):
 294             print >>err, " in fact, it doesn't look like a directory at all!"
 295         sys.exit(1)
 296     rc = subprocess.call(["python", twistd, "-y", tac,], cwd=basedir)
 297     if rc == 0:
 298         print >>out, "%s node probably started" % type
 299         return 0
 300     else:
 301         print >>err, "%s node probably not started" % type
 302         return 1
 303
 304 def stop(basedir, config, out=sys.stdout, err=sys.stderr):
 305     print >>out, "STOPPING", basedir
 306     pidfile = os.path.join(basedir, "twistd.pid")
 307     if not os.path.exists(pidfile):
 308         print >>err, "%s does not look like a running node directory (no twistd.pid)" % basedir
 309         return 1
 310     pid = open(pidfile, "r").read()
 311     pid = int(pid)
 312
 313     timer = 0
 314     os.kill(pid, signal.SIGTERM)
 315     time.sleep(0.1)
 316     while timer < 5:
 317         # poll once per second until twistd.pid goes away, up to 5 seconds
 318         try:
 319             os.kill(pid, 0)
 320         except OSError:
 321             print >>out, "process %d is dead" % pid
 322             return
 323         timer += 1
 324         time.sleep(1)
 325     print >>err, "never saw process go away"
 326     return 1
 327
 328 def dump_uri_extension(config, out=sys.stdout, err=sys.stderr):
 329     from allmydata import uri
 330
 331     filename = config['filename']
 332     unpacked = uri.unpack_extension_readable(open(filename,"rb").read())
 333     keys1 = ("size", "num_segments", "segment_size",
 334              "needed_shares", "total_shares")
 335     keys2 = ("codec_name", "codec_params", "tail_codec_params")
 336     keys3 = ("plaintext_hash", "plaintext_root_hash",
 337              "crypttext_hash", "crypttext_root_hash",
 338              "share_root_hash")
 339     for k in keys1:
 340         if k in unpacked:
 341             print >>out, "%19s: %s" % (k, unpacked[k])
 342     print >>out
 343     for k in keys2:
 344         if k in unpacked:
 345             print >>out, "%19s: %s" % (k, unpacked[k])
 346     print >>out
 347     for k in keys3:
 348         if k in unpacked:
 349             print >>out, "%19s: %s" % (k, unpacked[k])
 350
 351     leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3)
 352     if leftover:
 353         print >>out
 354         for k in sorted(leftover):
 355             print >>out, "%s: %s" % (k, unpacked[k])
 356
 357     print >>out
 358     return 0
 359
 360 def dump_root_dirnode(basedir, config, out=sys.stdout, err=sys.stderr):
 361     from allmydata import uri
 362
 363     root_dirnode_file = os.path.join(basedir, "vdrive", "root")
 364     try:
 365         f = open(root_dirnode_file, "rb")
 366         key = f.read()
 367         rooturi = uri.pack_dirnode_uri("fakeFURL", key)
 368         print >>out, rooturi
 369         return 0
 370     except EnvironmentError:
 371         print >>out,  "unable to read root dirnode file from %s" % \
 372               root_dirnode_file
 373         return 1
 374
 375 def dump_directory_node(basedir, config, out=sys.stdout, err=sys.stderr):
 376     from allmydata import filetable, vdrive, uri
 377     from allmydata.util import hashutil, idlib
 378     dir_uri = config['uri']
 379     verbose = config['verbose']
 380
 381     furl, key = uri.unpack_dirnode_uri(dir_uri)
 382     if uri.is_mutable_dirnode_uri(dir_uri):
 383         wk, we, rk, index = hashutil.generate_dirnode_keys_from_writekey(key)
 384     else:
 385         wk, we, rk, index = hashutil.generate_dirnode_keys_from_readkey(key)
 386
 387     filename = os.path.join(basedir, "vdrive", idlib.b2a(index))
 388
 389     print >>out
 390     print >>out, "dirnode uri: %s" % dir_uri
 391     print >>out, "filename : %s" % filename
 392     print >>out, "index        : %s" % idlib.b2a(index)
 393     if wk:
 394         print >>out, "writekey     : %s" % idlib.b2a(wk)
 395         print >>out, "write_enabler: %s" % idlib.b2a(we)
 396     else:
 397         print >>out, "writekey     : None"
 398         print >>out, "write_enabler: None"
 399     print >>out, "readkey      : %s" % idlib.b2a(rk)
 400
 401     print >>out
 402
 403     vds = filetable.VirtualDriveServer(os.path.join(basedir, "vdrive"), False)
 404     data = vds._read_from_file(index)
 405     if we:
 406         if we != data[0]:
 407             print >>out, "ERROR: write_enabler does not match"
 408
 409     for (H_key, E_key, E_write, E_read) in data[1]:
 410         if verbose:
 411             print >>out, " H_key %s" % idlib.b2a(H_key)
 412             print >>out, " E_key %s" % idlib.b2a(E_key)
 413             print >>out, " E_write %s" % idlib.b2a(E_write)
 414             print >>out, " E_read %s" % idlib.b2a(E_read)
 415         key = vdrive.decrypt(rk, E_key)
 416         print >>out, " key %s" % key
 417         if hashutil.dir_name_hash(rk, key) != H_key:
 418             print >>out, "  ERROR: H_key does not match"
 419         if wk and E_write:
 420             if len(E_write) < 14:
 421                 print >>out, "  ERROR: write data is short:", idlib.b2a(E_write)
 422             write = vdrive.decrypt(wk, E_write)
 423             print >>out, "   write: %s" % write
 424         read = vdrive.decrypt(rk, E_read)
 425         print >>out, "   read: %s" % read
 426         print >>out
 427
 428     return 0