a first crack at the "put" command-line
authorZooko O'Whielacronx <zooko@zooko.com>
Thu, 16 Aug 2007 19:15:38 +0000 (12:15 -0700)
committerZooko O'Whielacronx <zooko@zooko.com>
Thu, 16 Aug 2007 19:15:38 +0000 (12:15 -0700)
There are actually two versions in this patch, one of which requires twisted.web2 and the other of which uses the Python standard library's socket module.  The socketish one doesn't know when the web server is done so it hangs after doing its thing.  (Oh -- maybe I should add an HTTP header asking the web server to close the connection when finished.)  The web2ish one works better in that respect.  Neither of them handle error responses from the server very well yet.

After lunch I intend to finish the socketish one.

To try one, mv src/allmydata/scripts/tahoe_put-{socketish,web2ish}.py src/allmydata/scripts/tahoe_put.py .

If you want to try the web2ish one, and you can't find a web2 package to install, you can get one from:

http://allmydata.org/~zooko/repos/twistedweb2snarf/

src/allmydata/scripts/cli.py
src/allmydata/scripts/runner.py
src/allmydata/scripts/tahoe_put-socketish.py [new file with mode: 0644]
src/allmydata/scripts/tahoe_put-web2ish.py [new file with mode: 0644]

index d9a079d9acb56753e44be6a789c9aca1ef76cf6b..ef14a2f557590975730bb65eb7ded0210a7b9b79 100644 (file)
@@ -2,7 +2,13 @@
 import os.path, sys
 from twisted.python import usage
 
-class VDriveOptions(usage.Options):
+class OptionsMixin(usage.Options):
+    optFlags = [
+        ["quiet", "q", "Operate silently."],
+        ["version", "V", "Display version numbers and exit."],
+        ]
+
+class VDriveOptions(OptionsMixin):
     optParameters = [
         ["vdrive", "d", "global",
          "which virtual drive to use: 'global' or 'private'"],
@@ -24,13 +30,27 @@ class GetOptions(VDriveOptions):
         return "%s get VDRIVE_FILE LOCAL_FILE" % (os.path.basename(sys.argv[0]),)
 
     longdesc = """Retrieve a file from the virtual drive and write it to the
-    local disk. If LOCAL_FILE is omitted or '-', the contents of the file
+    local filesystem. If LOCAL_FILE is omitted or '-', the contents of the file
     will be written to stdout."""
 
+class PutOptions(VDriveOptions):
+    def parseArgs(self, local_filename, vdrive_filename):
+        self['vdrive_filename'] = vdrive_filename
+        self['local_filename'] = local_filename
+
+    def getSynopsis(self):
+        return "%s put LOCAL_FILEVDRI VE_FILE" % (os.path.basename(sys.argv[0]),)
+
+    longdesc = """Put a file into the virtual drive (copying the file's
+    contents from the local filesystem). If LOCAL_FILE is omitted or '-', the
+    contents of the file will be read from stdin."""
+
+
 
 subCommands = [
     ["ls", None, ListOptions, "List a directory"],
     ["get", None, GetOptions, "Retrieve a file from the virtual drive."],
+    ["put", None, PutOptions, "Upload a file into the virtual drive."],
     ]
 
 def list(config, stdout, stderr):
@@ -59,8 +79,24 @@ def get(config, stdout, stderr):
                   (vdrive_filename, local_filename)
     return rc
 
+def put(config, stdout, stderr):
+    from allmydata.scripts import tahoe_put
+    vdrive_filename = config['vdrive_filename']
+    local_filename = config['local_filename']
+    if config['quiet']:
+        verbosity = 0
+    else:
+        verbosity = 2
+    rc = tahoe_put.put(config['server'],
+                       config['vdrive'],
+                       vdrive_filename,
+                       local_filename,
+                       verbosity)
+    return rc
+
 dispatch = {
     "ls": list,
     "get": get,
+    "put": put,
     }
 
index 1b6af812d8f1d1349b7e0ee0e253b51839561639..ad2f8fa4755dbe48550f7ad9081b0955dba992c1 100644 (file)
@@ -3,16 +3,11 @@ import sys
 from cStringIO import StringIO
 from twisted.python import usage
 
-from allmydata.scripts import debug, create_node, startstop_node, cli
+import debug, create_node, startstop_node, cli
 
-class Options(usage.Options):
+class Options(cli.OptionsMixin):
     synopsis = "Usage:  allmydata <command> [command options]"
 
-    optFlags = [
-        ["quiet", "q", "Operate silently."],
-        ["version", "V", "Display version numbers and exit."],
-        ]
-
     subCommands = []
     subCommands += create_node.subCommands
     subCommands += startstop_node.subCommands
diff --git a/src/allmydata/scripts/tahoe_put-socketish.py b/src/allmydata/scripts/tahoe_put-socketish.py
new file mode 100644 (file)
index 0000000..7c912d7
--- /dev/null
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+import re, socket, sys
+
+SERVERURL_RE=re.compile("http://([^:]*)(:([1-9][0-9]*))?")
+
+def put(serverurl, vdrive, vdrive_fname, local_fname, verbosity):
+    """
+    @param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose
+
+    @return: a Deferred which eventually fires with the exit code
+    """
+    mo = SERVERURL_RE.match(serverurl)
+    if not mo:
+        raise ValueError("serverurl is required to look like \"http://HOSTNAMEORADDR:PORT\"")
+    host = mo.group(1)
+    port = int(mo.group(3))
+
+    url = "/vdrive/" + vdrive + "/"
+    if vdrive_fname:
+        url += vdrive_fname
+
+    if local_fname is None or local_fname == "-":
+        infileobj = sys.stdin
+    else:
+        infileobj = open(local_fname, "rb")
+
+    so = socket.socket()
+    so.connect((host, port,))
+
+    CHUNKSIZE=2**16
+    data = "PUT %s HTTP/1.1\r\nHostname: %s\r\n\r\n" % (url, host,)
+    while data:
+        try:
+            sent = so.send(data)
+            print "XXXXXXXX I just sent %s" % (data[:sent],)
+        except Exception, le:
+            print "BOOOOO le: %r" % (le,)
+            return -1
+
+        if sent == len(data):
+            data = infileobj.read(CHUNKSIZE)
+        else:
+            data = data[sent:]
+
+    respbuf = []
+    data = so.recv(CHUNKSIZE)
+    while data:
+        print "WHEEEEE okay now we've got some more data: %r" % (data,)
+        respbuf.append(data)
+        data = so.recv(CHUNKSIZE)
+
+    so.shutdown(socket.SHUT_WR)
+    data = so.recv(CHUNKSIZE)
+    while data:
+        print "WHEEEEE 22222 okay now we've got some more data: %r" % (data,)
+        respbuf.append(data)
+        data = so.recv(CHUNKSIZE)
+
+def main():
+    import optparse
+    parser = optparse.OptionParser()
+    parser.add_option("-d", "--vdrive", dest="vdrive", default="global")
+    parser.add_option("-s", "--server", dest="server", default="http://tahoebs1.allmydata.com:8011")
+
+    (options, args) = parser.parse_args()
+
+    local_file = args[0]
+    vdrive_file = None
+    if len(args) > 1:
+        vdrive_file = args[1]
+
+    put(options.server, options.vdrive, vdrive_file, local_file)
+
+if __name__ == '__main__':
+    main()
diff --git a/src/allmydata/scripts/tahoe_put-web2ish.py b/src/allmydata/scripts/tahoe_put-web2ish.py
new file mode 100644 (file)
index 0000000..9f6cc11
--- /dev/null
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+import re, sys
+
+from twisted import web2 
+from twisted.web2 import client, http, stream
+import twisted.web2.client.http
+
+from twisted.internet import defer, reactor, protocol
+
+SERVERURL_RE=re.compile("http://([^:]*)(:([1-9][0-9]*))?")
+
+def _put(serverurl, vdrive, vdrive_fname, local_fname, verbosity):
+    """
+    @param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose
+
+    @return: a Deferred which eventually fires with the exit code
+    """
+    mo = SERVERURL_RE.match(serverurl)
+    if not mo:
+        raise ValueError("serverurl is required to look like \"http://HOSTNAMEORADDR:PORT\"")
+    host = mo.group(1)
+    port = int(mo.group(3))
+
+    d = defer.Deferred()
+    
+    url = "/vdrive/" + vdrive + "/"
+    if vdrive_fname:
+        url += vdrive_fname
+
+    if local_fname is None or local_fname == "-":
+        infileobj = sys.stdin
+    else:
+        infileobj = open(local_fname, "rb")
+    instream = web2.stream.FileStream(infileobj)
+    
+    d2 = protocol.ClientCreator(reactor, web2.client.http.HTTPClientProtocol).connectTCP(host, port)
+
+    def got_resp(resp):
+        # If this isn't a 200 or 201, then write out the response data and
+        # exit with resp.code as our exit value.
+        if resp.code not in (200, 201,):
+            def writeit(data):
+                sys.stdout.write(data)
+
+            def exit(dummy):
+                d.errback(resp.code)
+
+            return web2.http.stream.readStream(resp.stream, writeit).addCallback(exit)
+
+        # If we are in quiet mode, then just exit with the resp.code.
+        if verbosity == 0:
+            d.callback(resp.code)
+            return
+
+        # Else, this is a successful request and we are not in quiet mode:
+        uribuffer = []
+        def gather_uri(data):
+            uribuffer.append(data)
+
+        def output_result(thingie):
+            uri = ''.join(uribuffer)
+            outbuf = []
+            if resp.code == 200:
+                outbuf.append("200 (OK); ")
+            elif resp.code == 201:
+                outbuf.append("201 (Created); ")
+             
+            if verbosity == 2:
+                if resp.code == 200:
+                    outbuf.append("modified existing mapping in vdrive %s of name %s to point to " % (vdrive, vdrive_fname,))
+                elif resp.code == 201:
+                    outbuf.append("created new mapping in vdrive %s of name %s to point to " % (vdrive, vdrive_fname,))
+
+            outbuf.append("URI: %s" % (uri,))
+        
+            sys.stdout.write(''.join(outbuf))
+
+            d.callback(resp.code)
+
+        web2.http.stream.readStream(resp.stream, gather_uri).addCallback(output_result)
+                
+    def send_req(proto):
+        proto.submitRequest(web2.client.http.ClientRequest('PUT', url, {}, instream)).addCallback(got_resp)
+
+    d2.addCallback(send_req)
+
+    return d
+
+def put(server, vdrive, vdrive_fname, local_fname, verbosity):
+    """
+    This starts the reactor, does the PUT command, waits for the result, stops
+    the reactor, and returns the exit code. 
+
+    @param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose
+
+    @return: the exit code
+    """
+    d = _put(server, vdrive, vdrive_fname, local_fname, verbosity)
+    exitcode = [ None ]
+    def exit(result):
+        exitcode[0] = result
+        reactor.stop()
+        return result
+        
+    d.addCallbacks(exit, exit)
+    reactor.run()
+    return exitcode[0]
+