From f7d2fcc233d4ec025cc71395f863c66531c7b237 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 1 Aug 2008 19:27:29 -0700 Subject: [PATCH] CLI: change one-arg forms of 'tahoe put' to make an unlinked file, fix replace-mutable #441 --- docs/CLI.txt | 39 ++++++- src/allmydata/scripts/cli.py | 32 +++--- src/allmydata/scripts/common.py | 1 + src/allmydata/scripts/tahoe_put.py | 25 ++++- src/allmydata/test/test_cli.py | 167 +++++++++++++++++++++++++++-- src/allmydata/test/test_system.py | 4 +- 6 files changed, 235 insertions(+), 33 deletions(-) diff --git a/docs/CLI.txt b/docs/CLI.txt index a4f76a74..0f61dcb8 100644 --- a/docs/CLI.txt +++ b/docs/CLI.txt @@ -178,8 +178,12 @@ tahoe list-aliases tahoe mkdir tahoe mkdir [alias:]path tahoe ls [alias:][path] -tahoe put [localfrom:-] -tahoe put [localfrom:-] [alias:]to +tahoe put [--mutable] [localfrom:-] +tahoe put [--mutable] [localfrom:-] [alias:]to +tahoe put [--mutable] [localfrom:-] [alias:]subdir/to +tahoe put [--mutable] [localfrom:-] dircap:to +tahoe put [--mutable] [localfrom:-] dircap:./subdir/to +tahoe put [localfrom:-] mutable-file-writecap tahoe get [alias:]from [localto:-] tahoe cp [-r] [alias:]frompath [alias:]topath tahoe rm [alias:]what @@ -235,7 +239,14 @@ tahoe put /tmp/file.txt tahoe put ~/file.txt These upload the local file into the grid, and prints the new read-cap to - stdout. The uploaded file is not attached to any directory. + stdout. The uploaded file is not attached to any directory. All one-argument + forms of "tahoe put" perform an unlinked upload. + +tahoe put - +tahoe put + + These also perform an unlinked upload, but the data to be uploaded is taken + from stdin. tahoe put file.txt uploaded.txt tahoe put file.txt tahoe:uploaded.txt @@ -243,6 +254,28 @@ tahoe put file.txt tahoe:uploaded.txt These upload the local file and add it to your root with the name "uploaded.txt" +tahoe put file.txt subdir/foo.txt +tahoe put - subdir/foo.txt +tahoe put file.txt tahoe:subdir/foo.txt +tahoe put file.txt DIRCAP:./foo.txt +tahoe put file.txt DIRCAP:./subdir/foo.txt + + These upload the named file and attach them to a subdirectory of the given + root directory, under the name "foo.txt". Note that to use a directory + write-cap instead of an alias, you must use ":./" as a separator, rather + than ":", to help the CLI parser figure out where the dircap ends. When the + source file is named "-", the contents are taken from stdin. + +tahoe put file.txt --mutable + + Create a new mutable file, fill it with the contents of file.txt, and print + the new write-cap to stdout. + +tahoe put file.txt MUTABLE-FILE-WRITECAP + + Replace the contents of the given mutable file with the contents of file.txt + and prints the same write-cap to stdout. + tahoe cp file.txt tahoe:uploaded.txt tahoe cp file.txt tahoe: tahoe cp file.txt tahoe:/ diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 42388f35..5c207ee2 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -113,8 +113,9 @@ class PutOptions(VDriveOptions): def parseArgs(self, arg1=None, arg2=None): # cat FILE > tahoe put # create unlinked file from stdin - # cat FILE > tahoe put FOO # create tahoe:FOO from stdin - # cat FILE > tahoe put tahoe:FOO # same + # cat FILE > tahoe put - # same + # tahoe put bar # create unlinked file from local 'bar' + # cat FILE > tahoe put - FOO # create tahoe:FOO from stdin # tahoe put bar FOO # copy local 'bar' to tahoe:FOO # tahoe put bar tahoe:FOO # same @@ -122,11 +123,11 @@ class PutOptions(VDriveOptions): self.from_file = arg1 self.to_file = arg2 elif arg1 is not None and arg2 is None: - self.from_file = None - self.to_file = arg1 + self.from_file = arg1 # might be "-" + self.to_file = None else: - self.from_file = arg1 - self.to_file = arg2 + self.from_file = None + self.to_file = None if self.from_file == "-": self.from_file = None @@ -134,19 +135,22 @@ class PutOptions(VDriveOptions): return "%s put LOCAL_FILE VDRIVE_FILE" % (os.path.basename(sys.argv[0]),) longdesc = """Put a file into the virtual drive (copying the file's - contents from the local filesystem). If LOCAL_FILE is missing or '-', - data will be copied from stdin. VDRIVE_FILE is assumed to start with - tahoe: unless otherwise specified.""" + contents from the local filesystem). If VDRIVE_FILE is missing, upload + the file but do not link it into a directory: prints the new filecap to + stdout. If LOCAL_FILE is missing or '-', data will be copied from stdin. + VDRIVE_FILE is assumed to start with tahoe: unless otherwise specified.""" def getUsage(self, width=None): t = VDriveOptions.getUsage(self, width) t += """ Examples: - % cat FILE > tahoe put # create unlinked file from stdin - % cat FILE > tahoe put FOO # create tahoe:FOO from stdin - % cat FILE > tahoe put tahoe:FOO # same - % tahoe put bar FOO # copy local 'bar' to tahoe:FOO - % tahoe put bar tahoe:FOO # same + % cat FILE > tahoe put # create unlinked file from stdin + % cat FILE > tahoe - # same + % tahoe put bar # create unlinked file from local 'bar' + % cat FILE > tahoe put - FOO # create tahoe:FOO from stdin + % tahoe put bar FOO # copy local 'bar' to tahoe:FOO + % tahoe put bar tahoe:FOO # same + % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place """ return t diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 41c1fccf..54b195cd 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -104,6 +104,7 @@ def get_alias(aliases, path, default): # If default=None, then an empty alias is indicated by returning # DefaultAliasMarker. We special-case "URI:" to make it easy to access # specific files/directories by their read-cap. + path = path.strip() if path.startswith("URI:"): # The only way to get a sub-path is to use URI:blah:./foo, and we # strip out the :./ sequence. diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index 25787693..82c8a608 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -1,5 +1,6 @@ from cStringIO import StringIO +import os.path import urllib from allmydata.scripts.common_http import do_http from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path @@ -26,16 +27,30 @@ def put(options): if nodeurl[-1] != "/": nodeurl += "/" if to_file: - rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS) - url = nodeurl + "uri/%s/" % urllib.quote(rootcap) - if path: - url += escape_path(path) + # several possibilities for the TO_FILE argument. + # : unlinked upload + # foo : TAHOE_ALIAS/foo + # subdir/foo : TAHOE_ALIAS/subdir/foo + # ALIAS:foo : aliases[ALIAS]/foo + # ALIAS:subdir/foo : aliases[ALIAS]/subdir/foo + # DIRCAP:./foo : DIRCAP/foo + # DIRCAP:./subdir/foo : DIRCAP/subdir/foo + # MUTABLE-FILE-WRITECAP : filecap + + if to_file.startswith("URI:SSK:"): + url = nodeurl + "uri/%s" % urllib.quote(to_file) + else: + rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS) + url = nodeurl + "uri/%s/" % urllib.quote(rootcap) + if path: + url += escape_path(path) else: + # unlinked upload url = nodeurl + "uri" if mutable: url += "?mutable=true" if from_file: - infileobj = open(from_file, "rb") + infileobj = open(os.path.expanduser(from_file), "rb") else: # do_http() can't use stdin directly: for one thing, we need a # Content-Length field. So we currently must copy it. diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py index 15b230c8..fb59478b 100644 --- a/src/allmydata/test/test_cli.py +++ b/src/allmydata/test/test_cli.py @@ -264,7 +264,10 @@ class Put(SystemTestMixin, unittest.TestCase): d.addCallback(_done) return d - def test_put_immutable(self): + def test_unlinked_immutable_stdin(self): + # tahoe get `echo DATA | tahoe put` + # tahoe get `echo DATA | tahoe put -` + self.basedir = self.mktemp() DATA = "data" * 100 d = self.set_up_nodes() @@ -273,23 +276,153 @@ class Put(SystemTestMixin, unittest.TestCase): (stdout, stderr) = res self.failUnlessEqual(stderr, "waiting for file data on stdin..\n200 OK\n") - readcap = stdout - self.failUnless(readcap.startswith("URI:CHK:")) - return readcap + self.readcap = stdout + self.failUnless(self.readcap.startswith("URI:CHK:")) d.addCallback(_uploaded) - d.addCallback(lambda readcap: self.do_cli("get", readcap)) + d.addCallback(lambda res: self.do_cli("get", self.readcap)) def _downloaded(res): (stdout, stderr) = res self.failUnlessEqual(stderr, "") self.failUnlessEqual(stdout, DATA) d.addCallback(_downloaded) + d.addCallback(lambda res: self.do_cli("put", "-", stdin=DATA)) + d.addCallback(lambda (stdout,stderr): + self.failUnlessEqual(stdout, self.readcap)) return d - def test_put_mutable(self): - self.basedir = self.mktemp() + def test_unlinked_immutable_from_file(self): + # tahoe put file.txt + # tahoe put ./file.txt + # tahoe put /tmp/file.txt + # tahoe put ~/file.txt + self.basedir = os.path.dirname(self.mktemp()) + # this will be "allmydata.test.test_cli/Put/test_put_from_file/RANDOM" + # and the RANDOM directory will exist. Raw mktemp returns a filename. + + rel_fn = os.path.join(self.basedir, "DATAFILE") + abs_fn = os.path.abspath(rel_fn) + # we make the file small enough to fit in a LIT file, for speed + f = open(rel_fn, "w") + f.write("short file\n") + f.close() + d = self.set_up_nodes() + d.addCallback(lambda res: self.do_cli("put", rel_fn)) + def _uploaded((stdout,stderr)): + readcap = stdout + self.failUnless(readcap.startswith("URI:LIT:")) + self.readcap = readcap + d.addCallback(_uploaded) + d.addCallback(lambda res: self.do_cli("put", "./" + rel_fn)) + d.addCallback(lambda (stdout,stderr): + self.failUnlessEqual(stdout, self.readcap)) + d.addCallback(lambda res: self.do_cli("put", abs_fn)) + d.addCallback(lambda (stdout,stderr): + self.failUnlessEqual(stdout, self.readcap)) + # we just have to assume that ~ is handled properly + return d + + def test_immutable_from_file(self): + # tahoe put file.txt uploaded.txt + # tahoe - uploaded.txt + # tahoe put file.txt subdir/uploaded.txt + # tahoe put file.txt tahoe:uploaded.txt + # tahoe put file.txt tahoe:subdir/uploaded.txt + # tahoe put file.txt DIRCAP:./uploaded.txt + # tahoe put file.txt DIRCAP:./subdir/uploaded.txt + self.basedir = os.path.dirname(self.mktemp()) + + rel_fn = os.path.join(self.basedir, "DATAFILE") + abs_fn = os.path.abspath(rel_fn) + # we make the file small enough to fit in a LIT file, for speed + DATA = "short file\n" + DATA2 = "short file two\n" + f = open(rel_fn, "w") + f.write(DATA) + f.close() + + d = self.set_up_nodes() + d.addCallback(lambda res: self.do_cli("create-alias", "tahoe")) + + d.addCallback(lambda res: + self.do_cli("put", rel_fn, "uploaded.txt")) + def _uploaded((stdout,stderr)): + readcap = stdout.strip() + self.failUnless(readcap.startswith("URI:LIT:")) + self.failUnless("201 Created" in stderr, stderr) + self.readcap = readcap + d.addCallback(_uploaded) + d.addCallback(lambda res: + self.do_cli("get", "tahoe:uploaded.txt")) + d.addCallback(lambda (stdout,stderr): + self.failUnlessEqual(stdout, DATA)) + + d.addCallback(lambda res: + self.do_cli("put", "-", "uploaded.txt", stdin=DATA2)) + def _replaced((stdout,stderr)): + readcap = stdout.strip() + self.failUnless(readcap.startswith("URI:LIT:")) + self.failUnless("200 OK" in stderr, stderr) + d.addCallback(_replaced) + + d.addCallback(lambda res: + self.do_cli("put", rel_fn, "subdir/uploaded2.txt")) + d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt")) + d.addCallback(lambda (stdout,stderr): + self.failUnlessEqual(stdout, DATA)) + + d.addCallback(lambda res: + self.do_cli("put", rel_fn, "tahoe:uploaded3.txt")) + d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt")) + d.addCallback(lambda (stdout,stderr): + self.failUnlessEqual(stdout, DATA)) + + d.addCallback(lambda res: + self.do_cli("put", rel_fn, "tahoe:subdir/uploaded4.txt")) + d.addCallback(lambda res: + self.do_cli("get", "tahoe:subdir/uploaded4.txt")) + d.addCallback(lambda (stdout,stderr): + self.failUnlessEqual(stdout, DATA)) + + def _get_dircap(res): + self.dircap = get_aliases(self.getdir("client0"))["tahoe"] + d.addCallback(_get_dircap) + + d.addCallback(lambda res: + self.do_cli("put", rel_fn, + self.dircap+":./uploaded5.txt")) + d.addCallback(lambda res: + self.do_cli("get", "tahoe:uploaded5.txt")) + d.addCallback(lambda (stdout,stderr): + self.failUnlessEqual(stdout, DATA)) + + d.addCallback(lambda res: + self.do_cli("put", rel_fn, + self.dircap+":./subdir/uploaded6.txt")) + d.addCallback(lambda res: + self.do_cli("get", "tahoe:subdir/uploaded6.txt")) + d.addCallback(lambda (stdout,stderr): + self.failUnlessEqual(stdout, DATA)) + + return d + + def test_mutable_unlinked(self): + # FILECAP = `echo DATA | tahoe put --mutable` + # tahoe get FILECAP, compare against DATA + # echo DATA2 | tahoe put - FILECAP + # tahoe get FILECAP, compare against DATA2 + # tahoe put file.txt FILECAP + self.basedir = os.path.dirname(self.mktemp()) DATA = "data" * 100 DATA2 = "two" * 100 + rel_fn = os.path.join(self.basedir, "DATAFILE") + abs_fn = os.path.abspath(rel_fn) + DATA3 = "three" * 100 + f = open(rel_fn, "w") + f.write(DATA3) + f.close() + d = self.set_up_nodes() + d.addCallback(lambda res: self.do_cli("put", "--mutable", stdin=DATA)) def _created(res): (stdout, stderr) = res @@ -300,7 +433,8 @@ class Put(SystemTestMixin, unittest.TestCase): d.addCallback(_created) d.addCallback(lambda res: self.do_cli("get", self.filecap)) d.addCallback(lambda (out,err): self.failUnlessEqual(out, DATA)) - d.addCallback(lambda res: self.do_cli("put", self.filecap, stdin=DATA2)) + + d.addCallback(lambda res: self.do_cli("put", "-", self.filecap, stdin=DATA2)) def _replaced(res): (stdout, stderr) = res self.failUnlessEqual(stderr, @@ -309,5 +443,20 @@ class Put(SystemTestMixin, unittest.TestCase): d.addCallback(_replaced) d.addCallback(lambda res: self.do_cli("get", self.filecap)) d.addCallback(lambda (out,err): self.failUnlessEqual(out, DATA2)) + + d.addCallback(lambda res: self.do_cli("put", rel_fn, self.filecap)) + def _replaced2(res): + (stdout, stderr) = res + self.failUnlessEqual(stderr, "200 OK\n") + self.failUnlessEqual(self.filecap, stdout) + d.addCallback(_replaced2) + d.addCallback(lambda res: self.do_cli("get", self.filecap)) + d.addCallback(lambda (out,err): self.failUnlessEqual(out, DATA3)) + return d - test_put_mutable.todo = "put MUTABLE still fails, ticket #441" + + def test_mutable(self): + # tahoe put --mutable file.txt uploaded.txt + # tahoe put - uploaded.txt # should modify-in-place + pass # TODO + diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 4e8b3687..4125a328 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1439,9 +1439,9 @@ class SystemTest(SystemTestMixin, unittest.TestCase): # tahoe put FOO STDIN_DATA = "This is the file to upload from stdin." - d.addCallback(run, "put", "tahoe-file-stdin", stdin=STDIN_DATA) + d.addCallback(run, "put", "-", "tahoe-file-stdin", stdin=STDIN_DATA) # tahoe put tahoe:FOO - d.addCallback(run, "put", "tahoe:from-stdin", + d.addCallback(run, "put", "-", "tahoe:from-stdin", stdin="Other file from stdin.") d.addCallback(run, "ls") -- 2.37.2