Add the 'tahoe admin ls-container' command and tests. fixes #1759
authorDaira Hopwood <daira@jacaranda.org>
Tue, 8 Apr 2014 21:43:39 +0000 (22:43 +0100)
committerDaira Hopwood <daira@jacaranda.org>
Fri, 17 Apr 2015 21:31:41 +0000 (22:31 +0100)
Signed-off-by: Daira Hopwood <daira@jacaranda.org>
docs/frontends/CLI.rst
src/allmydata/scripts/admin.py
src/allmydata/test/test_cli.py
src/allmydata/test/test_storage.py

index f7d291b0cbbdca6875c4af7ec86c07019e661fb2..137cb455f51158ba447bf2da5acd661845ddf4b0 100644 (file)
@@ -139,9 +139,10 @@ is most often used by developers who have just modified the code and want to
 start using their changes.
 
 Some less frequently used administration commands, for key generation/derivation
-and for creating a cloud backend container, are grouped as subcommands of
-"``tahoe admin``". For a list of these use "``tahoe admin --help``", or for
-more detailed help on a particular command, use "``tahoe admin COMMAND --help``".
+and for creating and listing the contents of cloud backend containers, are
+grouped as subcommands of "``tahoe admin``". For a list of these use
+"``tahoe admin --help``", or for more detailed help on a particular command,
+use "``tahoe admin COMMAND --help``".
 
 
 Filesystem Manipulation
index 5d28138e9f977a6d632da197ba4db5b2bcadcb0e..335797483e21fcaf64ef4d4e0aeeb46691a45d77 100644 (file)
@@ -2,6 +2,7 @@
 import os
 
 from twisted.python import usage
+from allmydata.util.encodingutil import quote_output
 from allmydata.scripts.common import BaseOptions, BasedirOptions
 
 class GenerateKeypairOptions(BaseOptions):
@@ -101,6 +102,67 @@ def do_create_container(options):
     return d
 
 
+class ListContainerOptions(BasedirOptions):
+    def getSynopsis(self):
+        return "Usage: %s [global-opts] admin ls-container [NODEDIR]" % (self.command_name,)
+
+    def getUsage(self, width=None):
+        t = BasedirOptions.getUsage(self, width)
+        t += """
+List the contents of a storage container, using the name and credentials
+configured in tahoe.cfg. This currently works only for the cloud backend.
+"""
+        return t
+
+def ls_container(options):
+    from twisted.internet import reactor, defer
+
+    d = defer.maybeDeferred(do_ls_container, options)
+    d.addCallbacks(lambda ign: os._exit(0), lambda ign: os._exit(1))
+    reactor.run()
+
+def format_date(date):
+    datestr = str(date)
+    if datestr.endswith('+00:00'):
+        datestr = datestr[: -6] + 'Z'
+    return datestr
+
+def do_ls_container(options):
+    from twisted.internet import defer
+    from allmydata.node import ConfigOnly
+    from allmydata.client import Client
+
+    out = options.stdout
+    err = options.stderr
+
+    d = defer.succeed(None)
+    def _do_create(ign):
+        config = ConfigOnly(options['basedir'])
+        if not config.get_config("storage", "enabled", True, boolean=True):
+            raise AssertionError("'tahoe admin ls-container' is intended for administration of nodes running a storage service.\n"
+                                 "The node with base directory %s is not configured to provide storage."
+                                 % quote_output(options['basedir']))
+
+        (backend, _) = Client.configure_backend(config)
+
+        d2 = backend.list_container()
+        def _done(items):
+            print >>out, "Listing %d object(s):" % len(items)
+            print >>out, "    Size  Last modified         Key"
+            for item in items:
+                print >>out, "% 8s  %20s  %s" % (item.size, format_date(item.modification_date), item.key)
+        d2.addCallback(_done)
+        return d2
+    d.addCallback(_do_create)
+    def _failed(f):
+        print >>err, "Container listing failed."
+        print >>err, "%s: %s" % (f.value.__class__.__name__, f.value)
+        print >>err
+        return f
+    d.addErrback(_failed)
+    return d
+
+
 class AdminCommand(BaseOptions):
     subCommands = [
         ("generate-keypair", None, GenerateKeypairOptions,
@@ -109,6 +171,8 @@ class AdminCommand(BaseOptions):
          "Derive a public key from a private key."),
         ("create-container", None, CreateContainerOptions,
          "Create a container for the configured cloud backend."),
+        ("ls-container", None, ListContainerOptions,
+         "List the contents of the configured backend container."),
         ]
     def postOptions(self):
         if not hasattr(self, 'subOptions'):
@@ -127,6 +191,7 @@ subDispatch = {
     "generate-keypair": print_keypair,
     "derive-pubkey": derive_pubkey,
     "create-container": create_container,
+    "ls-container": ls_container,
     }
 
 def do_admin(options):
index 36c5bba5a542b3d66f5050dc6f8adfda4c2c1dfd..d1f8f8e060e5b0a02224ce0a4b1fc15c536c6c3a 100644 (file)
@@ -618,6 +618,10 @@ class Help(unittest.TestCase):
         help = str(admin.CreateContainerOptions())
         self.failUnlessIn(" [global-opts] admin create-container [NODEDIR]", help)
 
+    def test_create_admin_ls_container(self):
+        help = str(admin.ListContainerOptions())
+        self.failUnlessIn(" [global-opts] admin ls-container [NODEDIR]", help)
+
 
 class Ln(GridTestMixin, CLITestMixin, unittest.TestCase):
     def _create_test_file(self):
index 558f07b5f9365723717870861b9545d06b389ce3..20c88adb067fa4fd9872b3aa8e0ac30b2fd0e371 100644 (file)
@@ -57,7 +57,7 @@ from allmydata.test.common_util import ReallyEqualMixin
 from allmydata.test.common_web import WebRenderingMixin
 from allmydata.test.no_network import NoNetworkServer
 from allmydata.test.test_cli import parse_options
-from allmydata.scripts.admin import do_create_container
+from allmydata.scripts.admin import do_create_container, do_ls_container
 from allmydata.web.storage import StorageStatus, remove_prefix
 
 
@@ -1534,11 +1534,11 @@ class Namespace(object):
     pass
 
 
-class CreateContainer(unittest.TestCase, WorkdirMixin):
-    def test_create_container(self):
+class AdminContainerTests(unittest.TestCase, WorkdirMixin):
+    def test_admin_create_container(self):
         # We'll use the mock cloud backend to test this.
-        basedir = self.workdir("test_create_container")
-        os.makedirs(basedir)
+        basedir = self.workdir("test_admin_create_container")
+        fileutil.make_dirs(basedir)
         fileutil.write(os.path.join(basedir, "tahoe.cfg"),
                                     "[client]\n"
                                     "introducer.furl = \n"
@@ -1603,6 +1603,93 @@ class CreateContainer(unittest.TestCase, WorkdirMixin):
         d.addCallback(_check_failure)
         return d
 
+    def test_admin_ls_container(self):
+        self.patch(cloud_common, 'BACKOFF_SECONDS_BEFORE_RETRY', (0, 0.1, 0.2))
+
+        def _set_up(ign, basedir, storage_cfg):
+            self.basedir = basedir
+            fileutil.make_dirs(basedir)
+            fileutil.write(os.path.join(basedir, "tahoe.cfg"),
+                           "[client]\n"
+                           "introducer.furl = \n"
+                           "[storage]\n" +
+                           storage_cfg)
+
+        def _run_ls_container(ign):
+            # We're really only testing do_ls_container (to avoid problems with
+            # restarting the reactor or exiting), but that should be sufficient.
+
+            options = parse_options(self.basedir, "admin", ["ls-container"])
+            options.stdout = StringIO()
+            options.stderr = StringIO()
+            d = defer.maybeDeferred(do_ls_container, options)
+            d.addCallbacks(lambda ign: 0, lambda ign: 1)
+            d.addCallback(lambda rc: (options.stdout.getvalue(), options.stderr.getvalue(), rc))
+            return d
+
+        workdir = self.workdir("test_admin_ls_container")
+        d = defer.succeed(None)
+
+        d.addCallback(_set_up, os.path.join(workdir, "no_storage"),
+                               "enabled = false\n")
+        d.addCallback(_run_ls_container)
+        def _check_no_storage(res):
+            (out, err, rc) = res
+            self.failUnlessEqual(out, "", str(res))
+            self.failUnlessIn("Container listing failed.", err, str(res))
+            self.failUnlessIn("'tahoe admin ls-container' is intended for administration of nodes running a storage service",
+                              err, str(res))
+            self.failUnlessEqual(rc, 1, str(res))
+        d.addCallback(_check_no_storage)
+
+        d.addCallback(_set_up, os.path.join(workdir, "disk"),
+                               "enabled = true\n")
+        d.addCallback(_run_ls_container)
+        def _check_disk(res):
+            (out, err, rc) = res
+            self.failUnlessEqual(out, "", str(res))
+            self.failUnlessIn("Container listing failed.", err, str(res))
+            self.failUnlessIn("the disk backend does not support listing container contents", err, str(res))
+            self.failUnlessIn("tahoe debug catalog-shares", err, str(res))
+            self.failUnlessEqual(rc, 1, str(res))
+        d.addCallback(_check_disk)
+
+        d.addCallback(_set_up, os.path.join(workdir, "no_objects"),
+                               "enabled = true\n"
+                               "backend = mock_cloud\n")
+        d.addCallback(_run_ls_container)
+        def _check_no_objects(res):
+            (out, err, rc) = res
+            out_lines = out.split('\n')
+            self.failUnlessEqual(out_lines[0], "Listing 0 object(s):", str(res))
+            self.failUnless(re.match(r'^\s*Size\s+Last modified\s+Key$', out_lines[1]), str(res))
+            self.failUnlessEqual(out_lines[2], "", str(res))
+            self.failUnlessEqual(len(out_lines), 3)
+            self.failUnlessEqual(err, "", str(res))
+            self.failUnlessEqual(rc, 0, str(res))
+        d.addCallback(_check_no_objects)
+
+        d.addCallback(_set_up, os.path.join(workdir, "one_object"),
+                               "enabled = true\n"
+                               "backend = mock_cloud\n")
+        def _create_object(ign):
+            prefixdir = os.path.join(self.basedir, "storage", "shares", "fo", "foo")
+            fileutil.make_dirs(prefixdir)
+            fileutil.write(os.path.join(prefixdir, "0"), "0123456789")
+        d.addCallback(_create_object)
+        d.addCallback(_run_ls_container)
+        def _check_one_object(res):
+            (out, err, rc) = res
+            out_lines = out.split('\n')
+            self.failUnlessEqual(out_lines[0], "Listing 1 object(s):", str(res))
+            self.failUnless(re.match(r'^\s*Size\s+Last modified\s+Key$', out_lines[1]), str(res))
+            self.failUnless(re.match(r'^\s*10\s+\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d*)?Z\s+shares/fo/foo/0$', out_lines[2]), str(res))
+            self.failUnlessEqual(len(out_lines), 4)
+            self.failUnlessEqual(err, "", str(res))
+            self.failUnlessEqual(rc, 0, str(res))
+        d.addCallback(_check_one_object)
+        return d
+
 
 class ServerMixin:
     def allocate(self, account, storage_index, sharenums, size, canary=None):