From bf0280d0f87b690a70cc624b014ebb366d9d80a6 Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Fri, 10 Jul 2015 06:13:44 +0100 Subject: [PATCH] Add the 'tahoe admin ls-container' command and tests. fixes #1759 Signed-off-by: Daira Hopwood --- docs/frontends/CLI.rst | 7 ++- src/allmydata/scripts/admin.py | 65 ++++++++++++++++++++ src/allmydata/test/test_cli.py | 4 ++ src/allmydata/test/test_storage.py | 97 ++++++++++++++++++++++++++++-- 4 files changed, 165 insertions(+), 8 deletions(-) diff --git a/docs/frontends/CLI.rst b/docs/frontends/CLI.rst index 385dccc8..9884861b 100644 --- a/docs/frontends/CLI.rst +++ b/docs/frontends/CLI.rst @@ -139,9 +139,10 @@ is most often used by developers who have just modified the code and want to start using their changes. Some less frequently used administration commands, for key generation/derivation -and for creating a cloud backend container, are grouped as subcommands of -"``tahoe admin``". For a list of these use "``tahoe admin --help``", or for -more detailed help on a particular command, use "``tahoe admin COMMAND --help``". +and for creating and listing the contents of cloud backend containers, are +grouped as subcommands of "``tahoe admin``". For a list of these use +"``tahoe admin --help``", or for more detailed help on a particular command, +use "``tahoe admin COMMAND --help``". Filesystem Manipulation diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 9d255198..a68ad223 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -2,6 +2,7 @@ import os from twisted.python import usage +from allmydata.util.encodingutil import quote_output from allmydata.scripts.common import BaseOptions, BasedirOptions class GenerateKeypairOptions(BaseOptions): @@ -99,6 +100,67 @@ def do_create_container(options): return d +class ListContainerOptions(BasedirOptions): + def getSynopsis(self): + return "Usage: %s [global-opts] admin ls-container [NODEDIR]" % (self.command_name,) + + def getUsage(self, width=None): + t = BasedirOptions.getUsage(self, width) + t += """ +List the contents of a storage container, using the name and credentials +configured in tahoe.cfg. This currently works only for the cloud backend. +""" + return t + +def ls_container(options): + from twisted.internet import reactor, defer + + d = defer.maybeDeferred(do_ls_container, options) + d.addCallbacks(lambda ign: os._exit(0), lambda ign: os._exit(1)) + reactor.run() + +def format_date(date): + datestr = str(date) + if datestr.endswith('+00:00'): + datestr = datestr[: -6] + 'Z' + return datestr + +def do_ls_container(options): + from twisted.internet import defer + from allmydata.node import ConfigOnly + from allmydata.client import Client + + out = options.stdout + err = options.stderr + + d = defer.succeed(None) + def _do_create(ign): + config = ConfigOnly(options['basedir']) + if not config.get_config("storage", "enabled", True, boolean=True): + raise AssertionError("'tahoe admin ls-container' is intended for administration of nodes running a storage service.\n" + "The node with base directory %s is not configured to provide storage." + % quote_output(options['basedir'])) + + (backend, _) = Client.configure_backend(config) + + d2 = backend.list_container() + def _done(items): + print >>out, "Listing %d object(s):" % len(items) + print >>out, " Size Last modified Key" + for item in items: + print >>out, "% 8s %20s %s" % (item.size, format_date(item.modification_date), item.key) + d2.addCallback(_done) + return d2 + d.addCallback(_do_create) + def _failed(f): + print >>err, "Container listing failed." + print >>err, "%s: %s" % (f.value.__class__.__name__, f.value) + print >>err + return f + d.addErrback(_failed) + return d + + class AdminCommand(BaseOptions): subCommands = [ ("generate-keypair", None, GenerateKeypairOptions, @@ -107,6 +169,8 @@ class AdminCommand(BaseOptions): "Derive a public key from a private key."), ("create-container", None, CreateContainerOptions, "Create a container for the configured cloud backend."), + ("ls-container", None, ListContainerOptions, + "List the contents of the configured backend container."), ] def postOptions(self): if not hasattr(self, 'subOptions'): @@ -125,6 +189,7 @@ subDispatch = { "generate-keypair": print_keypair, "derive-pubkey": derive_pubkey, "create-container": create_container, + "ls-container": ls_container, } def do_admin(options): diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py index b51d5185..bc0a19d8 100644 --- a/src/allmydata/test/test_cli.py +++ b/src/allmydata/test/test_cli.py @@ -623,6 +623,10 @@ class Help(unittest.TestCase): help = str(admin.CreateContainerOptions()) self.failUnlessIn(" [global-options] admin create-container [NODEDIR]", help) + def test_create_admin_ls_container(self): + help = str(admin.ListContainerOptions()) + self.failUnlessIn(" [global-options] admin ls-container [NODEDIR]", help) + class Ln(GridTestMixin, CLITestMixin, unittest.TestCase): def _create_test_file(self): diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index cee8be07..c800d4e2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -57,7 +57,7 @@ from allmydata.test.common_util import ReallyEqualMixin from allmydata.test.common_web import WebRenderingMixin from allmydata.test.no_network import NoNetworkServer from allmydata.test.test_cli import parse_options -from allmydata.scripts.admin import do_create_container +from allmydata.scripts.admin import do_create_container, do_ls_container from allmydata.web.storage import StorageStatus, remove_prefix @@ -1534,11 +1534,11 @@ class Namespace(object): pass -class CreateContainer(unittest.TestCase, WorkdirMixin): - def test_create_container(self): +class AdminContainerTests(unittest.TestCase, WorkdirMixin): + def test_admin_create_container(self): # We'll use the mock cloud backend to test this. - basedir = self.workdir("test_create_container") - os.makedirs(basedir) + basedir = self.workdir("test_admin_create_container") + fileutil.make_dirs(basedir) fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[client]\n" "introducer.furl = \n" @@ -1603,6 +1603,93 @@ class CreateContainer(unittest.TestCase, WorkdirMixin): d.addCallback(_check_failure) return d + def test_admin_ls_container(self): + self.patch(cloud_common, 'BACKOFF_SECONDS_BEFORE_RETRY', (0, 0.1, 0.2)) + + def _set_up(ign, basedir, storage_cfg): + self.basedir = basedir + fileutil.make_dirs(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + "[client]\n" + "introducer.furl = \n" + "[storage]\n" + + storage_cfg) + + def _run_ls_container(ign): + # We're really only testing do_ls_container (to avoid problems with + # restarting the reactor or exiting), but that should be sufficient. + + options = parse_options(self.basedir, "admin", ["ls-container"]) + options.stdout = StringIO() + options.stderr = StringIO() + d = defer.maybeDeferred(do_ls_container, options) + d.addCallbacks(lambda ign: 0, lambda ign: 1) + d.addCallback(lambda rc: (options.stdout.getvalue(), options.stderr.getvalue(), rc)) + return d + + workdir = self.workdir("test_admin_ls_container") + d = defer.succeed(None) + + d.addCallback(_set_up, os.path.join(workdir, "no_storage"), + "enabled = false\n") + d.addCallback(_run_ls_container) + def _check_no_storage(res): + (out, err, rc) = res + self.failUnlessEqual(out, "", str(res)) + self.failUnlessIn("Container listing failed.", err, str(res)) + self.failUnlessIn("'tahoe admin ls-container' is intended for administration of nodes running a storage service", + err, str(res)) + self.failUnlessEqual(rc, 1, str(res)) + d.addCallback(_check_no_storage) + + d.addCallback(_set_up, os.path.join(workdir, "disk"), + "enabled = true\n") + d.addCallback(_run_ls_container) + def _check_disk(res): + (out, err, rc) = res + self.failUnlessEqual(out, "", str(res)) + self.failUnlessIn("Container listing failed.", err, str(res)) + self.failUnlessIn("the disk backend does not support listing container contents", err, str(res)) + self.failUnlessIn("tahoe debug catalog-shares", err, str(res)) + self.failUnlessEqual(rc, 1, str(res)) + d.addCallback(_check_disk) + + d.addCallback(_set_up, os.path.join(workdir, "no_objects"), + "enabled = true\n" + "backend = mock_cloud\n") + d.addCallback(_run_ls_container) + def _check_no_objects(res): + (out, err, rc) = res + out_lines = out.split('\n') + self.failUnlessEqual(out_lines[0], "Listing 0 object(s):", str(res)) + self.failUnless(re.match(r'^\s*Size\s+Last modified\s+Key$', out_lines[1]), str(res)) + self.failUnlessEqual(out_lines[2], "", str(res)) + self.failUnlessEqual(len(out_lines), 3) + self.failUnlessEqual(err, "", str(res)) + self.failUnlessEqual(rc, 0, str(res)) + d.addCallback(_check_no_objects) + + d.addCallback(_set_up, os.path.join(workdir, "one_object"), + "enabled = true\n" + "backend = mock_cloud\n") + def _create_object(ign): + prefixdir = os.path.join(self.basedir, "storage", "shares", "fo", "foo") + fileutil.make_dirs(prefixdir) + fileutil.write(os.path.join(prefixdir, "0"), "0123456789") + d.addCallback(_create_object) + d.addCallback(_run_ls_container) + def _check_one_object(res): + (out, err, rc) = res + out_lines = out.split('\n') + self.failUnlessEqual(out_lines[0], "Listing 1 object(s):", str(res)) + self.failUnless(re.match(r'^\s*Size\s+Last modified\s+Key$', out_lines[1]), str(res)) + self.failUnless(re.match(r'^\s*10\s+\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d*)?Z\s+shares/fo/foo/0$', out_lines[2]), str(res)) + self.failUnlessEqual(len(out_lines), 4) + self.failUnlessEqual(err, "", str(res)) + self.failUnlessEqual(rc, 0, str(res)) + d.addCallback(_check_one_object) + return d + class ServerMixin: def allocate(self, account, storage_index, sharenums, size, canary=None): -- 2.45.2