src/allmydata/test/test_cli_list.py

   1 from twisted.trial import unittest
   2 from twisted.internet import defer
   3
   4 from allmydata.immutable import upload
   5 from allmydata.interfaces import MDMF_VERSION, SDMF_VERSION
   6 from allmydata.mutable.publish import MutableData
   7 from allmydata.test.no_network import GridTestMixin
   8 from allmydata.util.encodingutil import quote_output, get_io_encoding
   9 from .test_cli import CLITestMixin
  10
  11 timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s
  12
  13 class List(GridTestMixin, CLITestMixin, unittest.TestCase):
  14     def test_list(self):
  15         self.basedir = "cli/List/list"
  16         self.set_up_grid()
  17         c0 = self.g.clients[0]
  18         small = "small"
  19
  20         # u"g\u00F6\u00F6d" might not be representable in the argv and/or output encodings.
  21         # It is initially included in the directory in any case.
  22         try:
  23             good_arg = u"g\u00F6\u00F6d".encode(get_io_encoding())
  24         except UnicodeEncodeError:
  25             good_arg = None
  26
  27         try:
  28             good_out = u"g\u00F6\u00F6d".encode(get_io_encoding())
  29         except UnicodeEncodeError:
  30             good_out = None
  31
  32         d = c0.create_dirnode()
  33         def _stash_root_and_create_file(n):
  34             self.rootnode = n
  35             self.rooturi = n.get_uri()
  36             return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence=""))
  37         d.addCallback(_stash_root_and_create_file)
  38         def _stash_goodcap(n):
  39             self.goodcap = n.get_uri()
  40         d.addCallback(_stash_goodcap)
  41         d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"1share"))
  42         d.addCallback(lambda n:
  43                       self.delete_shares_numbered(n.get_uri(), range(1,10)))
  44         d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"0share"))
  45         d.addCallback(lambda n:
  46                       self.delete_shares_numbered(n.get_uri(), range(0,10)))
  47         d.addCallback(lambda ign:
  48                       self.do_cli("add-alias", "tahoe", self.rooturi))
  49         d.addCallback(lambda ign: self.do_cli("ls"))
  50         def _check1((rc,out,err)):
  51             if good_out is None:
  52                 self.failUnlessReallyEqual(rc, 1)
  53                 self.failUnlessIn("files whose names could not be converted", err)
  54                 self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err)
  55                 self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share"]))
  56             else:
  57                 self.failUnlessReallyEqual(rc, 0)
  58                 self.failUnlessReallyEqual(err, "")
  59                 self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", good_out]))
  60         d.addCallback(_check1)
  61         d.addCallback(lambda ign: self.do_cli("ls", "missing"))
  62         def _check2((rc,out,err)):
  63             self.failIfEqual(rc, 0)
  64             self.failUnlessReallyEqual(err.strip(), "No such file or directory")
  65             self.failUnlessReallyEqual(out, "")
  66         d.addCallback(_check2)
  67         d.addCallback(lambda ign: self.do_cli("ls", "1share"))
  68         def _check3((rc,out,err)):
  69             self.failIfEqual(rc, 0)
  70             self.failUnlessIn("Error during GET: 410 Gone", err)
  71             self.failUnlessIn("UnrecoverableFileError:", err)
  72             self.failUnlessIn("could not be retrieved, because there were "
  73                               "insufficient good shares.", err)
  74             self.failUnlessReallyEqual(out, "")
  75         d.addCallback(_check3)
  76         d.addCallback(lambda ign: self.do_cli("ls", "0share"))
  77         d.addCallback(_check3)
  78         def _check4((rc, out, err)):
  79             if good_out is None:
  80                 self.failUnlessReallyEqual(rc, 1)
  81                 self.failUnlessIn("files whose names could not be converted", err)
  82                 self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err)
  83                 self.failUnlessReallyEqual(out, "")
  84             else:
  85                 # listing a file (as dir/filename) should have the edge metadata,
  86                 # including the filename
  87                 self.failUnlessReallyEqual(rc, 0)
  88                 self.failUnlessIn(good_out, out)
  89                 self.failIfIn("-r-- %d -" % len(small), out,
  90                               "trailing hyphen means unknown date")
  91
  92         if good_arg is not None:
  93             d.addCallback(lambda ign: self.do_cli("ls", "-l", good_arg))
  94             d.addCallback(_check4)
  95             # listing a file as $DIRCAP/filename should work just like dir/filename
  96             d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + "/" + good_arg))
  97             d.addCallback(_check4)
  98             # and similarly for $DIRCAP:./filename
  99             d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + ":./" + good_arg))
 100             d.addCallback(_check4)
 101
 102         def _check5((rc, out, err)):
 103             # listing a raw filecap should not explode, but it will have no
 104             # metadata, just the size
 105             self.failUnlessReallyEqual(rc, 0)
 106             self.failUnlessReallyEqual("-r-- %d -" % len(small), out.strip())
 107         d.addCallback(lambda ign: self.do_cli("ls", "-l", self.goodcap))
 108         d.addCallback(_check5)
 109
 110         # Now rename 'g\u00F6\u00F6d' to 'good' and repeat the tests that might have been skipped due
 111         # to encoding problems.
 112         d.addCallback(lambda ign: self.rootnode.move_child_to(u"g\u00F6\u00F6d", self.rootnode, u"good"))
 113
 114         d.addCallback(lambda ign: self.do_cli("ls"))
 115         def _check1_ascii((rc,out,err)):
 116             self.failUnlessReallyEqual(rc, 0)
 117             self.failUnlessReallyEqual(err, "")
 118             self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", "good"]))
 119         d.addCallback(_check1_ascii)
 120         def _check4_ascii((rc, out, err)):
 121             # listing a file (as dir/filename) should have the edge metadata,
 122             # including the filename
 123             self.failUnlessReallyEqual(rc, 0)
 124             self.failUnlessIn("good", out)
 125             self.failIfIn("-r-- %d -" % len(small), out,
 126                           "trailing hyphen means unknown date")
 127
 128         d.addCallback(lambda ign: self.do_cli("ls", "-l", "good"))
 129         d.addCallback(_check4_ascii)
 130         # listing a file as $DIRCAP/filename should work just like dir/filename
 131         d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + "/good"))
 132         d.addCallback(_check4_ascii)
 133         # and similarly for $DIRCAP:./filename
 134         d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + ":./good"))
 135         d.addCallback(_check4_ascii)
 136
 137         unknown_immcap = "imm.URI:unknown"
 138         def _create_unknown(ign):
 139             nm = c0.nodemaker
 140             kids = {u"unknownchild-imm": (nm.create_from_cap(unknown_immcap), {})}
 141             return self.rootnode.create_subdirectory(u"unknown", initial_children=kids,
 142                                                      mutable=False)
 143         d.addCallback(_create_unknown)
 144         def _check6((rc, out, err)):
 145             # listing a directory referencing an unknown object should print
 146             # an extra message to stderr
 147             self.failUnlessReallyEqual(rc, 0)
 148             self.failUnlessIn("?r-- ? - unknownchild-imm\n", out)
 149             self.failUnlessIn("included unknown objects", err)
 150         d.addCallback(lambda ign: self.do_cli("ls", "-l", "unknown"))
 151         d.addCallback(_check6)
 152         def _check7((rc, out, err)):
 153             # listing an unknown cap directly should print an extra message
 154             # to stderr (currently this only works if the URI starts with 'URI:'
 155             # after any 'ro.' or 'imm.' prefix, otherwise it will be confused
 156             # with an alias).
 157             self.failUnlessReallyEqual(rc, 0)
 158             self.failUnlessIn("?r-- ? -\n", out)
 159             self.failUnlessIn("included unknown objects", err)
 160         d.addCallback(lambda ign: self.do_cli("ls", "-l", unknown_immcap))
 161         d.addCallback(_check7)
 162         return d
 163
 164     def test_list_without_alias(self):
 165         # doing just 'tahoe ls' without specifying an alias or first
 166         # doing 'tahoe create-alias tahoe' should fail gracefully.
 167         self.basedir = "cli/List/list_without_alias"
 168         self.set_up_grid()
 169         d = self.do_cli("ls")
 170         def _check((rc, out, err)):
 171             self.failUnlessReallyEqual(rc, 1)
 172             self.failUnlessIn("error:", err)
 173             self.failUnlessReallyEqual(out, "")
 174         d.addCallback(_check)
 175         return d
 176
 177     def test_list_with_nonexistent_alias(self):
 178         # doing 'tahoe ls' while specifying an alias that doesn't already
 179         # exist should fail with an informative error message
 180         self.basedir = "cli/List/list_with_nonexistent_alias"
 181         self.set_up_grid()
 182         d = self.do_cli("ls", "nonexistent:")
 183         def _check((rc, out, err)):
 184             self.failUnlessReallyEqual(rc, 1)
 185             self.failUnlessIn("error:", err)
 186             self.failUnlessIn("nonexistent", err)
 187             self.failUnlessReallyEqual(out, "")
 188         d.addCallback(_check)
 189         return d
 190
 191     def _create_directory_structure(self):
 192         # Create a simple directory structure that we can use for MDMF,
 193         # SDMF, and immutable testing.
 194         assert self.g
 195
 196         client = self.g.clients[0]
 197         # Create a dirnode
 198         d = client.create_dirnode()
 199         def _got_rootnode(n):
 200             # Add a few nodes.
 201             self._dircap = n.get_uri()
 202             nm = n._nodemaker
 203             # The uploaders may run at the same time, so we need two
 204             # MutableData instances or they'll fight over offsets &c and
 205             # break.
 206             mutable_data = MutableData("data" * 100000)
 207             mutable_data2 = MutableData("data" * 100000)
 208             # Add both kinds of mutable node.
 209             d1 = nm.create_mutable_file(mutable_data,
 210                                         version=MDMF_VERSION)
 211             d2 = nm.create_mutable_file(mutable_data2,
 212                                         version=SDMF_VERSION)
 213             # Add an immutable node. We do this through the directory,
 214             # with add_file.
 215             immutable_data = upload.Data("immutable data" * 100000,
 216                                          convergence="")
 217             d3 = n.add_file(u"immutable", immutable_data)
 218             ds = [d1, d2, d3]
 219             dl = defer.DeferredList(ds)
 220             def _made_files((r1, r2, r3)):
 221                 self.failUnless(r1[0])
 222                 self.failUnless(r2[0])
 223                 self.failUnless(r3[0])
 224
 225                 # r1, r2, and r3 contain nodes.
 226                 mdmf_node = r1[1]
 227                 sdmf_node = r2[1]
 228                 imm_node = r3[1]
 229
 230                 self._mdmf_uri = mdmf_node.get_uri()
 231                 self._mdmf_readonly_uri = mdmf_node.get_readonly_uri()
 232                 self._sdmf_uri = mdmf_node.get_uri()
 233                 self._sdmf_readonly_uri = sdmf_node.get_readonly_uri()
 234                 self._imm_uri = imm_node.get_uri()
 235
 236                 d1 = n.set_node(u"mdmf", mdmf_node)
 237                 d2 = n.set_node(u"sdmf", sdmf_node)
 238                 return defer.DeferredList([d1, d2])
 239             # We can now list the directory by listing self._dircap.
 240             dl.addCallback(_made_files)
 241             return dl
 242         d.addCallback(_got_rootnode)
 243         return d
 244
 245     def test_list_mdmf(self):
 246         # 'tahoe ls' should include MDMF files.
 247         self.basedir = "cli/List/list_mdmf"
 248         self.set_up_grid()
 249         d = self._create_directory_structure()
 250         d.addCallback(lambda ignored:
 251             self.do_cli("ls", self._dircap))
 252         def _got_ls((rc, out, err)):
 253             self.failUnlessEqual(rc, 0)
 254             self.failUnlessEqual(err, "")
 255             self.failUnlessIn("immutable", out)
 256             self.failUnlessIn("mdmf", out)
 257             self.failUnlessIn("sdmf", out)
 258         d.addCallback(_got_ls)
 259         return d
 260
 261     def test_list_mdmf_json(self):
 262         # 'tahoe ls' should include MDMF caps when invoked with MDMF
 263         # caps.
 264         self.basedir = "cli/List/list_mdmf_json"
 265         self.set_up_grid()
 266         d = self._create_directory_structure()
 267         d.addCallback(lambda ignored:
 268             self.do_cli("ls", "--json", self._dircap))
 269         def _got_json((rc, out, err)):
 270             self.failUnlessEqual(rc, 0)
 271             self.failUnlessEqual(err, "")
 272             self.failUnlessIn(self._mdmf_uri, out)
 273             self.failUnlessIn(self._mdmf_readonly_uri, out)
 274             self.failUnlessIn(self._sdmf_uri, out)
 275             self.failUnlessIn(self._sdmf_readonly_uri, out)
 276             self.failUnlessIn(self._imm_uri, out)
 277             self.failUnlessIn('"format": "SDMF"', out)
 278             self.failUnlessIn('"format": "MDMF"', out)
 279         d.addCallback(_got_json)
 280         return d