src/allmydata/test/test_cli_check.py

   1 import os.path
   2 import simplejson
   3 from twisted.trial import unittest
   4 from cStringIO import StringIO
   5
   6 from allmydata import uri
   7 from allmydata.util import base32
   8 from allmydata.util.encodingutil import quote_output, to_str
   9 from allmydata.mutable.publish import MutableData
  10 from allmydata.immutable import upload
  11 from allmydata.scripts import debug
  12 from .no_network import GridTestMixin
  13 from .test_cli import CLITestMixin
  14
  15 timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s
  16
  17 class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
  18
  19     def test_check(self):
  20         self.basedir = "cli/Check/check"
  21         self.set_up_grid()
  22         c0 = self.g.clients[0]
  23         DATA = "data" * 100
  24         DATA_uploadable = MutableData(DATA)
  25         d = c0.create_mutable_file(DATA_uploadable)
  26         def _stash_uri(n):
  27             self.uri = n.get_uri()
  28         d.addCallback(_stash_uri)
  29
  30         d.addCallback(lambda ign: self.do_cli("check", self.uri))
  31         def _check1((rc, out, err)):
  32             self.failUnlessReallyEqual(err, "")
  33             self.failUnlessReallyEqual(rc, 0)
  34             lines = out.splitlines()
  35             self.failUnless("Summary: Healthy" in lines, out)
  36             self.failUnless(" good-shares: 10 (encoding is 3-of-10)" in lines, out)
  37         d.addCallback(_check1)
  38
  39         d.addCallback(lambda ign: self.do_cli("check", "--raw", self.uri))
  40         def _check2((rc, out, err)):
  41             self.failUnlessReallyEqual(err, "")
  42             self.failUnlessReallyEqual(rc, 0)
  43             data = simplejson.loads(out)
  44             self.failUnlessReallyEqual(to_str(data["summary"]), "Healthy")
  45             self.failUnlessReallyEqual(data["results"]["healthy"], True)
  46         d.addCallback(_check2)
  47
  48         d.addCallback(lambda ign: c0.upload(upload.Data("literal", convergence="")))
  49         def _stash_lit_uri(n):
  50             self.lit_uri = n.get_uri()
  51         d.addCallback(_stash_lit_uri)
  52
  53         d.addCallback(lambda ign: self.do_cli("check", self.lit_uri))
  54         def _check_lit((rc, out, err)):
  55             self.failUnlessReallyEqual(err, "")
  56             self.failUnlessReallyEqual(rc, 0)
  57             lines = out.splitlines()
  58             self.failUnless("Summary: Healthy (LIT)" in lines, out)
  59         d.addCallback(_check_lit)
  60
  61         d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri))
  62         def _check_lit_raw((rc, out, err)):
  63             self.failUnlessReallyEqual(err, "")
  64             self.failUnlessReallyEqual(rc, 0)
  65             data = simplejson.loads(out)
  66             self.failUnlessReallyEqual(data["results"]["healthy"], True)
  67         d.addCallback(_check_lit_raw)
  68
  69         d.addCallback(lambda ign: c0.create_immutable_dirnode({}, convergence=""))
  70         def _stash_lit_dir_uri(n):
  71             self.lit_dir_uri = n.get_uri()
  72         d.addCallback(_stash_lit_dir_uri)
  73
  74         d.addCallback(lambda ign: self.do_cli("check", self.lit_dir_uri))
  75         d.addCallback(_check_lit)
  76
  77         d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri))
  78         d.addCallback(_check_lit_raw)
  79
  80         def _clobber_shares(ignored):
  81             # delete one, corrupt a second
  82             shares = self.find_uri_shares(self.uri)
  83             self.failUnlessReallyEqual(len(shares), 10)
  84             os.unlink(shares[0][2])
  85             cso = debug.CorruptShareOptions()
  86             cso.stdout = StringIO()
  87             cso.parseOptions([shares[1][2]])
  88             storage_index = uri.from_string(self.uri).get_storage_index()
  89             self._corrupt_share_line = "  server %s, SI %s, shnum %d" % \
  90                                        (base32.b2a(shares[1][1]),
  91                                         base32.b2a(storage_index),
  92                                         shares[1][0])
  93             debug.corrupt_share(cso)
  94         d.addCallback(_clobber_shares)
  95
  96         d.addCallback(lambda ign: self.do_cli("check", "--verify", self.uri))
  97         def _check3((rc, out, err)):
  98             self.failUnlessReallyEqual(err, "")
  99             self.failUnlessReallyEqual(rc, 0)
 100             lines = out.splitlines()
 101             summary = [l for l in lines if l.startswith("Summary")][0]
 102             self.failUnless("Summary: Unhealthy: 8 shares (enc 3-of-10)"
 103                             in summary, summary)
 104             self.failUnless(" good-shares: 8 (encoding is 3-of-10)" in lines, out)
 105             self.failUnless(" corrupt shares:" in lines, out)
 106             self.failUnless(self._corrupt_share_line in lines, out)
 107         d.addCallback(_check3)
 108
 109         d.addCallback(lambda ign: self.do_cli("check", "--verify", "--raw", self.uri))
 110         def _check3_raw((rc, out, err)):
 111             self.failUnlessReallyEqual(err, "")
 112             self.failUnlessReallyEqual(rc, 0)
 113             data = simplejson.loads(out)
 114             self.failUnlessReallyEqual(data["results"]["healthy"], False)
 115             self.failUnlessIn("Unhealthy: 8 shares (enc 3-of-10)", data["summary"])
 116             self.failUnlessReallyEqual(data["results"]["count-shares-good"], 8)
 117             self.failUnlessReallyEqual(data["results"]["count-corrupt-shares"], 1)
 118             self.failUnlessIn("list-corrupt-shares", data["results"])
 119         d.addCallback(_check3_raw)
 120
 121         d.addCallback(lambda ign:
 122                       self.do_cli("check", "--verify", "--repair", self.uri))
 123         def _check4((rc, out, err)):
 124             self.failUnlessReallyEqual(err, "")
 125             self.failUnlessReallyEqual(rc, 0)
 126             lines = out.splitlines()
 127             self.failUnless("Summary: not healthy" in lines, out)
 128             self.failUnless(" good-shares: 8 (encoding is 3-of-10)" in lines, out)
 129             self.failUnless(" corrupt shares:" in lines, out)
 130             self.failUnless(self._corrupt_share_line in lines, out)
 131             self.failUnless(" repair successful" in lines, out)
 132         d.addCallback(_check4)
 133
 134         d.addCallback(lambda ign:
 135                       self.do_cli("check", "--verify", "--repair", self.uri))
 136         def _check5((rc, out, err)):
 137             self.failUnlessReallyEqual(err, "")
 138             self.failUnlessReallyEqual(rc, 0)
 139             lines = out.splitlines()
 140             self.failUnless("Summary: healthy" in lines, out)
 141             self.failUnless(" good-shares: 10 (encoding is 3-of-10)" in lines, out)
 142             self.failIf(" corrupt shares:" in lines, out)
 143         d.addCallback(_check5)
 144
 145         return d
 146
 147     def test_deep_check(self):
 148         self.basedir = "cli/Check/deep_check"
 149         self.set_up_grid()
 150         c0 = self.g.clients[0]
 151         self.uris = {}
 152         self.fileurls = {}
 153         DATA = "data" * 100
 154         quoted_good = quote_output(u"g\u00F6\u00F6d")
 155
 156         d = c0.create_dirnode()
 157         def _stash_root_and_create_file(n):
 158             self.rootnode = n
 159             self.rooturi = n.get_uri()
 160             return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence=""))
 161         d.addCallback(_stash_root_and_create_file)
 162         def _stash_uri(fn, which):
 163             self.uris[which] = fn.get_uri()
 164             return fn
 165         d.addCallback(_stash_uri, u"g\u00F6\u00F6d")
 166         d.addCallback(lambda ign:
 167                       self.rootnode.add_file(u"small",
 168                                            upload.Data("literal",
 169                                                         convergence="")))
 170         d.addCallback(_stash_uri, "small")
 171         d.addCallback(lambda ign:
 172             c0.create_mutable_file(MutableData(DATA+"1")))
 173         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
 174         d.addCallback(_stash_uri, "mutable")
 175
 176         d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi))
 177         def _check1((rc, out, err)):
 178             self.failUnlessReallyEqual(err, "")
 179             self.failUnlessReallyEqual(rc, 0)
 180             lines = out.splitlines()
 181             self.failUnless("done: 4 objects checked, 4 healthy, 0 unhealthy"
 182                             in lines, out)
 183         d.addCallback(_check1)
 184
 185         # root
 186         # root/g\u00F6\u00F6d
 187         # root/small
 188         # root/mutable
 189
 190         d.addCallback(lambda ign: self.do_cli("deep-check", "--verbose",
 191                                               self.rooturi))
 192         def _check2((rc, out, err)):
 193             self.failUnlessReallyEqual(err, "")
 194             self.failUnlessReallyEqual(rc, 0)
 195             lines = out.splitlines()
 196             self.failUnless("'<root>': Healthy" in lines, out)
 197             self.failUnless("'small': Healthy (LIT)" in lines, out)
 198             self.failUnless((quoted_good + ": Healthy") in lines, out)
 199             self.failUnless("'mutable': Healthy" in lines, out)
 200             self.failUnless("done: 4 objects checked, 4 healthy, 0 unhealthy"
 201                             in lines, out)
 202         d.addCallback(_check2)
 203
 204         d.addCallback(lambda ign: self.do_cli("stats", self.rooturi))
 205         def _check_stats((rc, out, err)):
 206             self.failUnlessReallyEqual(err, "")
 207             self.failUnlessReallyEqual(rc, 0)
 208             lines = out.splitlines()
 209             self.failUnlessIn(" count-immutable-files: 1", lines)
 210             self.failUnlessIn("   count-mutable-files: 1", lines)
 211             self.failUnlessIn("   count-literal-files: 1", lines)
 212             self.failUnlessIn("     count-directories: 1", lines)
 213             self.failUnlessIn("  size-immutable-files: 400", lines)
 214             self.failUnlessIn("Size Histogram:", lines)
 215             self.failUnlessIn("   4-10   : 1    (10 B, 10 B)", lines)
 216             self.failUnlessIn(" 317-1000 : 1    (1000 B, 1000 B)", lines)
 217         d.addCallback(_check_stats)
 218
 219         def _clobber_shares(ignored):
 220             shares = self.find_uri_shares(self.uris[u"g\u00F6\u00F6d"])
 221             self.failUnlessReallyEqual(len(shares), 10)
 222             os.unlink(shares[0][2])
 223
 224             shares = self.find_uri_shares(self.uris["mutable"])
 225             cso = debug.CorruptShareOptions()
 226             cso.stdout = StringIO()
 227             cso.parseOptions([shares[1][2]])
 228             storage_index = uri.from_string(self.uris["mutable"]).get_storage_index()
 229             self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \
 230                                        (base32.b2a(shares[1][1]),
 231                                         base32.b2a(storage_index),
 232                                         shares[1][0])
 233             debug.corrupt_share(cso)
 234         d.addCallback(_clobber_shares)
 235
 236         # root
 237         # root/g\u00F6\u00F6d  [9 shares]
 238         # root/small
 239         # root/mutable [1 corrupt share]
 240
 241         d.addCallback(lambda ign:
 242                       self.do_cli("deep-check", "--verbose", self.rooturi))
 243         def _check3((rc, out, err)):
 244             self.failUnlessReallyEqual(err, "")
 245             self.failUnlessReallyEqual(rc, 0)
 246             lines = out.splitlines()
 247             self.failUnless("'<root>': Healthy" in lines, out)
 248             self.failUnless("'small': Healthy (LIT)" in lines, out)
 249             self.failUnless("'mutable': Healthy" in lines, out) # needs verifier
 250             self.failUnless((quoted_good + ": Not Healthy: 9 shares (enc 3-of-10)") in lines, out)
 251             self.failIf(self._corrupt_share_line in lines, out)
 252             self.failUnless("done: 4 objects checked, 3 healthy, 1 unhealthy"
 253                             in lines, out)
 254         d.addCallback(_check3)
 255
 256         d.addCallback(lambda ign:
 257                       self.do_cli("deep-check", "--verbose", "--verify",
 258                                   self.rooturi))
 259         def _check4((rc, out, err)):
 260             self.failUnlessReallyEqual(err, "")
 261             self.failUnlessReallyEqual(rc, 0)
 262             lines = out.splitlines()
 263             self.failUnless("'<root>': Healthy" in lines, out)
 264             self.failUnless("'small': Healthy (LIT)" in lines, out)
 265             mutable = [l for l in lines if l.startswith("'mutable'")][0]
 266             self.failUnless(mutable.startswith("'mutable': Unhealthy: 9 shares (enc 3-of-10)"),
 267                             mutable)
 268             self.failUnless(self._corrupt_share_line in lines, out)
 269             self.failUnless((quoted_good + ": Not Healthy: 9 shares (enc 3-of-10)") in lines, out)
 270             self.failUnless("done: 4 objects checked, 2 healthy, 2 unhealthy"
 271                             in lines, out)
 272         d.addCallback(_check4)
 273
 274         d.addCallback(lambda ign:
 275                       self.do_cli("deep-check", "--raw",
 276                                   self.rooturi))
 277         def _check5((rc, out, err)):
 278             self.failUnlessReallyEqual(err, "")
 279             self.failUnlessReallyEqual(rc, 0)
 280             lines = out.splitlines()
 281             units = [simplejson.loads(line) for line in lines]
 282             # root, small, g\u00F6\u00F6d, mutable,  stats
 283             self.failUnlessReallyEqual(len(units), 4+1)
 284         d.addCallback(_check5)
 285
 286         d.addCallback(lambda ign:
 287                       self.do_cli("deep-check",
 288                                   "--verbose", "--verify", "--repair",
 289                                   self.rooturi))
 290         def _check6((rc, out, err)):
 291             self.failUnlessReallyEqual(err, "")
 292             self.failUnlessReallyEqual(rc, 0)
 293             lines = out.splitlines()
 294             self.failUnless("'<root>': healthy" in lines, out)
 295             self.failUnless("'small': healthy" in lines, out)
 296             self.failUnless("'mutable': not healthy" in lines, out)
 297             self.failUnless(self._corrupt_share_line in lines, out)
 298             self.failUnless((quoted_good + ": not healthy") in lines, out)
 299             self.failUnless("done: 4 objects checked" in lines, out)
 300             self.failUnless(" pre-repair: 2 healthy, 2 unhealthy" in lines, out)
 301             self.failUnless(" 2 repairs attempted, 2 successful, 0 failed"
 302                             in lines, out)
 303             self.failUnless(" post-repair: 4 healthy, 0 unhealthy" in lines,out)
 304         d.addCallback(_check6)
 305
 306         # now add a subdir, and a file below that, then make the subdir
 307         # unrecoverable
 308
 309         d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"subdir"))
 310         d.addCallback(_stash_uri, "subdir")
 311         d.addCallback(lambda fn:
 312                       fn.add_file(u"subfile", upload.Data(DATA+"2", "")))
 313         d.addCallback(lambda ign:
 314                       self.delete_shares_numbered(self.uris["subdir"],
 315                                                   range(10)))
 316
 317         # root
 318         # rootg\u00F6\u00F6d/
 319         # root/small
 320         # root/mutable
 321         # root/subdir [unrecoverable: 0 shares]
 322         # root/subfile
 323
 324         d.addCallback(lambda ign: self.do_cli("manifest", self.rooturi))
 325         def _manifest_failed((rc, out, err)):
 326             self.failIfEqual(rc, 0)
 327             self.failUnlessIn("ERROR: UnrecoverableFileError", err)
 328             # the fatal directory should still show up, as the last line
 329             self.failUnlessIn(" subdir\n", out)
 330         d.addCallback(_manifest_failed)
 331
 332         d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi))
 333         def _deep_check_failed((rc, out, err)):
 334             self.failIfEqual(rc, 0)
 335             self.failUnlessIn("ERROR: UnrecoverableFileError", err)
 336             # we want to make sure that the error indication is the last
 337             # thing that gets emitted
 338             self.failIf("done:" in out, out)
 339         d.addCallback(_deep_check_failed)
 340
 341         # this test is disabled until the deep-repair response to an
 342         # unrepairable directory is fixed. The failure-to-repair should not
 343         # throw an exception, but the failure-to-traverse that follows
 344         # should throw UnrecoverableFileError.
 345
 346         #d.addCallback(lambda ign:
 347         #              self.do_cli("deep-check", "--repair", self.rooturi))
 348         #def _deep_check_repair_failed((rc, out, err)):
 349         #    self.failIfEqual(rc, 0)
 350         #    print err
 351         #    self.failUnlessIn("ERROR: UnrecoverableFileError", err)
 352         #    self.failIf("done:" in out, out)
 353         #d.addCallback(_deep_check_repair_failed)
 354
 355         return d
 356
 357     def test_check_without_alias(self):
 358         # 'tahoe check' should output a sensible error message if it needs to
 359         # find the default alias and can't
 360         self.basedir = "cli/Check/check_without_alias"
 361         self.set_up_grid()
 362         d = self.do_cli("check")
 363         def _check((rc, out, err)):
 364             self.failUnlessReallyEqual(rc, 1)
 365             self.failUnlessIn("error:", err)
 366             self.failUnlessReallyEqual(out, "")
 367         d.addCallback(_check)
 368         d.addCallback(lambda ign: self.do_cli("deep-check"))
 369         d.addCallback(_check)
 370         return d
 371
 372     def test_check_with_nonexistent_alias(self):
 373         # 'tahoe check' should output a sensible error message if it needs to
 374         # find an alias and can't.
 375         self.basedir = "cli/Check/check_with_nonexistent_alias"
 376         self.set_up_grid()
 377         d = self.do_cli("check", "nonexistent:")
 378         def _check((rc, out, err)):
 379             self.failUnlessReallyEqual(rc, 1)
 380             self.failUnlessIn("error:", err)
 381             self.failUnlessIn("nonexistent", err)
 382             self.failUnlessReallyEqual(out, "")
 383         d.addCallback(_check)
 384         return d
 385
 386     def test_check_with_multiple_aliases(self):
 387         self.basedir = "cli/Check/check_with_multiple_aliases"
 388         self.set_up_grid()
 389         self.uriList = []
 390         c0 = self.g.clients[0]
 391         d = c0.create_dirnode()
 392         def _stash_uri(n):
 393             self.uriList.append(n.get_uri())
 394         d.addCallback(_stash_uri)
 395         d = c0.create_dirnode()
 396         d.addCallback(_stash_uri)
 397
 398         d.addCallback(lambda ign: self.do_cli("check", self.uriList[0], self.uriList[1]))
 399         def _check((rc, out, err)):
 400             self.failUnlessReallyEqual(rc, 0)
 401             self.failUnlessReallyEqual(err, "")
 402             #Ensure healthy appears for each uri
 403             self.failUnlessIn("Healthy", out[:len(out)/2])
 404             self.failUnlessIn("Healthy", out[len(out)/2:])
 405         d.addCallback(_check)
 406
 407         d.addCallback(lambda ign: self.do_cli("check", self.uriList[0], "nonexistent:"))
 408         def _check2((rc, out, err)):
 409             self.failUnlessReallyEqual(rc, 1)
 410             self.failUnlessIn("Healthy", out)
 411             self.failUnlessIn("error:", err)
 412             self.failUnlessIn("nonexistent", err)
 413         d.addCallback(_check2)
 414
 415         return d