src/allmydata/test/test_checker.py

   1
   2 import simplejson
   3 from twisted.trial import unittest
   4 from allmydata import check_results, uri
   5 from allmydata.web import check_results as web_check_results
   6 from allmydata.storage_client import StorageFarmBroker, NativeStorageClientDescriptor
   7 from allmydata.monitor import Monitor
   8 from allmydata.test.no_network import GridTestMixin
   9 from allmydata.immutable.upload import Data
  10 from common_web import WebRenderingMixin
  11
  12 class FakeClient:
  13     def get_storage_broker(self):
  14         return self.storage_broker
  15
  16 class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
  17
  18     def create_fake_client(self):
  19         sb = StorageFarmBroker(None, True)
  20         for (peerid, nickname) in [("\x00"*20, "peer-0"),
  21                                    ("\xff"*20, "peer-f"),
  22                                    ("\x11"*20, "peer-11")] :
  23             ann_d = { "version": 0,
  24                       "service-name": "storage",
  25                       "FURL": "fake furl",
  26                       "nickname": unicode(nickname),
  27                       "app-versions": {}, # need #466 and v2 introducer
  28                       "my-version": "ver",
  29                       "oldest-supported": "oldest",
  30                       }
  31             dsc = NativeStorageClientDescriptor(peerid, ann_d)
  32             sb.test_add_descriptor(peerid, dsc)
  33         c = FakeClient()
  34         c.storage_broker = sb
  35         return c
  36
  37     def render_json(self, page):
  38         d = self.render1(page, args={"output": ["json"]})
  39         return d
  40
  41     def test_literal(self):
  42         c = self.create_fake_client()
  43         lcr = web_check_results.LiteralCheckResults(c)
  44
  45         d = self.render1(lcr)
  46         def _check(html):
  47             s = self.remove_tags(html)
  48             self.failUnlessIn("Literal files are always healthy", s)
  49         d.addCallback(_check)
  50         d.addCallback(lambda ignored:
  51                       self.render1(lcr, args={"return_to": ["FOOURL"]}))
  52         def _check_return_to(html):
  53             s = self.remove_tags(html)
  54             self.failUnlessIn("Literal files are always healthy", s)
  55             self.failUnlessIn('<a href="FOOURL">Return to file.</a>',
  56                               html)
  57         d.addCallback(_check_return_to)
  58         d.addCallback(lambda ignored: self.render_json(lcr))
  59         def _check_json(json):
  60             j = simplejson.loads(json)
  61             self.failUnlessEqual(j["storage-index"], "")
  62             self.failUnlessEqual(j["results"]["healthy"], True)
  63         d.addCallback(_check_json)
  64         return d
  65
  66     def test_check(self):
  67         c = self.create_fake_client()
  68         serverid_1 = "\x00"*20
  69         serverid_f = "\xff"*20
  70         u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234)
  71         cr = check_results.CheckResults(u, u.get_storage_index())
  72         cr.set_healthy(True)
  73         cr.set_needs_rebalancing(False)
  74         cr.set_summary("groovy")
  75         data = { "count-shares-needed": 3,
  76                  "count-shares-expected": 9,
  77                  "count-shares-good": 10,
  78                  "count-good-share-hosts": 11,
  79                  "list-corrupt-shares": [],
  80                  "count-wrong-shares": 0,
  81                  "sharemap": {"shareid1": [serverid_1, serverid_f]},
  82                  "count-recoverable-versions": 1,
  83                  "count-unrecoverable-versions": 0,
  84                  "servers-responding": [],
  85                  }
  86         cr.set_data(data)
  87
  88         w = web_check_results.CheckResults(c, cr)
  89         html = self.render2(w)
  90         s = self.remove_tags(html)
  91         self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
  92         self.failUnlessIn("Healthy : groovy", s)
  93         self.failUnlessIn("Share Counts: need 3-of-9, have 10", s)
  94         self.failUnlessIn("Hosts with good shares: 11", s)
  95         self.failUnlessIn("Corrupt shares: none", s)
  96         self.failUnlessIn("Wrong Shares: 0", s)
  97         self.failUnlessIn("Recoverable Versions: 1", s)
  98         self.failUnlessIn("Unrecoverable Versions: 0", s)
  99
 100         cr.set_healthy(False)
 101         cr.set_recoverable(True)
 102         cr.set_summary("ungroovy")
 103         html = self.render2(w)
 104         s = self.remove_tags(html)
 105         self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
 106         self.failUnlessIn("Not Healthy! : ungroovy", s)
 107
 108         cr.set_healthy(False)
 109         cr.set_recoverable(False)
 110         cr.set_summary("rather dead")
 111         data["list-corrupt-shares"] = [(serverid_1, u.get_storage_index(), 2)]
 112         cr.set_data(data)
 113         html = self.render2(w)
 114         s = self.remove_tags(html)
 115         self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
 116         self.failUnlessIn("Not Recoverable! : rather dead", s)
 117         self.failUnlessIn("Corrupt shares: Share ID Nickname Node ID sh#2 peer-0 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", s)
 118
 119         html = self.render2(w)
 120         s = self.remove_tags(html)
 121         self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
 122         self.failUnlessIn("Not Recoverable! : rather dead", s)
 123
 124         html = self.render2(w, args={"return_to": ["FOOURL"]})
 125         self.failUnlessIn('<a href="FOOURL">Return to file/directory.</a>',
 126                           html)
 127
 128         d = self.render_json(w)
 129         def _check_json(jdata):
 130             j = simplejson.loads(jdata)
 131             self.failUnlessEqual(j["summary"], "rather dead")
 132             self.failUnlessEqual(j["storage-index"],
 133                                  "2k6avpjga3dho3zsjo6nnkt7n4")
 134             expected = {'needs-rebalancing': False,
 135                         'count-shares-expected': 9,
 136                         'healthy': False,
 137                         'count-unrecoverable-versions': 0,
 138                         'count-shares-needed': 3,
 139                         'sharemap': {"shareid1":
 140                                      ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 141                                       "77777777777777777777777777777777"]},
 142                         'count-recoverable-versions': 1,
 143                         'list-corrupt-shares':
 144                         [["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 145                           "2k6avpjga3dho3zsjo6nnkt7n4", 2]],
 146                         'count-good-share-hosts': 11,
 147                         'count-wrong-shares': 0,
 148                         'count-shares-good': 10,
 149                         'count-corrupt-shares': 0,
 150                         'servers-responding': [],
 151                         'recoverable': False,
 152                         }
 153             self.failUnlessEqual(j["results"], expected)
 154         d.addCallback(_check_json)
 155         d.addCallback(lambda ignored: self.render1(w))
 156         def _check(html):
 157             s = self.remove_tags(html)
 158             self.failUnlessIn("File Check Results for SI=2k6avp", s)
 159             self.failUnlessIn("Not Recoverable! : rather dead", s)
 160         d.addCallback(_check)
 161         return d
 162
 163
 164     def test_check_and_repair(self):
 165         c = self.create_fake_client()
 166         serverid_1 = "\x00"*20
 167         serverid_f = "\xff"*20
 168         u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234)
 169
 170         pre_cr = check_results.CheckResults(u, u.get_storage_index())
 171         pre_cr.set_healthy(False)
 172         pre_cr.set_recoverable(True)
 173         pre_cr.set_needs_rebalancing(False)
 174         pre_cr.set_summary("illing")
 175         data = { "count-shares-needed": 3,
 176                  "count-shares-expected": 10,
 177                  "count-shares-good": 6,
 178                  "count-good-share-hosts": 7,
 179                  "list-corrupt-shares": [],
 180                  "count-wrong-shares": 0,
 181                  "sharemap": {"shareid1": [serverid_1, serverid_f]},
 182                  "count-recoverable-versions": 1,
 183                  "count-unrecoverable-versions": 0,
 184                  "servers-responding": [],
 185                  }
 186         pre_cr.set_data(data)
 187
 188         post_cr = check_results.CheckResults(u, u.get_storage_index())
 189         post_cr.set_healthy(True)
 190         post_cr.set_recoverable(True)
 191         post_cr.set_needs_rebalancing(False)
 192         post_cr.set_summary("groovy")
 193         data = { "count-shares-needed": 3,
 194                  "count-shares-expected": 10,
 195                  "count-shares-good": 10,
 196                  "count-good-share-hosts": 11,
 197                  "list-corrupt-shares": [],
 198                  "count-wrong-shares": 0,
 199                  "sharemap": {"shareid1": [serverid_1, serverid_f]},
 200                  "count-recoverable-versions": 1,
 201                  "count-unrecoverable-versions": 0,
 202                  "servers-responding": [],
 203                  }
 204         post_cr.set_data(data)
 205
 206         crr = check_results.CheckAndRepairResults(u.get_storage_index())
 207         crr.pre_repair_results = pre_cr
 208         crr.post_repair_results = post_cr
 209         crr.repair_attempted = False
 210
 211         w = web_check_results.CheckAndRepairResults(c, crr)
 212         html = self.render2(w)
 213         s = self.remove_tags(html)
 214
 215         self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
 216         self.failUnlessIn("Healthy : groovy", s)
 217         self.failUnlessIn("No repair necessary", s)
 218         self.failUnlessIn("Post-Repair Checker Results:", s)
 219         self.failUnlessIn("Share Counts: need 3-of-10, have 10", s)
 220
 221         crr.repair_attempted = True
 222         crr.repair_successful = True
 223         html = self.render2(w)
 224         s = self.remove_tags(html)
 225
 226         self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
 227         self.failUnlessIn("Healthy : groovy", s)
 228         self.failUnlessIn("Repair successful", s)
 229         self.failUnlessIn("Post-Repair Checker Results:", s)
 230
 231         crr.repair_attempted = True
 232         crr.repair_successful = False
 233         post_cr.set_healthy(False)
 234         post_cr.set_summary("better")
 235         html = self.render2(w)
 236         s = self.remove_tags(html)
 237
 238         self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
 239         self.failUnlessIn("Not Healthy! : better", s)
 240         self.failUnlessIn("Repair unsuccessful", s)
 241         self.failUnlessIn("Post-Repair Checker Results:", s)
 242
 243         crr.repair_attempted = True
 244         crr.repair_successful = False
 245         post_cr.set_healthy(False)
 246         post_cr.set_recoverable(False)
 247         post_cr.set_summary("worse")
 248         html = self.render2(w)
 249         s = self.remove_tags(html)
 250
 251         self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
 252         self.failUnlessIn("Not Recoverable! : worse", s)
 253         self.failUnlessIn("Repair unsuccessful", s)
 254         self.failUnlessIn("Post-Repair Checker Results:", s)
 255
 256         d = self.render_json(w)
 257         def _got_json(data):
 258             j = simplejson.loads(data)
 259             self.failUnlessEqual(j["repair-attempted"], True)
 260             self.failUnlessEqual(j["storage-index"],
 261                                  "2k6avpjga3dho3zsjo6nnkt7n4")
 262             self.failUnlessEqual(j["pre-repair-results"]["summary"], "illing")
 263             self.failUnlessEqual(j["post-repair-results"]["summary"], "worse")
 264         d.addCallback(_got_json)
 265
 266         w2 = web_check_results.CheckAndRepairResults(c, None)
 267         d.addCallback(lambda ignored: self.render_json(w2))
 268         def _got_lit_results(data):
 269             j = simplejson.loads(data)
 270             self.failUnlessEqual(j["repair-attempted"], False)
 271             self.failUnlessEqual(j["storage-index"], "")
 272         d.addCallback(_got_lit_results)
 273         return d
 274
 275 class AddLease(GridTestMixin, unittest.TestCase):
 276     # test for #875, in which failures in the add-lease call cause
 277     # false-negatives in the checker
 278
 279     def test_875(self):
 280         self.basedir = "checker/AddLease/875"
 281         self.set_up_grid(num_servers=1)
 282         c0 = self.g.clients[0]
 283         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
 284         self.uris = {}
 285         DATA = "data" * 100
 286         d = c0.upload(Data(DATA, convergence=""))
 287         def _stash_immutable(ur):
 288             self.imm = c0.create_node_from_uri(ur.uri)
 289         d.addCallback(_stash_immutable)
 290         d.addCallback(lambda ign: c0.create_mutable_file("contents"))
 291         def _stash_mutable(node):
 292             self.mut = node
 293         d.addCallback(_stash_mutable)
 294
 295         def _check_cr(cr, which):
 296             self.failUnless(cr.is_healthy(), which)
 297
 298         # these two should work normally
 299         d.addCallback(lambda ign: self.imm.check(Monitor(), add_lease=True))
 300         d.addCallback(_check_cr, "immutable-normal")
 301         d.addCallback(lambda ign: self.mut.check(Monitor(), add_lease=True))
 302         d.addCallback(_check_cr, "mutable-normal")
 303
 304         really_did_break = []
 305         # now break the server's remote_add_lease call
 306         def _break_add_lease(ign):
 307             def broken_add_lease(*args, **kwargs):
 308                 really_did_break.append(1)
 309                 raise KeyError("intentional failure, should be ignored")
 310             assert self.g.servers_by_number[0].remote_add_lease
 311             self.g.servers_by_number[0].remote_add_lease = broken_add_lease
 312         d.addCallback(_break_add_lease)
 313
 314         # and confirm that the files still look healthy
 315         d.addCallback(lambda ign: self.mut.check(Monitor(), add_lease=True))
 316         d.addCallback(_check_cr, "mutable-broken")
 317         d.addCallback(lambda ign: self.imm.check(Monitor(), add_lease=True))
 318         d.addCallback(_check_cr, "immutable-broken")
 319
 320         d.addCallback(lambda ign: self.failUnless(really_did_break))
 321         return d