4 from twisted.trial import unittest
5 from twisted.internet import defer
6 from allmydata import check_results, uri
7 from allmydata import uri as tahoe_uri
8 from allmydata.web import check_results as web_check_results
9 from allmydata.storage_client import StorageFarmBroker, NativeStorageServer
10 from allmydata.storage.server import storage_index_to_dir
11 from allmydata.monitor import Monitor
12 from allmydata.test.no_network import GridTestMixin
13 from allmydata.immutable.upload import Data
14 from allmydata.test.common_web import WebRenderingMixin
15 from allmydata.mutable.publish import MutableData
18 def get_storage_broker(self):
19 return self.storage_broker
21 class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
23 def create_fake_client(self):
24 sb = StorageFarmBroker(None, True)
25 for (peerid, nickname) in [("\x00"*20, "peer-0"),
26 ("\xff"*20, "peer-f"),
27 ("\x11"*20, "peer-11")] :
29 "service-name": "storage",
30 "anonymous-storage-FURL": "pb://abcde@nowhere/fake",
31 "permutation-seed-base32": "",
32 "nickname": unicode(nickname),
33 "app-versions": {}, # need #466 and v2 introducer
35 "oldest-supported": "oldest",
37 s = NativeStorageServer(peerid, ann)
38 sb.test_add_server(peerid, s)
43 def render_json(self, page):
44 d = self.render1(page, args={"output": ["json"]})
47 def test_literal(self):
48 c = self.create_fake_client()
49 lcr = web_check_results.LiteralCheckResultsRenderer(c)
53 s = self.remove_tags(html)
54 self.failUnlessIn("Literal files are always healthy", s)
56 d.addCallback(lambda ignored:
57 self.render1(lcr, args={"return_to": ["FOOURL"]}))
58 def _check_return_to(html):
59 s = self.remove_tags(html)
60 self.failUnlessIn("Literal files are always healthy", s)
61 self.failUnlessIn('<a href="FOOURL">Return to file.</a>',
63 d.addCallback(_check_return_to)
64 d.addCallback(lambda ignored: self.render_json(lcr))
65 def _check_json(json):
66 j = simplejson.loads(json)
67 self.failUnlessEqual(j["storage-index"], "")
68 self.failUnlessEqual(j["results"]["healthy"], True)
69 d.addCallback(_check_json)
73 c = self.create_fake_client()
74 serverid_1 = "\x00"*20
75 serverid_f = "\xff"*20
76 u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234)
77 cr = check_results.CheckResults(u, u.get_storage_index())
79 cr.set_needs_rebalancing(False)
80 cr.set_summary("groovy")
81 data = { "count-shares-needed": 3,
82 "count-shares-expected": 9,
83 "count-shares-good": 10,
84 "count-good-share-hosts": 11,
85 "list-corrupt-shares": [],
86 "count-wrong-shares": 0,
87 "sharemap": {"shareid1": [serverid_1, serverid_f]},
88 "count-recoverable-versions": 1,
89 "count-unrecoverable-versions": 0,
90 "servers-responding": [],
94 w = web_check_results.CheckResultsRenderer(c, cr)
95 html = self.render2(w)
96 s = self.remove_tags(html)
97 self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
98 self.failUnlessIn("Healthy : groovy", s)
99 self.failUnlessIn("Share Counts: need 3-of-9, have 10", s)
100 self.failUnlessIn("Hosts with good shares: 11", s)
101 self.failUnlessIn("Corrupt shares: none", s)
102 self.failUnlessIn("Wrong Shares: 0", s)
103 self.failUnlessIn("Recoverable Versions: 1", s)
104 self.failUnlessIn("Unrecoverable Versions: 0", s)
106 cr.set_healthy(False)
107 cr.set_recoverable(True)
108 cr.set_summary("ungroovy")
109 html = self.render2(w)
110 s = self.remove_tags(html)
111 self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
112 self.failUnlessIn("Not Healthy! : ungroovy", s)
114 cr.set_healthy(False)
115 cr.set_recoverable(False)
116 cr.set_summary("rather dead")
117 data["list-corrupt-shares"] = [(serverid_1, u.get_storage_index(), 2)]
119 html = self.render2(w)
120 s = self.remove_tags(html)
121 self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
122 self.failUnlessIn("Not Recoverable! : rather dead", s)
123 self.failUnlessIn("Corrupt shares: Share ID Nickname Node ID sh#2 peer-0 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", s)
125 html = self.render2(w)
126 s = self.remove_tags(html)
127 self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
128 self.failUnlessIn("Not Recoverable! : rather dead", s)
130 html = self.render2(w, args={"return_to": ["FOOURL"]})
131 self.failUnlessIn('<a href="FOOURL">Return to file/directory.</a>',
134 d = self.render_json(w)
135 def _check_json(jdata):
136 j = simplejson.loads(jdata)
137 self.failUnlessEqual(j["summary"], "rather dead")
138 self.failUnlessEqual(j["storage-index"],
139 "2k6avpjga3dho3zsjo6nnkt7n4")
140 expected = {'needs-rebalancing': False,
141 'count-shares-expected': 9,
143 'count-unrecoverable-versions': 0,
144 'count-shares-needed': 3,
145 'sharemap': {"shareid1":
146 ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
147 "77777777777777777777777777777777"]},
148 'count-recoverable-versions': 1,
149 'list-corrupt-shares':
150 [["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
151 "2k6avpjga3dho3zsjo6nnkt7n4", 2]],
152 'count-good-share-hosts': 11,
153 'count-wrong-shares': 0,
154 'count-shares-good': 10,
155 'count-corrupt-shares': 0,
156 'servers-responding': [],
157 'recoverable': False,
159 self.failUnlessEqual(j["results"], expected)
160 d.addCallback(_check_json)
161 d.addCallback(lambda ignored: self.render1(w))
163 s = self.remove_tags(html)
164 self.failUnlessIn("File Check Results for SI=2k6avp", s)
165 self.failUnlessIn("Not Recoverable! : rather dead", s)
166 d.addCallback(_check)
170 def test_check_and_repair(self):
171 c = self.create_fake_client()
172 serverid_1 = "\x00"*20
173 serverid_f = "\xff"*20
174 u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234)
176 pre_cr = check_results.CheckResults(u, u.get_storage_index())
177 pre_cr.set_healthy(False)
178 pre_cr.set_recoverable(True)
179 pre_cr.set_needs_rebalancing(False)
180 pre_cr.set_summary("illing")
181 data = { "count-shares-needed": 3,
182 "count-shares-expected": 10,
183 "count-shares-good": 6,
184 "count-good-share-hosts": 7,
185 "list-corrupt-shares": [],
186 "count-wrong-shares": 0,
187 "sharemap": {"shareid1": [serverid_1, serverid_f]},
188 "count-recoverable-versions": 1,
189 "count-unrecoverable-versions": 0,
190 "servers-responding": [],
192 pre_cr.set_data(data)
194 post_cr = check_results.CheckResults(u, u.get_storage_index())
195 post_cr.set_healthy(True)
196 post_cr.set_recoverable(True)
197 post_cr.set_needs_rebalancing(False)
198 post_cr.set_summary("groovy")
199 data = { "count-shares-needed": 3,
200 "count-shares-expected": 10,
201 "count-shares-good": 10,
202 "count-good-share-hosts": 11,
203 "list-corrupt-shares": [],
204 "count-wrong-shares": 0,
205 "sharemap": {"shareid1": [serverid_1, serverid_f]},
206 "count-recoverable-versions": 1,
207 "count-unrecoverable-versions": 0,
208 "servers-responding": [],
210 post_cr.set_data(data)
212 crr = check_results.CheckAndRepairResults(u.get_storage_index())
213 crr.pre_repair_results = pre_cr
214 crr.post_repair_results = post_cr
215 crr.repair_attempted = False
217 w = web_check_results.CheckAndRepairResultsRenderer(c, crr)
218 html = self.render2(w)
219 s = self.remove_tags(html)
221 self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
222 self.failUnlessIn("Healthy : groovy", s)
223 self.failUnlessIn("No repair necessary", s)
224 self.failUnlessIn("Post-Repair Checker Results:", s)
225 self.failUnlessIn("Share Counts: need 3-of-10, have 10", s)
227 crr.repair_attempted = True
228 crr.repair_successful = True
229 html = self.render2(w)
230 s = self.remove_tags(html)
232 self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
233 self.failUnlessIn("Healthy : groovy", s)
234 self.failUnlessIn("Repair successful", s)
235 self.failUnlessIn("Post-Repair Checker Results:", s)
237 crr.repair_attempted = True
238 crr.repair_successful = False
239 post_cr.set_healthy(False)
240 post_cr.set_summary("better")
241 html = self.render2(w)
242 s = self.remove_tags(html)
244 self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
245 self.failUnlessIn("Not Healthy! : better", s)
246 self.failUnlessIn("Repair unsuccessful", s)
247 self.failUnlessIn("Post-Repair Checker Results:", s)
249 crr.repair_attempted = True
250 crr.repair_successful = False
251 post_cr.set_healthy(False)
252 post_cr.set_recoverable(False)
253 post_cr.set_summary("worse")
254 html = self.render2(w)
255 s = self.remove_tags(html)
257 self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
258 self.failUnlessIn("Not Recoverable! : worse", s)
259 self.failUnlessIn("Repair unsuccessful", s)
260 self.failUnlessIn("Post-Repair Checker Results:", s)
262 d = self.render_json(w)
264 j = simplejson.loads(data)
265 self.failUnlessEqual(j["repair-attempted"], True)
266 self.failUnlessEqual(j["storage-index"],
267 "2k6avpjga3dho3zsjo6nnkt7n4")
268 self.failUnlessEqual(j["pre-repair-results"]["summary"], "illing")
269 self.failUnlessEqual(j["post-repair-results"]["summary"], "worse")
270 d.addCallback(_got_json)
272 w2 = web_check_results.CheckAndRepairResultsRenderer(c, None)
273 d.addCallback(lambda ignored: self.render_json(w2))
274 def _got_lit_results(data):
275 j = simplejson.loads(data)
276 self.failUnlessEqual(j["repair-attempted"], False)
277 self.failUnlessEqual(j["storage-index"], "")
278 d.addCallback(_got_lit_results)
281 class BalancingAct(GridTestMixin, unittest.TestCase):
282 # test for #1115 regarding the 'count-good-share-hosts' metric
285 def add_server(self, server_number, readonly=False):
286 assert self.g, "I tried to find a grid at self.g, but failed"
287 ss = self.g.make_server(server_number, readonly)
288 #log.msg("just created a server, number: %s => %s" % (server_number, ss,))
289 self.g.add_server(server_number, ss)
291 def add_server_with_share(self, server_number, uri, share_number=None,
293 self.add_server(server_number, readonly)
294 if share_number is not None:
295 self.copy_share_to_server(uri, share_number, server_number)
297 def copy_share_to_server(self, uri, share_number, server_number):
298 ss = self.g.servers_by_number[server_number]
299 # Copy share i from the directory associated with the first
300 # storage server to the directory associated with this one.
301 assert self.g, "I tried to find a grid at self.g, but failed"
302 assert self.shares, "I tried to find shares at self.shares, but failed"
303 old_share_location = self.shares[share_number][2]
304 new_share_location = os.path.join(ss.storedir, "shares")
305 si = tahoe_uri.from_string(self.uri).get_storage_index()
306 new_share_location = os.path.join(new_share_location,
307 storage_index_to_dir(si))
308 if not os.path.exists(new_share_location):
309 os.makedirs(new_share_location)
310 new_share_location = os.path.join(new_share_location,
312 if old_share_location != new_share_location:
313 shutil.copy(old_share_location, new_share_location)
314 shares = self.find_uri_shares(uri)
315 # Make sure that the storage server has the share.
316 self.failUnless((share_number, ss.my_nodeid, new_share_location)
319 def _pretty_shares_chart(self, uri):
320 # Servers are labeled A-Z, shares are labeled 0-9
321 letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
322 assert len(self.g.servers_by_number) < len(letters), \
323 "This little printing function is only meant for < 26 servers"
325 names = dict(zip([ss.my_nodeid for _,ss in
326 self.g.servers_by_number.iteritems()], letters))
327 for shnum, serverid, _ in self.find_uri_shares(uri):
328 shares_chart.setdefault(shnum, []).append(names[serverid])
332 self.basedir = "checker/BalancingAct/1115"
333 self.set_up_grid(num_servers=1)
334 c0 = self.g.clients[0]
335 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
336 c0.DEFAULT_ENCODING_PARAMETERS['n'] = 4
337 c0.DEFAULT_ENCODING_PARAMETERS['k'] = 3
340 d = c0.upload(Data(DATA, convergence=""))
341 def _stash_immutable(ur):
342 self.imm = c0.create_node_from_uri(ur.uri)
343 self.uri = self.imm.get_uri()
344 d.addCallback(_stash_immutable)
345 d.addCallback(lambda ign:
346 self.find_uri_shares(self.uri))
347 def _store_shares(shares):
349 d.addCallback(_store_shares)
352 # Add a new server with just share 3
353 self.add_server_with_share(i, self.uri, 3)
354 #print self._pretty_shares_chart(self.uri)
356 d.addCallback(add_three, i)
358 def _check_and_repair(_):
359 return self.imm.check_and_repair(Monitor())
360 def _check_counts(crr, shares_good, good_share_hosts):
361 p_crr = crr.get_post_repair_results().data
362 #print self._pretty_shares_chart(self.uri)
363 self.failUnless(p_crr['count-shares-good'] == shares_good)
364 self.failUnless(p_crr['count-good-share-hosts'] == good_share_hosts)
368 0:[A] 1:[A] 2:[A] 3:[A,B,C,D,E]
369 4 good shares, but 5 good hosts
370 After deleting all instances of share #3 and repairing:
371 0:[A,B], 1:[A,C], 2:[A,D], 3:[E]
372 Still 4 good shares and 5 good hosts
374 d.addCallback(_check_and_repair)
375 d.addCallback(_check_counts, 4, 5)
376 d.addCallback(lambda _: self.delete_shares_numbered(self.uri, [3]))
377 d.addCallback(_check_and_repair)
378 d.addCallback(_check_counts, 4, 5)
379 d.addCallback(lambda _: [self.g.break_server(sid) for sid
380 in self.g.get_all_serverids()])
381 d.addCallback(_check_and_repair)
382 d.addCallback(_check_counts, 0, 0)
385 class AddLease(GridTestMixin, unittest.TestCase):
386 # test for #875, in which failures in the add-lease call cause
387 # false-negatives in the checker
390 self.basedir = "checker/AddLease/875"
391 self.set_up_grid(num_servers=1)
392 c0 = self.g.clients[0]
393 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
396 d = c0.upload(Data(DATA, convergence=""))
397 def _stash_immutable(ur):
398 self.imm = c0.create_node_from_uri(ur.uri)
399 d.addCallback(_stash_immutable)
400 d.addCallback(lambda ign:
401 c0.create_mutable_file(MutableData("contents")))
402 def _stash_mutable(node):
404 d.addCallback(_stash_mutable)
406 def _check_cr(cr, which):
407 self.failUnless(cr.is_healthy(), which)
409 # these two should work normally
410 d.addCallback(lambda ign: self.imm.check(Monitor(), add_lease=True))
411 d.addCallback(_check_cr, "immutable-normal")
412 d.addCallback(lambda ign: self.mut.check(Monitor(), add_lease=True))
413 d.addCallback(_check_cr, "mutable-normal")
415 really_did_break = []
416 # now break the server's remote_add_lease call
417 def _break_add_lease(ign):
418 def broken_add_lease(*args, **kwargs):
419 really_did_break.append(1)
420 raise KeyError("intentional failure, should be ignored")
421 assert self.g.servers_by_number[0].remote_add_lease
422 self.g.servers_by_number[0].remote_add_lease = broken_add_lease
423 d.addCallback(_break_add_lease)
425 # and confirm that the files still look healthy
426 d.addCallback(lambda ign: self.mut.check(Monitor(), add_lease=True))
427 d.addCallback(_check_cr, "mutable-broken")
428 d.addCallback(lambda ign: self.imm.check(Monitor(), add_lease=True))
429 d.addCallback(_check_cr, "immutable-broken")
431 d.addCallback(lambda ign: self.failUnless(really_did_break))
434 class CounterHolder(object):
436 self._num_active_block_fetches = 0
437 self._max_active_block_fetches = 0
439 from allmydata.immutable.checker import ValidatedReadBucketProxy
440 class MockVRBP(ValidatedReadBucketProxy):
441 def __init__(self, sharenum, bucket, share_hash_tree, num_blocks, block_size, share_size, counterholder):
442 ValidatedReadBucketProxy.__init__(self, sharenum, bucket,
443 share_hash_tree, num_blocks,
444 block_size, share_size)
445 self.counterholder = counterholder
447 def get_block(self, blocknum):
448 self.counterholder._num_active_block_fetches += 1
449 if self.counterholder._num_active_block_fetches > self.counterholder._max_active_block_fetches:
450 self.counterholder._max_active_block_fetches = self.counterholder._num_active_block_fetches
451 d = ValidatedReadBucketProxy.get_block(self, blocknum)
452 def _mark_no_longer_active(res):
453 self.counterholder._num_active_block_fetches -= 1
455 d.addBoth(_mark_no_longer_active)
458 class TooParallel(GridTestMixin, unittest.TestCase):
459 # bug #1395: immutable verifier was aggressively parallized, checking all
460 # blocks of all shares at the same time, blowing our memory budget and
461 # crashing with MemoryErrors on >1GB files.
463 def test_immutable(self):
464 import allmydata.immutable.checker
465 origVRBP = allmydata.immutable.checker.ValidatedReadBucketProxy
467 self.basedir = "checker/TooParallel/immutable"
469 # If any code asks to instantiate a ValidatedReadBucketProxy,
470 # we give them a MockVRBP which is configured to use our
472 counterholder = CounterHolder()
473 def make_mock_VRBP(*args, **kwargs):
474 return MockVRBP(counterholder=counterholder, *args, **kwargs)
475 allmydata.immutable.checker.ValidatedReadBucketProxy = make_mock_VRBP
477 d = defer.succeed(None)
479 self.set_up_grid(num_servers=4)
480 self.c0 = self.g.clients[0]
481 self.c0.DEFAULT_ENCODING_PARAMETERS = { "k": 1,
484 "max_segment_size": 5,
487 DATA = "data" * 100 # 400/5 = 80 blocks
488 return self.c0.upload(Data(DATA, convergence=""))
489 d.addCallback(_start)
491 n = self.c0.create_node_from_uri(ur.uri)
492 return n.check(Monitor(), verify=True)
493 d.addCallback(_do_check)
495 # the verifier works on all 4 shares in parallel, but only
496 # fetches one block from each share at a time, so we expect to
497 # see 4 parallel fetches
498 self.failUnlessEqual(counterholder._max_active_block_fetches, 4)
499 d.addCallback(_check)
501 allmydata.immutable.checker.ValidatedReadBucketProxy = origVRBP
506 test_immutable.timeout = 80