2 def foo(): pass # keep the line number constant
5 from StringIO import StringIO
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.python.failure import Failure
9 from twisted.python import log
11 from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
12 from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
13 from allmydata.util import limiter, time_format, pollmixin, cachedir
14 from allmydata.util import statistics, dictutil, pipeline
15 from allmydata.util import log as tahoe_log
17 class Base32(unittest.TestCase):
18 def test_b2a_matches_Pythons(self):
20 y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0"
21 x = base64.b32encode(y)
22 while x and x[-1] == '=':
25 self.failUnlessEqual(base32.b2a(y), x)
27 self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a")
28 def test_b2a_or_none(self):
29 self.failUnlessEqual(base32.b2a_or_none(None), None)
30 self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a")
32 self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34")
33 self.failUnlessRaises(AssertionError, base32.a2b, "b0gus")
35 class IDLib(unittest.TestCase):
36 def test_nodeid_b2a(self):
37 self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
39 class NoArgumentException(Exception):
43 class HumanReadable(unittest.TestCase):
46 self.failUnlessEqual(hr(foo), "<foo() at test_util.py:2>")
47 self.failUnlessEqual(hr(self.test_repr),
48 "<bound method HumanReadable.test_repr of <allmydata.test.test_util.HumanReadable testMethod=test_repr>>")
49 self.failUnlessEqual(hr(1L), "1")
50 self.failUnlessEqual(hr(10**40),
51 "100000000000000000...000000000000000000")
52 self.failUnlessEqual(hr(self), "<allmydata.test.test_util.HumanReadable testMethod=test_repr>")
53 self.failUnlessEqual(hr([1,2]), "[1, 2]")
54 self.failUnlessEqual(hr({1:2}), "{1:2}")
59 hr(e) == "<ValueError: ()>" # python-2.4
60 or hr(e) == "ValueError()") # python-2.5
62 raise ValueError("oops")
65 hr(e) == "<ValueError: 'oops'>" # python-2.4
66 or hr(e) == "ValueError('oops',)") # python-2.5
68 raise NoArgumentException
71 hr(e) == "<NoArgumentException>" # python-2.4
72 or hr(e) == "NoArgumentException()") # python-2.5
78 class Math(unittest.TestCase):
79 def test_div_ceil(self):
81 self.failUnlessEqual(f(0, 1), 0)
82 self.failUnlessEqual(f(0, 2), 0)
83 self.failUnlessEqual(f(0, 3), 0)
84 self.failUnlessEqual(f(1, 3), 1)
85 self.failUnlessEqual(f(2, 3), 1)
86 self.failUnlessEqual(f(3, 3), 1)
87 self.failUnlessEqual(f(4, 3), 2)
88 self.failUnlessEqual(f(5, 3), 2)
89 self.failUnlessEqual(f(6, 3), 2)
90 self.failUnlessEqual(f(7, 3), 3)
92 def test_next_multiple(self):
93 f = mathutil.next_multiple
94 self.failUnlessEqual(f(5, 1), 5)
95 self.failUnlessEqual(f(5, 2), 6)
96 self.failUnlessEqual(f(5, 3), 6)
97 self.failUnlessEqual(f(5, 4), 8)
98 self.failUnlessEqual(f(5, 5), 5)
99 self.failUnlessEqual(f(5, 6), 6)
100 self.failUnlessEqual(f(32, 1), 32)
101 self.failUnlessEqual(f(32, 2), 32)
102 self.failUnlessEqual(f(32, 3), 33)
103 self.failUnlessEqual(f(32, 4), 32)
104 self.failUnlessEqual(f(32, 5), 35)
105 self.failUnlessEqual(f(32, 6), 36)
106 self.failUnlessEqual(f(32, 7), 35)
107 self.failUnlessEqual(f(32, 8), 32)
108 self.failUnlessEqual(f(32, 9), 36)
109 self.failUnlessEqual(f(32, 10), 40)
110 self.failUnlessEqual(f(32, 11), 33)
111 self.failUnlessEqual(f(32, 12), 36)
112 self.failUnlessEqual(f(32, 13), 39)
113 self.failUnlessEqual(f(32, 14), 42)
114 self.failUnlessEqual(f(32, 15), 45)
115 self.failUnlessEqual(f(32, 16), 32)
116 self.failUnlessEqual(f(32, 17), 34)
117 self.failUnlessEqual(f(32, 18), 36)
118 self.failUnlessEqual(f(32, 589), 589)
120 def test_pad_size(self):
121 f = mathutil.pad_size
122 self.failUnlessEqual(f(0, 4), 0)
123 self.failUnlessEqual(f(1, 4), 3)
124 self.failUnlessEqual(f(2, 4), 2)
125 self.failUnlessEqual(f(3, 4), 1)
126 self.failUnlessEqual(f(4, 4), 0)
127 self.failUnlessEqual(f(5, 4), 3)
129 def test_is_power_of_k(self):
130 f = mathutil.is_power_of_k
131 for i in range(1, 100):
132 if i in (1, 2, 4, 8, 16, 32, 64):
133 self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
135 self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
136 for i in range(1, 100):
137 if i in (1, 3, 9, 27, 81):
138 self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
140 self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
142 def test_next_power_of_k(self):
143 f = mathutil.next_power_of_k
144 self.failUnlessEqual(f(0,2), 1)
145 self.failUnlessEqual(f(1,2), 1)
146 self.failUnlessEqual(f(2,2), 2)
147 self.failUnlessEqual(f(3,2), 4)
148 self.failUnlessEqual(f(4,2), 4)
149 for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
150 for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
151 for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
152 for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
153 for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
155 self.failUnlessEqual(f(0,3), 1)
156 self.failUnlessEqual(f(1,3), 1)
157 self.failUnlessEqual(f(2,3), 3)
158 self.failUnlessEqual(f(3,3), 3)
159 for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
160 for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
161 for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
162 for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
166 self.failUnlessEqual(f([1,2,3]), 2)
167 self.failUnlessEqual(f([0,0,0,4]), 1)
168 self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
170 def test_round_sigfigs(self):
171 f = mathutil.round_sigfigs
172 self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002)
174 class Statistics(unittest.TestCase):
175 def should_assert(self, msg, func, *args, **kwargs):
177 func(*args, **kwargs)
179 except AssertionError:
182 def failUnlessListEqual(self, a, b, msg = None):
183 self.failUnlessEqual(len(a), len(b))
184 for i in range(len(a)):
185 self.failUnlessEqual(a[i], b[i], msg)
187 def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None):
188 self.failUnlessEqual(len(a), len(b))
189 for i in range(len(a)):
190 self.failUnlessAlmostEqual(a[i], b[i], places, msg)
192 def test_binomial_coeff(self):
193 f = statistics.binomial_coeff
194 self.failUnlessEqual(f(20, 0), 1)
195 self.failUnlessEqual(f(20, 1), 20)
196 self.failUnlessEqual(f(20, 2), 190)
197 self.failUnlessEqual(f(20, 8), f(20, 12))
198 self.should_assert("Should assert if n < k", f, 2, 3)
200 def test_binomial_distribution_pmf(self):
201 f = statistics.binomial_distribution_pmf
204 pmf_stat = [0.81, 0.18, 0.01]
205 self.failUnlessListAlmostEqual(pmf_comp, pmf_stat)
207 # Summing across a PMF should give the total probability 1
208 self.failUnlessAlmostEqual(sum(pmf_comp), 1)
209 self.should_assert("Should assert if not 0<=p<=1", f, 1, -1)
210 self.should_assert("Should assert if n < 1", f, 0, .1)
213 statistics.print_pmf(pmf_comp, out=out)
214 lines = out.getvalue().splitlines()
215 self.failUnlessEqual(lines[0], "i=0: 0.81")
216 self.failUnlessEqual(lines[1], "i=1: 0.18")
217 self.failUnlessEqual(lines[2], "i=2: 0.01")
219 def test_survival_pmf(self):
220 f = statistics.survival_pmf
221 # Cross-check binomial-distribution method against convolution
223 p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20
224 pmf1 = statistics.survival_pmf_via_conv(p_list)
225 pmf2 = statistics.survival_pmf_via_bd(p_list)
226 self.failUnlessListAlmostEqual(pmf1, pmf2)
227 self.failUnlessTrue(statistics.valid_pmf(pmf1))
228 self.should_assert("Should assert if p_i > 1", f, [1.1]);
229 self.should_assert("Should assert if p_i < 0", f, [-.1]);
231 def test_repair_count_pmf(self):
232 survival_pmf = statistics.binomial_distribution_pmf(5, .9)
233 repair_pmf = statistics.repair_count_pmf(survival_pmf, 3)
234 # repair_pmf[0] == sum(survival_pmf[0,1,2,5])
235 # repair_pmf[1] == survival_pmf[4]
236 # repair_pmf[2] = survival_pmf[3]
237 self.failUnlessListAlmostEqual(repair_pmf,
238 [0.00001 + 0.00045 + 0.0081 + 0.59049,
243 def test_repair_cost(self):
244 survival_pmf = statistics.binomial_distribution_pmf(5, .9)
245 bwcost = statistics.bandwidth_cost_function
246 cost = statistics.mean_repair_cost(bwcost, 1000,
247 survival_pmf, 3, ul_dl_ratio=1.0)
248 self.failUnlessAlmostEqual(cost, 558.90)
249 cost = statistics.mean_repair_cost(bwcost, 1000,
250 survival_pmf, 3, ul_dl_ratio=8.0)
251 self.failUnlessAlmostEqual(cost, 1664.55)
253 # I haven't manually checked the math beyond here -warner
254 cost = statistics.eternal_repair_cost(bwcost, 1000,
256 discount_rate=0, ul_dl_ratio=1.0)
257 self.failUnlessAlmostEqual(cost, 65292.056074766246)
258 cost = statistics.eternal_repair_cost(bwcost, 1000,
262 self.failUnlessAlmostEqual(cost, 9133.6097158191551)
264 def test_convolve(self):
265 f = statistics.convolve
269 v1v2result = [ 4, 13, 28, 27, 18 ]
270 # Convolution is commutative
273 self.failUnlessListEqual(r1, r2, "Convolution should be commutative")
274 self.failUnlessListEqual(r1, v1v2result, "Didn't match known result")
275 # Convolution is associative
276 r1 = f(f(v1, v2), v3)
277 r2 = f(v1, f(v2, v3))
278 self.failUnlessListEqual(r1, r2, "Convolution should be associative")
279 # Convolution is distributive
280 r1 = f(v3, [ a + b for a, b in zip(v1, v2) ])
283 r2 = [ a + b for a, b in zip(tmp1, tmp2) ]
284 self.failUnlessListEqual(r1, r2, "Convolution should be distributive")
285 # Convolution is scalar multiplication associative
287 r1 = [ a * 4 for a in tmp1 ]
288 tmp2 = [ a * 4 for a in v1 ]
290 self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative")
292 def test_find_k(self):
293 f = statistics.find_k
294 g = statistics.pr_file_loss
295 plist = [.9] * 10 + [.8] * 10 # N=20
298 self.failUnlessEqual(k, 10)
299 self.failUnless(g(plist, k) < t)
301 def test_pr_file_loss(self):
302 f = statistics.pr_file_loss
304 self.failUnlessEqual(f(plist, 3), .0546875)
306 def test_pr_backup_file_loss(self):
307 f = statistics.pr_backup_file_loss
309 self.failUnlessEqual(f(plist, .5, 3), .02734375)
312 class Asserts(unittest.TestCase):
313 def should_assert(self, func, *args, **kwargs):
315 func(*args, **kwargs)
316 except AssertionError, e:
319 self.fail("assert failed with non-AssertionError: %s" % e)
320 self.fail("assert was not caught")
322 def should_not_assert(self, func, *args, **kwargs):
324 func(*args, **kwargs)
325 except AssertionError, e:
326 self.fail("assertion fired when it should not have: %s" % e)
328 self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e)
332 def test_assert(self):
333 f = assertutil._assert
334 self.should_assert(f)
335 self.should_assert(f, False)
336 self.should_not_assert(f, True)
338 m = self.should_assert(f, False, "message")
339 self.failUnlessEqual(m, "'message' <type 'str'>", m)
340 m = self.should_assert(f, False, "message1", othermsg=12)
341 self.failUnlessEqual("'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
342 m = self.should_assert(f, False, othermsg="message2")
343 self.failUnlessEqual("othermsg: 'message2' <type 'str'>", m)
345 def test_precondition(self):
346 f = assertutil.precondition
347 self.should_assert(f)
348 self.should_assert(f, False)
349 self.should_not_assert(f, True)
351 m = self.should_assert(f, False, "message")
352 self.failUnlessEqual("precondition: 'message' <type 'str'>", m)
353 m = self.should_assert(f, False, "message1", othermsg=12)
354 self.failUnlessEqual("precondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
355 m = self.should_assert(f, False, othermsg="message2")
356 self.failUnlessEqual("precondition: othermsg: 'message2' <type 'str'>", m)
358 def test_postcondition(self):
359 f = assertutil.postcondition
360 self.should_assert(f)
361 self.should_assert(f, False)
362 self.should_not_assert(f, True)
364 m = self.should_assert(f, False, "message")
365 self.failUnlessEqual("postcondition: 'message' <type 'str'>", m)
366 m = self.should_assert(f, False, "message1", othermsg=12)
367 self.failUnlessEqual("postcondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
368 m = self.should_assert(f, False, othermsg="message2")
369 self.failUnlessEqual("postcondition: othermsg: 'message2' <type 'str'>", m)
371 class FileUtil(unittest.TestCase):
372 def mkdir(self, basedir, path, mode=0777):
373 fn = os.path.join(basedir, path)
374 fileutil.make_dirs(fn, mode)
376 def touch(self, basedir, path, mode=None, data="touch\n"):
377 fn = os.path.join(basedir, path)
384 def test_rm_dir(self):
385 basedir = "util/FileUtil/test_rm_dir"
386 fileutil.make_dirs(basedir)
387 # create it again to test idempotency
388 fileutil.make_dirs(basedir)
389 d = os.path.join(basedir, "doomed")
391 self.touch(d, "a/b/1.txt")
392 self.touch(d, "a/b/2.txt", 0444)
393 self.touch(d, "a/b/3.txt", 0)
395 self.touch(d, "a/c/1.txt")
396 self.touch(d, "a/c/2.txt", 0444)
397 self.touch(d, "a/c/3.txt", 0)
398 os.chmod(os.path.join(d, "a/c"), 0444)
400 self.touch(d, "a/d/1.txt")
401 self.touch(d, "a/d/2.txt", 0444)
402 self.touch(d, "a/d/3.txt", 0)
403 os.chmod(os.path.join(d, "a/d"), 0)
406 self.failIf(os.path.exists(d))
407 # remove it again to test idempotency
410 def test_remove_if_possible(self):
411 basedir = "util/FileUtil/test_remove_if_possible"
412 fileutil.make_dirs(basedir)
413 self.touch(basedir, "here")
414 fn = os.path.join(basedir, "here")
415 fileutil.remove_if_possible(fn)
416 self.failIf(os.path.exists(fn))
417 fileutil.remove_if_possible(fn) # should be idempotent
418 fileutil.rm_dir(basedir)
419 fileutil.remove_if_possible(fn) # should survive errors
421 def test_open_or_create(self):
422 basedir = "util/FileUtil/test_open_or_create"
423 fileutil.make_dirs(basedir)
424 fn = os.path.join(basedir, "here")
425 f = fileutil.open_or_create(fn)
428 f = fileutil.open_or_create(fn)
435 self.failUnlessEqual(data, "stuff.more.")
437 def test_NamedTemporaryDirectory(self):
438 basedir = "util/FileUtil/test_NamedTemporaryDirectory"
439 fileutil.make_dirs(basedir)
440 td = fileutil.NamedTemporaryDirectory(dir=basedir)
442 self.failUnless(basedir in name)
443 self.failUnless(basedir in repr(td))
444 self.failUnless(os.path.isdir(name))
446 # it is conceivable that we need to force gc here, but I'm not sure
447 self.failIf(os.path.isdir(name))
449 def test_rename(self):
450 basedir = "util/FileUtil/test_rename"
451 fileutil.make_dirs(basedir)
452 self.touch(basedir, "here")
453 fn = os.path.join(basedir, "here")
454 fn2 = os.path.join(basedir, "there")
455 fileutil.rename(fn, fn2)
456 self.failIf(os.path.exists(fn))
457 self.failUnless(os.path.exists(fn2))
460 basedir = "util/FileUtil/test_du"
461 fileutil.make_dirs(basedir)
462 d = os.path.join(basedir, "space-consuming")
464 self.touch(d, "a/b/1.txt", data="a"*10)
465 self.touch(d, "a/b/2.txt", data="b"*11)
467 self.touch(d, "a/c/1.txt", data="c"*12)
468 self.touch(d, "a/c/2.txt", data="d"*13)
470 used = fileutil.du(basedir)
471 self.failUnlessEqual(10+11+12+13, used)
473 def test_abspath_expanduser_unicode(self):
474 self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring")
476 saved_cwd = os.path.normpath(os.getcwdu())
477 abspath_cwd = fileutil.abspath_expanduser_unicode(u".")
478 self.failUnless(isinstance(saved_cwd, unicode), saved_cwd)
479 self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd)
480 self.failUnlessEqual(abspath_cwd, saved_cwd)
482 # adapted from <http://svn.python.org/view/python/branches/release26-maint/Lib/test/test_posixpath.py?view=markup&pathrev=78279#test_abspath>
484 self.failUnlessIn(u"foo", fileutil.abspath_expanduser_unicode(u"foo"))
485 self.failIfIn(u"~", fileutil.abspath_expanduser_unicode(u"~"))
489 cwds.append(u'\xe7w\xf0'.encode(sys.getfilesystemencoding()
491 except UnicodeEncodeError:
492 pass # the cwd can't be encoded -- test with ascii cwd only
498 for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'):
499 uabspath = fileutil.abspath_expanduser_unicode(upath)
500 self.failUnless(isinstance(uabspath, unicode), uabspath)
504 class PollMixinTests(unittest.TestCase):
506 self.pm = pollmixin.PollMixin()
508 def test_PollMixin_True(self):
509 d = self.pm.poll(check_f=lambda : True,
513 def test_PollMixin_False_then_True(self):
514 i = iter([False, True])
515 d = self.pm.poll(check_f=i.next,
519 def test_timeout(self):
520 d = self.pm.poll(check_f=lambda: False,
524 self.fail("poll should have failed, not returned %s" % (res,))
526 f.trap(pollmixin.TimeoutError)
527 return None # success
528 d.addCallbacks(_suc, _err)
531 class DeferredUtilTests(unittest.TestCase):
532 def test_gather_results(self):
533 d1 = defer.Deferred()
534 d2 = defer.Deferred()
535 res = deferredutil.gatherResults([d1, d2])
536 d1.errback(ValueError("BAD"))
538 self.fail("Should have errbacked, not resulted in %s" % (res,))
540 thef.trap(ValueError)
541 res.addCallbacks(_callb, _errb)
544 def test_success(self):
545 d1, d2 = defer.Deferred(), defer.Deferred()
548 dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
549 dlss.addCallbacks(good.append, bad.append)
552 self.failUnlessEqual(good, [[1,2]])
553 self.failUnlessEqual(bad, [])
555 def test_failure(self):
556 d1, d2 = defer.Deferred(), defer.Deferred()
559 dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
560 dlss.addCallbacks(good.append, bad.append)
561 d1.addErrback(lambda _ignore: None)
562 d2.addErrback(lambda _ignore: None)
564 d2.errback(ValueError())
565 self.failUnlessEqual(good, [])
566 self.failUnlessEqual(len(bad), 1)
568 self.failUnless(isinstance(f, Failure))
569 self.failUnless(f.check(ValueError))
571 class HashUtilTests(unittest.TestCase):
573 def test_random_key(self):
574 k = hashutil.random_key()
575 self.failUnlessEqual(len(k), hashutil.KEYLEN)
577 def test_sha256d(self):
578 h1 = hashutil.tagged_hash("tag1", "value")
579 h2 = hashutil.tagged_hasher("tag1")
583 self.failUnlessEqual(h1, h2a)
584 self.failUnlessEqual(h2a, h2b)
586 def test_sha256d_truncated(self):
587 h1 = hashutil.tagged_hash("tag1", "value", 16)
588 h2 = hashutil.tagged_hasher("tag1", 16)
591 self.failUnlessEqual(len(h1), 16)
592 self.failUnlessEqual(len(h2), 16)
593 self.failUnlessEqual(h1, h2)
596 h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret")
597 h2 = hashutil.convergence_hasher(3, 10, 1000, "secret")
600 self.failUnlessEqual(h1, h2)
602 def test_hashers(self):
603 h1 = hashutil.block_hash("foo")
604 h2 = hashutil.block_hasher()
606 self.failUnlessEqual(h1, h2.digest())
608 h1 = hashutil.uri_extension_hash("foo")
609 h2 = hashutil.uri_extension_hasher()
611 self.failUnlessEqual(h1, h2.digest())
613 h1 = hashutil.plaintext_hash("foo")
614 h2 = hashutil.plaintext_hasher()
616 self.failUnlessEqual(h1, h2.digest())
618 h1 = hashutil.crypttext_hash("foo")
619 h2 = hashutil.crypttext_hasher()
621 self.failUnlessEqual(h1, h2.digest())
623 h1 = hashutil.crypttext_segment_hash("foo")
624 h2 = hashutil.crypttext_segment_hasher()
626 self.failUnlessEqual(h1, h2.digest())
628 h1 = hashutil.plaintext_segment_hash("foo")
629 h2 = hashutil.plaintext_segment_hasher()
631 self.failUnlessEqual(h1, h2.digest())
633 def test_constant_time_compare(self):
634 self.failUnless(hashutil.constant_time_compare("a", "a"))
635 self.failUnless(hashutil.constant_time_compare("ab", "ab"))
636 self.failIf(hashutil.constant_time_compare("a", "b"))
637 self.failIf(hashutil.constant_time_compare("a", "aa"))
639 def _testknown(self, hashf, expected_a, *args):
641 got_a = base32.b2a(got)
642 self.failUnlessEqual(got_a, expected_a)
644 def test_known_answers(self):
645 # assert backwards compatibility
646 self._testknown(hashutil.storage_index_hash, "qb5igbhcc5esa6lwqorsy7e6am", "")
647 self._testknown(hashutil.block_hash, "msjr5bh4evuh7fa3zw7uovixfbvlnstr5b65mrerwfnvjxig2jvq", "")
648 self._testknown(hashutil.uri_extension_hash, "wthsu45q7zewac2mnivoaa4ulh5xvbzdmsbuyztq2a5fzxdrnkka", "")
649 self._testknown(hashutil.plaintext_hash, "5lz5hwz3qj3af7n6e3arblw7xzutvnd3p3fjsngqjcb7utf3x3da", "")
650 self._testknown(hashutil.crypttext_hash, "itdj6e4njtkoiavlrmxkvpreosscssklunhwtvxn6ggho4rkqwga", "")
651 self._testknown(hashutil.crypttext_segment_hash, "aovy5aa7jej6ym5ikgwyoi4pxawnoj3wtaludjz7e2nb5xijb7aa", "")
652 self._testknown(hashutil.plaintext_segment_hash, "4fdgf6qruaisyukhqcmoth4t3li6bkolbxvjy4awwcpprdtva7za", "")
653 self._testknown(hashutil.convergence_hash, "3mo6ni7xweplycin6nowynw2we", 3, 10, 100, "", "converge")
654 self._testknown(hashutil.my_renewal_secret_hash, "ujhr5k5f7ypkp67jkpx6jl4p47pyta7hu5m527cpcgvkafsefm6q", "")
655 self._testknown(hashutil.my_cancel_secret_hash, "rjwzmafe2duixvqy6h47f5wfrokdziry6zhx4smew4cj6iocsfaa", "")
656 self._testknown(hashutil.file_renewal_secret_hash, "hzshk2kf33gzbd5n3a6eszkf6q6o6kixmnag25pniusyaulqjnia", "", "si")
657 self._testknown(hashutil.file_cancel_secret_hash, "bfciwvr6w7wcavsngxzxsxxaszj72dej54n4tu2idzp6b74g255q", "", "si")
658 self._testknown(hashutil.bucket_renewal_secret_hash, "e7imrzgzaoashsncacvy3oysdd2m5yvtooo4gmj4mjlopsazmvuq", "", "\x00"*20)
659 self._testknown(hashutil.bucket_cancel_secret_hash, "dvdujeyxeirj6uux6g7xcf4lvesk632aulwkzjar7srildvtqwma", "", "\x00"*20)
660 self._testknown(hashutil.hmac, "c54ypfi6pevb3nvo6ba42jtglpkry2kbdopqsi7dgrm4r7tw5sra", "tag", "")
661 self._testknown(hashutil.mutable_rwcap_key_hash, "6rvn2iqrghii5n4jbbwwqqsnqu", "iv", "wk")
662 self._testknown(hashutil.ssk_writekey_hash, "ykpgmdbpgbb6yqz5oluw2q26ye", "")
663 self._testknown(hashutil.ssk_write_enabler_master_hash, "izbfbfkoait4dummruol3gy2bnixrrrslgye6ycmkuyujnenzpia", "")
664 self._testknown(hashutil.ssk_write_enabler_hash, "fuu2dvx7g6gqu5x22vfhtyed7p4pd47y5hgxbqzgrlyvxoev62tq", "wk", "\x00"*20)
665 self._testknown(hashutil.ssk_pubkey_fingerprint_hash, "3opzw4hhm2sgncjx224qmt5ipqgagn7h5zivnfzqycvgqgmgz35q", "")
666 self._testknown(hashutil.ssk_readkey_hash, "vugid4as6qbqgeq2xczvvcedai", "")
667 self._testknown(hashutil.ssk_readkey_data_hash, "73wsaldnvdzqaf7v4pzbr2ae5a", "iv", "rk")
668 self._testknown(hashutil.ssk_storage_index_hash, "j7icz6kigb6hxrej3tv4z7ayym", "")
671 class Abbreviate(unittest.TestCase):
673 a = abbreviate.abbreviate_time
674 self.failUnlessEqual(a(None), "unknown")
675 self.failUnlessEqual(a(0), "0 seconds")
676 self.failUnlessEqual(a(1), "1 second")
677 self.failUnlessEqual(a(2), "2 seconds")
678 self.failUnlessEqual(a(119), "119 seconds")
680 self.failUnlessEqual(a(2*MIN), "2 minutes")
681 self.failUnlessEqual(a(60*MIN), "60 minutes")
682 self.failUnlessEqual(a(179*MIN), "179 minutes")
684 self.failUnlessEqual(a(180*MIN), "3 hours")
685 self.failUnlessEqual(a(4*HOUR), "4 hours")
688 self.failUnlessEqual(a(2*DAY), "2 days")
689 self.failUnlessEqual(a(2*MONTH), "2 months")
691 self.failUnlessEqual(a(5*YEAR), "5 years")
693 def test_space(self):
694 tests_si = [(None, "unknown"),
701 (20*1000, "20.00 kB"),
702 (1024*1024, "1.05 MB"),
703 (1000*1000, "1.00 MB"),
704 (1000*1000*1000, "1.00 GB"),
705 (1000*1000*1000*1000, "1.00 TB"),
706 (1000*1000*1000*1000*1000, "1.00 PB"),
707 (1234567890123456, "1.23 PB"),
709 for (x, expected) in tests_si:
710 got = abbreviate.abbreviate_space(x, SI=True)
711 self.failUnlessEqual(got, expected)
713 tests_base1024 = [(None, "unknown"),
720 (20*1024, "20.00 kiB"),
721 (1000*1000, "976.56 kiB"),
722 (1024*1024, "1.00 MiB"),
723 (1024*1024*1024, "1.00 GiB"),
724 (1024*1024*1024*1024, "1.00 TiB"),
725 (1000*1000*1000*1000*1000, "909.49 TiB"),
726 (1024*1024*1024*1024*1024, "1.00 PiB"),
727 (1234567890123456, "1.10 PiB"),
729 for (x, expected) in tests_base1024:
730 got = abbreviate.abbreviate_space(x, SI=False)
731 self.failUnlessEqual(got, expected)
733 self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567),
734 "(1.23 MB, 1.18 MiB)")
736 def test_parse_space(self):
737 p = abbreviate.parse_abbreviated_size
738 self.failUnlessEqual(p(""), None)
739 self.failUnlessEqual(p(None), None)
740 self.failUnlessEqual(p("123"), 123)
741 self.failUnlessEqual(p("123B"), 123)
742 self.failUnlessEqual(p("2K"), 2000)
743 self.failUnlessEqual(p("2kb"), 2000)
744 self.failUnlessEqual(p("2KiB"), 2048)
745 self.failUnlessEqual(p("10MB"), 10*1000*1000)
746 self.failUnlessEqual(p("10MiB"), 10*1024*1024)
747 self.failUnlessEqual(p("5G"), 5*1000*1000*1000)
748 self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024)
749 e = self.failUnlessRaises(ValueError, p, "12 cubits")
750 self.failUnless("12 cubits" in str(e))
752 class Limiter(unittest.TestCase):
753 timeout = 480 # This takes longer than 240 seconds on Francois's arm box.
755 def job(self, i, foo):
756 self.calls.append( (i, foo) )
757 self.simultaneous += 1
758 self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous)
761 self.simultaneous -= 1
762 d.callback("done %d" % i)
763 reactor.callLater(1.0, _done)
766 def bad_job(self, i, foo):
767 raise ValueError("bad_job %d" % i)
769 def test_limiter(self):
771 self.simultaneous = 0
772 self.peak_simultaneous = 0
773 l = limiter.ConcurrencyLimiter()
776 dl.append(l.add(self.job, i, foo=str(i)))
777 d = defer.DeferredList(dl, fireOnOneErrback=True)
779 self.failUnlessEqual(self.simultaneous, 0)
780 self.failUnless(self.peak_simultaneous <= 10)
781 self.failUnlessEqual(len(self.calls), 20)
783 self.failUnless( (i, str(i)) in self.calls)
787 def test_errors(self):
789 self.simultaneous = 0
790 self.peak_simultaneous = 0
791 l = limiter.ConcurrencyLimiter()
794 dl.append(l.add(self.job, i, foo=str(i)))
795 d2 = l.add(self.bad_job, 21, "21")
796 d = defer.DeferredList(dl, fireOnOneErrback=True)
799 for (success, result) in res:
800 self.failUnlessEqual(success, True)
801 results.append(result)
803 expected_results = ["done %d" % i for i in range(20)]
804 expected_results.sort()
805 self.failUnlessEqual(results, expected_results)
806 self.failUnless(self.peak_simultaneous <= 10)
807 self.failUnlessEqual(len(self.calls), 20)
809 self.failUnless( (i, str(i)) in self.calls)
811 self.fail("should have failed, not got %s" % (res,))
814 self.failUnless("bad_job 21" in str(f))
815 d2.addCallbacks(_good, _err)
817 d.addCallback(_most_done)
819 self.failUnlessEqual(self.simultaneous, 0)
820 self.failUnless(self.peak_simultaneous <= 10)
821 self.failUnlessEqual(len(self.calls), 20)
823 self.failUnless( (i, str(i)) in self.calls)
824 d.addCallback(_all_done)
827 class TimeFormat(unittest.TestCase):
828 def test_epoch(self):
829 return self._help_test_epoch()
831 def test_epoch_in_London(self):
832 # Europe/London is a particularly troublesome timezone. Nowadays, its
833 # offset from GMT is 0. But in 1970, its offset from GMT was 1.
834 # (Apparently in 1970 Britain had redefined standard time to be GMT+1
835 # and stayed in standard time all year round, whereas today
836 # Europe/London standard time is GMT and Europe/London Daylight
837 # Savings Time is GMT+1.) The current implementation of
838 # time_format.iso_utc_time_to_localseconds() breaks if the timezone is
839 # Europe/London. (As soon as this unit test is done then I'll change
840 # that implementation to something that works even in this case...)
841 origtz = os.environ.get('TZ')
842 os.environ['TZ'] = "Europe/London"
843 if hasattr(time, 'tzset'):
846 return self._help_test_epoch()
851 os.environ['TZ'] = origtz
852 if hasattr(time, 'tzset'):
855 def _help_test_epoch(self):
856 origtzname = time.tzname
857 s = time_format.iso_utc_time_to_seconds("1970-01-01T00:00:01")
858 self.failUnlessEqual(s, 1.0)
859 s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01")
860 self.failUnlessEqual(s, 1.0)
861 s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01")
862 self.failUnlessEqual(s, 1.0)
864 self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01")
865 self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "),
866 "1970-01-01 00:00:01")
869 isostr = time_format.iso_utc(now)
870 timestamp = time_format.iso_utc_time_to_seconds(isostr)
871 self.failUnlessEqual(int(timestamp), int(now))
875 self.failUnlessEqual(time_format.iso_utc(t=my_time),
876 "1970-01-01_00:00:01")
877 e = self.failUnlessRaises(ValueError,
878 time_format.iso_utc_time_to_seconds,
879 "invalid timestring")
880 self.failUnless("not a complete ISO8601 timestamp" in str(e))
881 s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01.500")
882 self.failUnlessEqual(s, 1.5)
884 # Look for daylight-savings-related errors.
885 thatmomentinmarch = time_format.iso_utc_time_to_seconds("2009-03-20 21:49:02.226536")
886 self.failUnlessEqual(thatmomentinmarch, 1237585742.226536)
887 self.failUnlessEqual(origtzname, time.tzname)
889 def test_iso_utc(self):
890 when = 1266760143.7841301
891 out = time_format.iso_utc_date(when)
892 self.failUnlessEqual(out, "2010-02-21")
893 out = time_format.iso_utc_date(t=lambda: when)
894 self.failUnlessEqual(out, "2010-02-21")
895 out = time_format.iso_utc(when)
896 self.failUnlessEqual(out, "2010-02-21_13:49:03.784130")
897 out = time_format.iso_utc(when, sep="-")
898 self.failUnlessEqual(out, "2010-02-21-13:49:03.784130")
900 def test_parse_duration(self):
901 p = time_format.parse_duration
903 self.failUnlessEqual(p("1 day"), DAY)
904 self.failUnlessEqual(p("2 days"), 2*DAY)
905 self.failUnlessEqual(p("3 months"), 3*31*DAY)
906 self.failUnlessEqual(p("4 mo"), 4*31*DAY)
907 self.failUnlessEqual(p("5 years"), 5*365*DAY)
908 e = self.failUnlessRaises(ValueError, p, "123")
909 self.failUnlessIn("no unit (like day, month, or year) in '123'",
912 def test_parse_date(self):
913 self.failUnlessEqual(time_format.parse_date("2010-02-21"), 1266710400)
915 class CacheDir(unittest.TestCase):
916 def test_basic(self):
917 basedir = "test_util/CacheDir/test_basic"
919 def _failIfExists(name):
920 absfn = os.path.join(basedir, name)
921 self.failIf(os.path.exists(absfn),
922 "%s exists but it shouldn't" % absfn)
924 def _failUnlessExists(name):
925 absfn = os.path.join(basedir, name)
926 self.failUnless(os.path.exists(absfn),
927 "%s doesn't exist but it should" % absfn)
929 cdm = cachedir.CacheDirectoryManager(basedir)
930 a = cdm.get_file("a")
931 b = cdm.get_file("b")
932 c = cdm.get_file("c")
933 f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f
934 f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f
935 f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f
937 _failUnlessExists("a")
938 _failUnlessExists("b")
939 _failUnlessExists("c")
943 _failUnlessExists("a")
944 _failUnlessExists("b")
945 _failUnlessExists("c")
948 # this file won't be deleted yet, because it isn't old enough
950 _failUnlessExists("a")
951 _failUnlessExists("b")
952 _failUnlessExists("c")
954 # we change the definition of "old" to make everything old
959 _failUnlessExists("b")
960 _failUnlessExists("c")
968 _failUnlessExists("b")
969 _failUnlessExists("c")
971 b2 = cdm.get_file("b")
975 _failUnlessExists("b")
976 _failUnlessExists("c")
981 def __init__(self, x):
986 return "<%s %s>" % (self.__class__.__name__, self.x,)
989 def __le__(self, other):
990 return self.x <= other
991 def __lt__(self, other):
992 return self.x < other
993 def __ge__(self, other):
994 return self.x >= other
995 def __gt__(self, other):
996 return self.x > other
997 def __ne__(self, other):
998 return self.x != other
999 def __eq__(self, other):
1000 return self.x == other
1002 class DictUtil(unittest.TestCase):
1003 def _help_test_empty_dict(self, klass):
1007 self.failUnless(d1 == d2, "d1: %r, d2: %r" % (d1, d2,))
1008 self.failUnless(len(d1) == 0)
1009 self.failUnless(len(d2) == 0)
1011 def _help_test_nonempty_dict(self, klass):
1012 d1 = klass({'a': 1, 'b': "eggs", 3: "spam",})
1013 d2 = klass({'a': 1, 'b': "eggs", 3: "spam",})
1015 self.failUnless(d1 == d2)
1016 self.failUnless(len(d1) == 3, "%s, %s" % (len(d1), d1,))
1017 self.failUnless(len(d2) == 3)
1019 def _help_test_eq_but_notis(self, klass):
1020 d = klass({'a': 3, 'b': EqButNotIs(3), 'c': 3})
1025 d['b'] = EqButNotIs(3)
1030 d['b'] = EqButNotIs(3)
1036 d['a'] = EqButNotIs(3)
1041 fake3 = EqButNotIs(3)
1042 fake7 = EqButNotIs(7)
1046 self.failUnless(filter(lambda x: x is 8, d.itervalues()))
1047 self.failUnless(filter(lambda x: x is fake7, d.itervalues()))
1048 # The real 7 should have been ejected by the d[3] = 8.
1049 self.failUnless(not filter(lambda x: x is 7, d.itervalues()))
1050 self.failUnless(filter(lambda x: x is fake3, d.iterkeys()))
1051 self.failUnless(filter(lambda x: x is 3, d.iterkeys()))
1056 fake3 = EqButNotIs(3)
1057 fake7 = EqButNotIs(7)
1060 self.failUnless(filter(lambda x: x is 8, d.itervalues()))
1061 self.failUnless(filter(lambda x: x is fake7, d.itervalues()))
1062 # The real 7 should have been ejected by the d[3] = 8.
1063 self.failUnless(not filter(lambda x: x is 7, d.itervalues()))
1064 self.failUnless(filter(lambda x: x is fake3, d.iterkeys()))
1065 self.failUnless(filter(lambda x: x is 3, d.iterkeys()))
1069 self._help_test_eq_but_notis(dictutil.UtilDict)
1070 self._help_test_eq_but_notis(dictutil.NumDict)
1071 self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
1072 self._help_test_nonempty_dict(dictutil.UtilDict)
1073 self._help_test_nonempty_dict(dictutil.NumDict)
1074 self._help_test_nonempty_dict(dictutil.ValueOrderedDict)
1075 self._help_test_eq_but_notis(dictutil.UtilDict)
1076 self._help_test_eq_but_notis(dictutil.NumDict)
1077 self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
1079 def test_dict_of_sets(self):
1080 ds = dictutil.DictOfSets()
1085 self.failUnlessEqual(ds[1], set(["a"]))
1086 self.failUnlessEqual(ds[2], set(["b", "c"]))
1087 ds.discard(3, "d") # should not raise an exception
1089 self.failUnlessEqual(ds[2], set(["c"]))
1091 self.failIf(2 in ds)
1093 ds.union(1, ["a", "e"])
1095 self.failUnlessEqual(ds[1], set(["a","e"]))
1096 self.failUnlessEqual(ds[3], set(["f"]))
1097 ds2 = dictutil.DictOfSets()
1102 self.failUnlessEqual(ds[1], set(["a","e"]))
1103 self.failUnlessEqual(ds[3], set(["f", "g"]))
1104 self.failUnlessEqual(ds[4], set(["h"]))
1106 def test_move(self):
1107 d1 = {1: "a", 2: "b"}
1108 d2 = {2: "c", 3: "d"}
1109 dictutil.move(1, d1, d2)
1110 self.failUnlessEqual(d1, {2: "b"})
1111 self.failUnlessEqual(d2, {1: "a", 2: "c", 3: "d"})
1113 d1 = {1: "a", 2: "b"}
1114 d2 = {2: "c", 3: "d"}
1115 dictutil.move(2, d1, d2)
1116 self.failUnlessEqual(d1, {1: "a"})
1117 self.failUnlessEqual(d2, {2: "b", 3: "d"})
1119 d1 = {1: "a", 2: "b"}
1120 d2 = {2: "c", 3: "d"}
1121 self.failUnlessRaises(KeyError, dictutil.move, 5, d1, d2, strict=True)
1123 def test_subtract(self):
1124 d1 = {1: "a", 2: "b"}
1125 d2 = {2: "c", 3: "d"}
1126 d3 = dictutil.subtract(d1, d2)
1127 self.failUnlessEqual(d3, {1: "a"})
1129 d1 = {1: "a", 2: "b"}
1131 d3 = dictutil.subtract(d1, d2)
1132 self.failUnlessEqual(d3, {1: "a"})
1134 def test_utildict(self):
1135 d = dictutil.UtilDict({1: "a", 2: "b"})
1138 self.failUnlessEqual(d, {2: "b"})
1141 self.failUnlessRaises(TypeError, eq, d, "not a dict")
1143 d = dictutil.UtilDict({1: "b", 2: "a"})
1144 self.failUnlessEqual(d.items_sorted_by_value(),
1145 [(2, "a"), (1, "b")])
1146 self.failUnlessEqual(d.items_sorted_by_key(),
1147 [(1, "b"), (2, "a")])
1148 self.failUnlessEqual(repr(d), "{1: 'b', 2: 'a'}")
1149 self.failUnless(1 in d)
1151 d2 = dictutil.UtilDict({3: "c", 4: "d"})
1152 self.failUnless(d != d2)
1153 self.failUnless(d2 > d)
1154 self.failUnless(d2 >= d)
1155 self.failUnless(d <= d2)
1156 self.failUnless(d < d2)
1157 self.failUnlessEqual(d[1], "b")
1158 self.failUnlessEqual(sorted(list([k for k in d])), [1,2])
1161 self.failUnlessEqual(d, d3)
1162 self.failUnless(isinstance(d3, dictutil.UtilDict))
1164 d4 = d.fromkeys([3,4], "e")
1165 self.failUnlessEqual(d4, {3: "e", 4: "e"})
1167 self.failUnlessEqual(d.get(1), "b")
1168 self.failUnlessEqual(d.get(3), None)
1169 self.failUnlessEqual(d.get(3, "default"), "default")
1170 self.failUnlessEqual(sorted(list(d.items())),
1171 [(1, "b"), (2, "a")])
1172 self.failUnlessEqual(sorted(list(d.iteritems())),
1173 [(1, "b"), (2, "a")])
1174 self.failUnlessEqual(sorted(d.keys()), [1, 2])
1175 self.failUnlessEqual(sorted(d.values()), ["a", "b"])
1176 x = d.setdefault(1, "new")
1177 self.failUnlessEqual(x, "b")
1178 self.failUnlessEqual(d[1], "b")
1179 x = d.setdefault(3, "new")
1180 self.failUnlessEqual(x, "new")
1181 self.failUnlessEqual(d[3], "new")
1185 self.failUnless(x in [(1, "b"), (2, "a")])
1187 self.failUnless(x in [(1, "b"), (2, "a")])
1188 self.failUnlessRaises(KeyError, d.popitem)
1190 def test_numdict(self):
1191 d = dictutil.NumDict({"a": 1, "b": 2})
1193 d.add_num("a", 10, 5)
1194 d.add_num("c", 20, 5)
1196 self.failUnlessEqual(d, {"a": 11, "b": 2, "c": 25, "d": 30})
1198 d.subtract_num("a", 10)
1199 d.subtract_num("e", 10)
1200 d.subtract_num("f", 10, 15)
1201 self.failUnlessEqual(d, {"a": 1, "b": 2, "c": 25, "d": 30,
1204 self.failUnlessEqual(d.sum(), sum([1, 2, 25, 30, -10, 5]))
1206 d = dictutil.NumDict()
1210 self.failUnlessEqual(d, {"a": 2, "b": 6})
1214 self.failUnlessEqual(d, {"a": 1, "b": 6, "c": -1, "d": 4})
1215 self.failUnlessEqual(d.items_sorted_by_key(),
1216 [("a", 1), ("b", 6), ("c", -1), ("d", 4)])
1217 self.failUnlessEqual(d.items_sorted_by_value(),
1218 [("c", -1), ("a", 1), ("d", 4), ("b", 6)])
1219 self.failUnlessEqual(d.item_with_largest_value(), ("b", 6))
1221 d = dictutil.NumDict({"a": 1, "b": 2})
1222 self.failUnlessEqual(repr(d), "{'a': 1, 'b': 2}")
1223 self.failUnless("a" in d)
1225 d2 = dictutil.NumDict({"c": 3, "d": 4})
1226 self.failUnless(d != d2)
1227 self.failUnless(d2 > d)
1228 self.failUnless(d2 >= d)
1229 self.failUnless(d <= d2)
1230 self.failUnless(d < d2)
1231 self.failUnlessEqual(d["a"], 1)
1232 self.failUnlessEqual(sorted(list([k for k in d])), ["a","b"])
1235 self.failUnlessRaises(TypeError, eq, d, "not a dict")
1238 self.failUnlessEqual(d, d3)
1239 self.failUnless(isinstance(d3, dictutil.NumDict))
1241 d4 = d.fromkeys(["a","b"], 5)
1242 self.failUnlessEqual(d4, {"a": 5, "b": 5})
1244 self.failUnlessEqual(d.get("a"), 1)
1245 self.failUnlessEqual(d.get("c"), 0)
1246 self.failUnlessEqual(d.get("c", 5), 5)
1247 self.failUnlessEqual(sorted(list(d.items())),
1248 [("a", 1), ("b", 2)])
1249 self.failUnlessEqual(sorted(list(d.iteritems())),
1250 [("a", 1), ("b", 2)])
1251 self.failUnlessEqual(sorted(d.keys()), ["a", "b"])
1252 self.failUnlessEqual(sorted(d.values()), [1, 2])
1253 self.failUnless(d.has_key("a"))
1254 self.failIf(d.has_key("c"))
1256 x = d.setdefault("c", 3)
1257 self.failUnlessEqual(x, 3)
1258 self.failUnlessEqual(d["c"], 3)
1259 x = d.setdefault("c", 5)
1260 self.failUnlessEqual(x, 3)
1261 self.failUnlessEqual(d["c"], 3)
1265 self.failUnless(x in [("a", 1), ("b", 2)])
1267 self.failUnless(x in [("a", 1), ("b", 2)])
1268 self.failUnlessRaises(KeyError, d.popitem)
1271 d.update({"c": 4, "d": 5})
1272 self.failUnlessEqual(d, {"c": 4, "d": 5})
1274 def test_del_if_present(self):
1275 d = {1: "a", 2: "b"}
1276 dictutil.del_if_present(d, 1)
1277 dictutil.del_if_present(d, 3)
1278 self.failUnlessEqual(d, {2: "b"})
1280 def test_valueordereddict(self):
1281 d = dictutil.ValueOrderedDict()
1286 self.failUnlessEqual(d, {"a": 3, "b": 2, "c": 1})
1287 self.failUnlessEqual(d.items(), [("c", 1), ("b", 2), ("a", 3)])
1288 self.failUnlessEqual(d.values(), [1, 2, 3])
1289 self.failUnlessEqual(d.keys(), ["c", "b", "a"])
1290 self.failUnlessEqual(repr(d), "<ValueOrderedDict {c: 1, b: 2, a: 3}>")
1293 self.failIf(d == {"a": 4})
1294 self.failUnless(d != {"a": 4})
1296 x = d.setdefault("d", 0)
1297 self.failUnlessEqual(x, 0)
1298 self.failUnlessEqual(d["d"], 0)
1299 x = d.setdefault("d", -1)
1300 self.failUnlessEqual(x, 0)
1301 self.failUnlessEqual(d["d"], 0)
1303 x = d.remove("e", "default", False)
1304 self.failUnlessEqual(x, "default")
1305 self.failUnlessRaises(KeyError, d.remove, "e", "default", True)
1306 x = d.remove("d", 5)
1307 self.failUnlessEqual(x, 0)
1309 x = d.__getitem__("c")
1310 self.failUnlessEqual(x, 1)
1311 x = d.__getitem__("e", "default", False)
1312 self.failUnlessEqual(x, "default")
1313 self.failUnlessRaises(KeyError, d.__getitem__, "e", "default", True)
1315 self.failUnlessEqual(d.popitem(), ("c", 1))
1316 self.failUnlessEqual(d.popitem(), ("b", 2))
1317 self.failUnlessEqual(d.popitem(), ("a", 3))
1318 self.failUnlessRaises(KeyError, d.popitem)
1320 d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1321 x = d.pop("d", "default", False)
1322 self.failUnlessEqual(x, "default")
1323 self.failUnlessRaises(KeyError, d.pop, "d", "default", True)
1325 self.failUnlessEqual(x, 2)
1326 self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1328 d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1329 x = d.pop_from_list(1) # pop the second item, b/2
1330 self.failUnlessEqual(x, "b")
1331 self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1333 def test_auxdict(self):
1334 d = dictutil.AuxValueDict()
1335 # we put the serialized form in the auxdata
1336 d.set_with_aux("key", ("filecap", "metadata"), "serialized")
1338 self.failUnlessEqual(d.keys(), ["key"])
1339 self.failUnlessEqual(d["key"], ("filecap", "metadata"))
1340 self.failUnlessEqual(d.get_aux("key"), "serialized")
1341 def _get_missing(key):
1343 self.failUnlessRaises(KeyError, _get_missing, "nonkey")
1344 self.failUnlessEqual(d.get("nonkey"), None)
1345 self.failUnlessEqual(d.get("nonkey", "nonvalue"), "nonvalue")
1346 self.failUnlessEqual(d.get_aux("nonkey"), None)
1347 self.failUnlessEqual(d.get_aux("nonkey", "nonvalue"), "nonvalue")
1349 d["key"] = ("filecap2", "metadata2")
1350 self.failUnlessEqual(d["key"], ("filecap2", "metadata2"))
1351 self.failUnlessEqual(d.get_aux("key"), None)
1353 d.set_with_aux("key2", "value2", "aux2")
1354 self.failUnlessEqual(sorted(d.keys()), ["key", "key2"])
1356 self.failUnlessEqual(d.keys(), ["key"])
1357 self.failIf("key2" in d)
1358 self.failUnlessRaises(KeyError, _get_missing, "key2")
1359 self.failUnlessEqual(d.get("key2"), None)
1360 self.failUnlessEqual(d.get_aux("key2"), None)
1361 d["key2"] = "newvalue2"
1362 self.failUnlessEqual(d.get("key2"), "newvalue2")
1363 self.failUnlessEqual(d.get_aux("key2"), None)
1365 d = dictutil.AuxValueDict({1:2,3:4})
1366 self.failUnlessEqual(sorted(d.keys()), [1,3])
1367 self.failUnlessEqual(d[1], 2)
1368 self.failUnlessEqual(d.get_aux(1), None)
1370 d = dictutil.AuxValueDict([ (1,2), (3,4) ])
1371 self.failUnlessEqual(sorted(d.keys()), [1,3])
1372 self.failUnlessEqual(d[1], 2)
1373 self.failUnlessEqual(d.get_aux(1), None)
1375 d = dictutil.AuxValueDict(one=1, two=2)
1376 self.failUnlessEqual(sorted(d.keys()), ["one","two"])
1377 self.failUnlessEqual(d["one"], 1)
1378 self.failUnlessEqual(d.get_aux("one"), None)
1380 class Pipeline(unittest.TestCase):
1381 def pause(self, *args, **kwargs):
1382 d = defer.Deferred()
1383 self.calls.append( (d, args, kwargs) )
1386 def failUnlessCallsAre(self, expected):
1389 self.failUnlessEqual(len(self.calls), len(expected), self.calls)
1390 for i,c in enumerate(self.calls):
1391 self.failUnlessEqual(c[1:], expected[i], str(i))
1393 def test_basic(self):
1396 p = pipeline.Pipeline(100)
1398 d = p.flush() # fires immediately
1399 d.addCallbacks(finished.append, log.err)
1400 self.failUnlessEqual(len(finished), 1)
1403 d = p.add(10, self.pause, "one")
1404 # the call should start right away, and our return Deferred should
1406 d.addCallbacks(finished.append, log.err)
1407 self.failUnlessEqual(len(finished), 1)
1408 self.failUnlessEqual(finished[0], None)
1409 self.failUnlessCallsAre([ ( ("one",) , {} ) ])
1410 self.failUnlessEqual(p.gauge, 10)
1415 d = p.add(20, self.pause, "two", kw=2)
1416 # pipeline: [one, two]
1418 # the call and the Deferred should fire right away
1419 d.addCallbacks(finished.append, log.err)
1420 self.failUnlessEqual(len(finished), 1)
1421 self.failUnlessEqual(finished[0], None)
1422 self.failUnlessCallsAre([ ( ("one",) , {} ),
1423 ( ("two",) , {"kw": 2} ),
1425 self.failUnlessEqual(p.gauge, 30)
1427 self.calls[0][0].callback("one-result")
1429 self.failUnlessEqual(p.gauge, 20)
1432 d = p.add(90, self.pause, "three", "posarg1")
1433 # pipeline: [two, three]
1436 fd.addCallbacks(flushed.append, log.err)
1437 self.failUnlessEqual(flushed, [])
1439 # the call will be made right away, but the return Deferred will not,
1440 # because the pipeline is now full.
1441 d.addCallbacks(finished.append, log.err)
1442 self.failUnlessEqual(len(finished), 0)
1443 self.failUnlessCallsAre([ ( ("one",) , {} ),
1444 ( ("two",) , {"kw": 2} ),
1445 ( ("three", "posarg1"), {} ),
1447 self.failUnlessEqual(p.gauge, 110)
1449 self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause)
1451 # retiring either call will unblock the pipeline, causing the #3
1453 self.calls[2][0].callback("three-result")
1456 self.failUnlessEqual(len(finished), 1)
1457 self.failUnlessEqual(finished[0], None)
1458 self.failUnlessEqual(flushed, [])
1460 # retiring call#2 will finally allow the flush() Deferred to fire
1461 self.calls[1][0].callback("two-result")
1462 self.failUnlessEqual(len(flushed), 1)
1464 def test_errors(self):
1466 p = pipeline.Pipeline(100)
1468 d1 = p.add(200, self.pause, "one")
1472 d1.addBoth(finished.append)
1473 self.failUnlessEqual(finished, [])
1476 d2.addBoth(flushed.append)
1477 self.failUnlessEqual(flushed, [])
1479 self.calls[0][0].errback(ValueError("oops"))
1481 self.failUnlessEqual(len(finished), 1)
1483 self.failUnless(isinstance(f, Failure))
1484 self.failUnless(f.check(pipeline.PipelineError))
1485 self.failUnlessIn("PipelineError", str(f.value))
1486 self.failUnlessIn("ValueError", str(f.value))
1488 self.failUnless("ValueError" in r, r)
1490 self.failUnless(f2.check(ValueError))
1492 self.failUnlessEqual(len(flushed), 1)
1494 self.failUnless(isinstance(f, Failure))
1495 self.failUnless(f.check(pipeline.PipelineError))
1497 self.failUnless(f2.check(ValueError))
1499 # now that the pipeline is in the failed state, any new calls will
1502 d3 = p.add(20, self.pause, "two")
1505 d3.addBoth(finished.append)
1506 self.failUnlessEqual(len(finished), 1)
1508 self.failUnless(isinstance(f, Failure))
1509 self.failUnless(f.check(pipeline.PipelineError))
1511 self.failUnless("ValueError" in r, r)
1513 self.failUnless(f2.check(ValueError))
1517 d4.addBoth(flushed.append)
1518 self.failUnlessEqual(len(flushed), 1)
1520 self.failUnless(isinstance(f, Failure))
1521 self.failUnless(f.check(pipeline.PipelineError))
1523 self.failUnless(f2.check(ValueError))
1525 def test_errors2(self):
1527 p = pipeline.Pipeline(100)
1529 d1 = p.add(10, self.pause, "one")
1530 d2 = p.add(20, self.pause, "two")
1531 d3 = p.add(30, self.pause, "three")
1534 # one call fails, then the second one succeeds: make sure
1535 # ExpandableDeferredList tolerates the second one
1538 d4.addBoth(flushed.append)
1539 self.failUnlessEqual(flushed, [])
1541 self.calls[0][0].errback(ValueError("oops"))
1542 self.failUnlessEqual(len(flushed), 1)
1544 self.failUnless(isinstance(f, Failure))
1545 self.failUnless(f.check(pipeline.PipelineError))
1547 self.failUnless(f2.check(ValueError))
1549 self.calls[1][0].callback("two-result")
1550 self.calls[2][0].errback(ValueError("three-error"))
1554 class SampleError(Exception):
1557 class Log(unittest.TestCase):
1559 if not hasattr(self, "flushLoggedErrors"):
1560 # without flushLoggedErrors, we can't get rid of the
1561 # twisted.log.err that tahoe_log records, so we can't keep this
1562 # test from [ERROR]ing
1563 raise unittest.SkipTest("needs flushLoggedErrors from Twisted-2.5.0")
1565 raise SampleError("simple sample")
1568 tahoe_log.err(format="intentional sample error",
1569 failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ")
1570 self.flushLoggedErrors(SampleError)