src/allmydata/test/test_util.py

   1
   2 def foo(): pass # keep the line number constant
   3
   4 import os, time
   5 from StringIO import StringIO
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.python.failure import Failure
   9 from twisted.python import log
  10
  11 from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
  12 from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
  13 from allmydata.util import limiter, time_format, pollmixin, cachedir
  14 from allmydata.util import statistics, dictutil, pipeline
  15 from allmydata.util import log as tahoe_log
  16
  17 class Base32(unittest.TestCase):
  18     def test_b2a_matches_Pythons(self):
  19         import base64
  20         y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0"
  21         x = base64.b32encode(y)
  22         while x and x[-1] == '=':
  23             x = x[:-1]
  24         x = x.lower()
  25         self.failUnlessEqual(base32.b2a(y), x)
  26     def test_b2a(self):
  27         self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a")
  28     def test_b2a_or_none(self):
  29         self.failUnlessEqual(base32.b2a_or_none(None), None)
  30         self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a")
  31     def test_a2b(self):
  32         self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34")
  33         self.failUnlessRaises(AssertionError, base32.a2b, "b0gus")
  34
  35 class IDLib(unittest.TestCase):
  36     def test_nodeid_b2a(self):
  37         self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
  38
  39 class NoArgumentException(Exception):
  40     def __init__(self):
  41         pass
  42
  43 class HumanReadable(unittest.TestCase):
  44     def test_repr(self):
  45         hr = humanreadable.hr
  46         self.failUnlessEqual(hr(foo), "<foo() at test_util.py:2>")
  47         self.failUnlessEqual(hr(self.test_repr),
  48                              "<bound method HumanReadable.test_repr of <allmydata.test.test_util.HumanReadable testMethod=test_repr>>")
  49         self.failUnlessEqual(hr(1L), "1")
  50         self.failUnlessEqual(hr(10**40),
  51                              "100000000000000000...000000000000000000")
  52         self.failUnlessEqual(hr(self), "<allmydata.test.test_util.HumanReadable testMethod=test_repr>")
  53         self.failUnlessEqual(hr([1,2]), "[1, 2]")
  54         self.failUnlessEqual(hr({1:2}), "{1:2}")
  55         try:
  56             raise ValueError
  57         except Exception, e:
  58             self.failUnless(
  59                 hr(e) == "<ValueError: ()>" # python-2.4
  60                 or hr(e) == "ValueError()") # python-2.5
  61         try:
  62             raise ValueError("oops")
  63         except Exception, e:
  64             self.failUnless(
  65                 hr(e) == "<ValueError: 'oops'>" # python-2.4
  66                 or hr(e) == "ValueError('oops',)") # python-2.5
  67         try:
  68             raise NoArgumentException
  69         except Exception, e:
  70             self.failUnless(
  71                 hr(e) == "<NoArgumentException>" # python-2.4
  72                 or hr(e) == "NoArgumentException()") # python-2.5
  73
  74
  75 class MyList(list):
  76     pass
  77
  78 class Math(unittest.TestCase):
  79     def test_div_ceil(self):
  80         f = mathutil.div_ceil
  81         self.failUnlessEqual(f(0, 1), 0)
  82         self.failUnlessEqual(f(0, 2), 0)
  83         self.failUnlessEqual(f(0, 3), 0)
  84         self.failUnlessEqual(f(1, 3), 1)
  85         self.failUnlessEqual(f(2, 3), 1)
  86         self.failUnlessEqual(f(3, 3), 1)
  87         self.failUnlessEqual(f(4, 3), 2)
  88         self.failUnlessEqual(f(5, 3), 2)
  89         self.failUnlessEqual(f(6, 3), 2)
  90         self.failUnlessEqual(f(7, 3), 3)
  91
  92     def test_next_multiple(self):
  93         f = mathutil.next_multiple
  94         self.failUnlessEqual(f(5, 1), 5)
  95         self.failUnlessEqual(f(5, 2), 6)
  96         self.failUnlessEqual(f(5, 3), 6)
  97         self.failUnlessEqual(f(5, 4), 8)
  98         self.failUnlessEqual(f(5, 5), 5)
  99         self.failUnlessEqual(f(5, 6), 6)
 100         self.failUnlessEqual(f(32, 1), 32)
 101         self.failUnlessEqual(f(32, 2), 32)
 102         self.failUnlessEqual(f(32, 3), 33)
 103         self.failUnlessEqual(f(32, 4), 32)
 104         self.failUnlessEqual(f(32, 5), 35)
 105         self.failUnlessEqual(f(32, 6), 36)
 106         self.failUnlessEqual(f(32, 7), 35)
 107         self.failUnlessEqual(f(32, 8), 32)
 108         self.failUnlessEqual(f(32, 9), 36)
 109         self.failUnlessEqual(f(32, 10), 40)
 110         self.failUnlessEqual(f(32, 11), 33)
 111         self.failUnlessEqual(f(32, 12), 36)
 112         self.failUnlessEqual(f(32, 13), 39)
 113         self.failUnlessEqual(f(32, 14), 42)
 114         self.failUnlessEqual(f(32, 15), 45)
 115         self.failUnlessEqual(f(32, 16), 32)
 116         self.failUnlessEqual(f(32, 17), 34)
 117         self.failUnlessEqual(f(32, 18), 36)
 118         self.failUnlessEqual(f(32, 589), 589)
 119
 120     def test_pad_size(self):
 121         f = mathutil.pad_size
 122         self.failUnlessEqual(f(0, 4), 0)
 123         self.failUnlessEqual(f(1, 4), 3)
 124         self.failUnlessEqual(f(2, 4), 2)
 125         self.failUnlessEqual(f(3, 4), 1)
 126         self.failUnlessEqual(f(4, 4), 0)
 127         self.failUnlessEqual(f(5, 4), 3)
 128
 129     def test_is_power_of_k(self):
 130         f = mathutil.is_power_of_k
 131         for i in range(1, 100):
 132             if i in (1, 2, 4, 8, 16, 32, 64):
 133                 self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
 134             else:
 135                 self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
 136         for i in range(1, 100):
 137             if i in (1, 3, 9, 27, 81):
 138                 self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
 139             else:
 140                 self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
 141
 142     def test_next_power_of_k(self):
 143         f = mathutil.next_power_of_k
 144         self.failUnlessEqual(f(0,2), 1)
 145         self.failUnlessEqual(f(1,2), 1)
 146         self.failUnlessEqual(f(2,2), 2)
 147         self.failUnlessEqual(f(3,2), 4)
 148         self.failUnlessEqual(f(4,2), 4)
 149         for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
 150         for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
 151         for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
 152         for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
 153         for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
 154
 155         self.failUnlessEqual(f(0,3), 1)
 156         self.failUnlessEqual(f(1,3), 1)
 157         self.failUnlessEqual(f(2,3), 3)
 158         self.failUnlessEqual(f(3,3), 3)
 159         for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
 160         for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
 161         for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
 162         for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
 163
 164     def test_ave(self):
 165         f = mathutil.ave
 166         self.failUnlessEqual(f([1,2,3]), 2)
 167         self.failUnlessEqual(f([0,0,0,4]), 1)
 168         self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
 169
 170     def test_round_sigfigs(self):
 171         f = mathutil.round_sigfigs
 172         self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002)
 173
 174 class Statistics(unittest.TestCase):
 175     def should_assert(self, msg, func, *args, **kwargs):
 176         try:
 177             func(*args, **kwargs)
 178             self.fail(msg)
 179         except AssertionError:
 180             pass
 181
 182     def failUnlessListEqual(self, a, b, msg = None):
 183         self.failUnlessEqual(len(a), len(b))
 184         for i in range(len(a)):
 185             self.failUnlessEqual(a[i], b[i], msg)
 186
 187     def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None):
 188         self.failUnlessEqual(len(a), len(b))
 189         for i in range(len(a)):
 190             self.failUnlessAlmostEqual(a[i], b[i], places, msg)
 191
 192     def test_binomial_coeff(self):
 193         f = statistics.binomial_coeff
 194         self.failUnlessEqual(f(20, 0), 1)
 195         self.failUnlessEqual(f(20, 1), 20)
 196         self.failUnlessEqual(f(20, 2), 190)
 197         self.failUnlessEqual(f(20, 8), f(20, 12))
 198         self.should_assert("Should assert if n < k", f, 2, 3)
 199
 200     def test_binomial_distribution_pmf(self):
 201         f = statistics.binomial_distribution_pmf
 202
 203         pmf_comp = f(2, .1)
 204         pmf_stat = [0.81, 0.18, 0.01]
 205         self.failUnlessListAlmostEqual(pmf_comp, pmf_stat)
 206
 207         # Summing across a PMF should give the total probability 1
 208         self.failUnlessAlmostEqual(sum(pmf_comp), 1)
 209         self.should_assert("Should assert if not 0<=p<=1", f, 1, -1)
 210         self.should_assert("Should assert if n < 1", f, 0, .1)
 211
 212         out = StringIO()
 213         statistics.print_pmf(pmf_comp, out=out)
 214         lines = out.getvalue().splitlines()
 215         self.failUnlessEqual(lines[0], "i=0: 0.81")
 216         self.failUnlessEqual(lines[1], "i=1: 0.18")
 217         self.failUnlessEqual(lines[2], "i=2: 0.01")
 218
 219     def test_survival_pmf(self):
 220         f = statistics.survival_pmf
 221         # Cross-check binomial-distribution method against convolution
 222         # method.
 223         p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20
 224         pmf1 = statistics.survival_pmf_via_conv(p_list)
 225         pmf2 = statistics.survival_pmf_via_bd(p_list)
 226         self.failUnlessListAlmostEqual(pmf1, pmf2)
 227         self.failUnlessTrue(statistics.valid_pmf(pmf1))
 228         self.should_assert("Should assert if p_i > 1", f, [1.1]);
 229         self.should_assert("Should assert if p_i < 0", f, [-.1]);
 230
 231     def test_repair_count_pmf(self):
 232         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 233         repair_pmf = statistics.repair_count_pmf(survival_pmf, 3)
 234         # repair_pmf[0] == sum(survival_pmf[0,1,2,5])
 235         # repair_pmf[1] == survival_pmf[4]
 236         # repair_pmf[2] = survival_pmf[3]
 237         self.failUnlessListAlmostEqual(repair_pmf,
 238                                        [0.00001 + 0.00045 + 0.0081 + 0.59049,
 239                                         .32805,
 240                                         .0729,
 241                                         0, 0, 0])
 242
 243     def test_repair_cost(self):
 244         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 245         bwcost = statistics.bandwidth_cost_function
 246         cost = statistics.mean_repair_cost(bwcost, 1000,
 247                                            survival_pmf, 3, ul_dl_ratio=1.0)
 248         self.failUnlessAlmostEqual(cost, 558.90)
 249         cost = statistics.mean_repair_cost(bwcost, 1000,
 250                                            survival_pmf, 3, ul_dl_ratio=8.0)
 251         self.failUnlessAlmostEqual(cost, 1664.55)
 252
 253         # I haven't manually checked the math beyond here -warner
 254         cost = statistics.eternal_repair_cost(bwcost, 1000,
 255                                               survival_pmf, 3,
 256                                               discount_rate=0, ul_dl_ratio=1.0)
 257         self.failUnlessAlmostEqual(cost, 65292.056074766246)
 258         cost = statistics.eternal_repair_cost(bwcost, 1000,
 259                                               survival_pmf, 3,
 260                                               discount_rate=0.05,
 261                                               ul_dl_ratio=1.0)
 262         self.failUnlessAlmostEqual(cost, 9133.6097158191551)
 263
 264     def test_convolve(self):
 265         f = statistics.convolve
 266         v1 = [ 1, 2, 3 ]
 267         v2 = [ 4, 5, 6 ]
 268         v3 = [ 7, 8 ]
 269         v1v2result = [ 4, 13, 28, 27, 18 ]
 270         # Convolution is commutative
 271         r1 = f(v1, v2)
 272         r2 = f(v2, v1)
 273         self.failUnlessListEqual(r1, r2, "Convolution should be commutative")
 274         self.failUnlessListEqual(r1, v1v2result, "Didn't match known result")
 275         # Convolution is associative
 276         r1 = f(f(v1, v2), v3)
 277         r2 = f(v1, f(v2, v3))
 278         self.failUnlessListEqual(r1, r2, "Convolution should be associative")
 279         # Convolution is distributive
 280         r1 = f(v3, [ a + b for a, b in zip(v1, v2) ])
 281         tmp1 = f(v3, v1)
 282         tmp2 = f(v3, v2)
 283         r2 = [ a + b for a, b in zip(tmp1, tmp2) ]
 284         self.failUnlessListEqual(r1, r2, "Convolution should be distributive")
 285         # Convolution is scalar multiplication associative
 286         tmp1 = f(v1, v2)
 287         r1 = [ a * 4 for a in tmp1 ]
 288         tmp2 = [ a * 4 for a in v1 ]
 289         r2 = f(tmp2, v2)
 290         self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative")
 291
 292     def test_find_k(self):
 293         f = statistics.find_k
 294         g = statistics.pr_file_loss
 295         plist = [.9] * 10 + [.8] * 10 # N=20
 296         t = .0001
 297         k = f(plist, t)
 298         self.failUnlessEqual(k, 10)
 299         self.failUnless(g(plist, k) < t)
 300
 301     def test_pr_file_loss(self):
 302         f = statistics.pr_file_loss
 303         plist = [.5] * 10
 304         self.failUnlessEqual(f(plist, 3), .0546875)
 305
 306     def test_pr_backup_file_loss(self):
 307         f = statistics.pr_backup_file_loss
 308         plist = [.5] * 10
 309         self.failUnlessEqual(f(plist, .5, 3), .02734375)
 310
 311
 312 class Asserts(unittest.TestCase):
 313     def should_assert(self, func, *args, **kwargs):
 314         try:
 315             func(*args, **kwargs)
 316         except AssertionError, e:
 317             return str(e)
 318         except Exception, e:
 319             self.fail("assert failed with non-AssertionError: %s" % e)
 320         self.fail("assert was not caught")
 321
 322     def should_not_assert(self, func, *args, **kwargs):
 323         try:
 324             func(*args, **kwargs)
 325         except AssertionError, e:
 326             self.fail("assertion fired when it should not have: %s" % e)
 327         except Exception, e:
 328             self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e)
 329         return # we're happy
 330
 331
 332     def test_assert(self):
 333         f = assertutil._assert
 334         self.should_assert(f)
 335         self.should_assert(f, False)
 336         self.should_not_assert(f, True)
 337
 338         m = self.should_assert(f, False, "message")
 339         self.failUnlessEqual(m, "'message' <type 'str'>", m)
 340         m = self.should_assert(f, False, "message1", othermsg=12)
 341         self.failUnlessEqual("'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 342         m = self.should_assert(f, False, othermsg="message2")
 343         self.failUnlessEqual("othermsg: 'message2' <type 'str'>", m)
 344
 345     def test_precondition(self):
 346         f = assertutil.precondition
 347         self.should_assert(f)
 348         self.should_assert(f, False)
 349         self.should_not_assert(f, True)
 350
 351         m = self.should_assert(f, False, "message")
 352         self.failUnlessEqual("precondition: 'message' <type 'str'>", m)
 353         m = self.should_assert(f, False, "message1", othermsg=12)
 354         self.failUnlessEqual("precondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 355         m = self.should_assert(f, False, othermsg="message2")
 356         self.failUnlessEqual("precondition: othermsg: 'message2' <type 'str'>", m)
 357
 358     def test_postcondition(self):
 359         f = assertutil.postcondition
 360         self.should_assert(f)
 361         self.should_assert(f, False)
 362         self.should_not_assert(f, True)
 363
 364         m = self.should_assert(f, False, "message")
 365         self.failUnlessEqual("postcondition: 'message' <type 'str'>", m)
 366         m = self.should_assert(f, False, "message1", othermsg=12)
 367         self.failUnlessEqual("postcondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 368         m = self.should_assert(f, False, othermsg="message2")
 369         self.failUnlessEqual("postcondition: othermsg: 'message2' <type 'str'>", m)
 370
 371 class FileUtil(unittest.TestCase):
 372     def mkdir(self, basedir, path, mode=0777):
 373         fn = os.path.join(basedir, path)
 374         fileutil.make_dirs(fn, mode)
 375
 376     def touch(self, basedir, path, mode=None, data="touch\n"):
 377         fn = os.path.join(basedir, path)
 378         f = open(fn, "w")
 379         f.write(data)
 380         f.close()
 381         if mode is not None:
 382             os.chmod(fn, mode)
 383
 384     def test_rm_dir(self):
 385         basedir = "util/FileUtil/test_rm_dir"
 386         fileutil.make_dirs(basedir)
 387         # create it again to test idempotency
 388         fileutil.make_dirs(basedir)
 389         d = os.path.join(basedir, "doomed")
 390         self.mkdir(d, "a/b")
 391         self.touch(d, "a/b/1.txt")
 392         self.touch(d, "a/b/2.txt", 0444)
 393         self.touch(d, "a/b/3.txt", 0)
 394         self.mkdir(d, "a/c")
 395         self.touch(d, "a/c/1.txt")
 396         self.touch(d, "a/c/2.txt", 0444)
 397         self.touch(d, "a/c/3.txt", 0)
 398         os.chmod(os.path.join(d, "a/c"), 0444)
 399         self.mkdir(d, "a/d")
 400         self.touch(d, "a/d/1.txt")
 401         self.touch(d, "a/d/2.txt", 0444)
 402         self.touch(d, "a/d/3.txt", 0)
 403         os.chmod(os.path.join(d, "a/d"), 0)
 404
 405         fileutil.rm_dir(d)
 406         self.failIf(os.path.exists(d))
 407         # remove it again to test idempotency
 408         fileutil.rm_dir(d)
 409
 410     def test_remove_if_possible(self):
 411         basedir = "util/FileUtil/test_remove_if_possible"
 412         fileutil.make_dirs(basedir)
 413         self.touch(basedir, "here")
 414         fn = os.path.join(basedir, "here")
 415         fileutil.remove_if_possible(fn)
 416         self.failIf(os.path.exists(fn))
 417         fileutil.remove_if_possible(fn) # should be idempotent
 418         fileutil.rm_dir(basedir)
 419         fileutil.remove_if_possible(fn) # should survive errors
 420
 421     def test_open_or_create(self):
 422         basedir = "util/FileUtil/test_open_or_create"
 423         fileutil.make_dirs(basedir)
 424         fn = os.path.join(basedir, "here")
 425         f = fileutil.open_or_create(fn)
 426         f.write("stuff.")
 427         f.close()
 428         f = fileutil.open_or_create(fn)
 429         f.seek(0, 2)
 430         f.write("more.")
 431         f.close()
 432         f = open(fn, "r")
 433         data = f.read()
 434         f.close()
 435         self.failUnlessEqual(data, "stuff.more.")
 436
 437     def test_NamedTemporaryDirectory(self):
 438         basedir = "util/FileUtil/test_NamedTemporaryDirectory"
 439         fileutil.make_dirs(basedir)
 440         td = fileutil.NamedTemporaryDirectory(dir=basedir)
 441         name = td.name
 442         self.failUnless(basedir in name)
 443         self.failUnless(basedir in repr(td))
 444         self.failUnless(os.path.isdir(name))
 445         del td
 446         # it is conceivable that we need to force gc here, but I'm not sure
 447         self.failIf(os.path.isdir(name))
 448
 449     def test_rename(self):
 450         basedir = "util/FileUtil/test_rename"
 451         fileutil.make_dirs(basedir)
 452         self.touch(basedir, "here")
 453         fn = os.path.join(basedir, "here")
 454         fn2 = os.path.join(basedir, "there")
 455         fileutil.rename(fn, fn2)
 456         self.failIf(os.path.exists(fn))
 457         self.failUnless(os.path.exists(fn2))
 458
 459     def test_du(self):
 460         basedir = "util/FileUtil/test_du"
 461         fileutil.make_dirs(basedir)
 462         d = os.path.join(basedir, "space-consuming")
 463         self.mkdir(d, "a/b")
 464         self.touch(d, "a/b/1.txt", data="a"*10)
 465         self.touch(d, "a/b/2.txt", data="b"*11)
 466         self.mkdir(d, "a/c")
 467         self.touch(d, "a/c/1.txt", data="c"*12)
 468         self.touch(d, "a/c/2.txt", data="d"*13)
 469
 470         used = fileutil.du(basedir)
 471         self.failUnlessEqual(10+11+12+13, used)
 472
 473 class PollMixinTests(unittest.TestCase):
 474     def setUp(self):
 475         self.pm = pollmixin.PollMixin()
 476
 477     def test_PollMixin_True(self):
 478         d = self.pm.poll(check_f=lambda : True,
 479                          pollinterval=0.1)
 480         return d
 481
 482     def test_PollMixin_False_then_True(self):
 483         i = iter([False, True])
 484         d = self.pm.poll(check_f=i.next,
 485                          pollinterval=0.1)
 486         return d
 487
 488     def test_timeout(self):
 489         d = self.pm.poll(check_f=lambda: False,
 490                          pollinterval=0.01,
 491                          timeout=1)
 492         def _suc(res):
 493             self.fail("poll should have failed, not returned %s" % (res,))
 494         def _err(f):
 495             f.trap(pollmixin.TimeoutError)
 496             return None # success
 497         d.addCallbacks(_suc, _err)
 498         return d
 499
 500 class DeferredUtilTests(unittest.TestCase):
 501     def test_gather_results(self):
 502         d1 = defer.Deferred()
 503         d2 = defer.Deferred()
 504         res = deferredutil.gatherResults([d1, d2])
 505         d1.errback(ValueError("BAD"))
 506         def _callb(res):
 507             self.fail("Should have errbacked, not resulted in %s" % (res,))
 508         def _errb(thef):
 509             thef.trap(ValueError)
 510         res.addCallbacks(_callb, _errb)
 511         return res
 512
 513     def test_success(self):
 514         d1, d2 = defer.Deferred(), defer.Deferred()
 515         good = []
 516         bad = []
 517         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 518         dlss.addCallbacks(good.append, bad.append)
 519         d1.callback(1)
 520         d2.callback(2)
 521         self.failUnlessEqual(good, [[1,2]])
 522         self.failUnlessEqual(bad, [])
 523
 524     def test_failure(self):
 525         d1, d2 = defer.Deferred(), defer.Deferred()
 526         good = []
 527         bad = []
 528         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 529         dlss.addCallbacks(good.append, bad.append)
 530         d1.addErrback(lambda _ignore: None)
 531         d2.addErrback(lambda _ignore: None)
 532         d1.callback(1)
 533         d2.errback(ValueError())
 534         self.failUnlessEqual(good, [])
 535         self.failUnlessEqual(len(bad), 1)
 536         f = bad[0]
 537         self.failUnless(isinstance(f, Failure))
 538         self.failUnless(f.check(ValueError))
 539
 540 class HashUtilTests(unittest.TestCase):
 541
 542     def test_random_key(self):
 543         k = hashutil.random_key()
 544         self.failUnlessEqual(len(k), hashutil.KEYLEN)
 545
 546     def test_sha256d(self):
 547         h1 = hashutil.tagged_hash("tag1", "value")
 548         h2 = hashutil.tagged_hasher("tag1")
 549         h2.update("value")
 550         h2a = h2.digest()
 551         h2b = h2.digest()
 552         self.failUnlessEqual(h1, h2a)
 553         self.failUnlessEqual(h2a, h2b)
 554
 555     def test_sha256d_truncated(self):
 556         h1 = hashutil.tagged_hash("tag1", "value", 16)
 557         h2 = hashutil.tagged_hasher("tag1", 16)
 558         h2.update("value")
 559         h2 = h2.digest()
 560         self.failUnlessEqual(len(h1), 16)
 561         self.failUnlessEqual(len(h2), 16)
 562         self.failUnlessEqual(h1, h2)
 563
 564     def test_chk(self):
 565         h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret")
 566         h2 = hashutil.convergence_hasher(3, 10, 1000, "secret")
 567         h2.update("data")
 568         h2 = h2.digest()
 569         self.failUnlessEqual(h1, h2)
 570
 571     def test_hashers(self):
 572         h1 = hashutil.block_hash("foo")
 573         h2 = hashutil.block_hasher()
 574         h2.update("foo")
 575         self.failUnlessEqual(h1, h2.digest())
 576
 577         h1 = hashutil.uri_extension_hash("foo")
 578         h2 = hashutil.uri_extension_hasher()
 579         h2.update("foo")
 580         self.failUnlessEqual(h1, h2.digest())
 581
 582         h1 = hashutil.plaintext_hash("foo")
 583         h2 = hashutil.plaintext_hasher()
 584         h2.update("foo")
 585         self.failUnlessEqual(h1, h2.digest())
 586
 587         h1 = hashutil.crypttext_hash("foo")
 588         h2 = hashutil.crypttext_hasher()
 589         h2.update("foo")
 590         self.failUnlessEqual(h1, h2.digest())
 591
 592         h1 = hashutil.crypttext_segment_hash("foo")
 593         h2 = hashutil.crypttext_segment_hasher()
 594         h2.update("foo")
 595         self.failUnlessEqual(h1, h2.digest())
 596
 597         h1 = hashutil.plaintext_segment_hash("foo")
 598         h2 = hashutil.plaintext_segment_hasher()
 599         h2.update("foo")
 600         self.failUnlessEqual(h1, h2.digest())
 601
 602     def test_constant_time_compare(self):
 603         self.failUnless(hashutil.constant_time_compare("a", "a"))
 604         self.failUnless(hashutil.constant_time_compare("ab", "ab"))
 605         self.failIf(hashutil.constant_time_compare("a", "b"))
 606         self.failIf(hashutil.constant_time_compare("a", "aa"))
 607
 608     def _testknown(self, hashf, expected_a, *args):
 609         got = hashf(*args)
 610         got_a = base32.b2a(got)
 611         self.failUnlessEqual(got_a, expected_a)
 612
 613     def test_known_answers(self):
 614         # assert backwards compatibility
 615         self._testknown(hashutil.storage_index_hash, "qb5igbhcc5esa6lwqorsy7e6am", "")
 616         self._testknown(hashutil.block_hash, "msjr5bh4evuh7fa3zw7uovixfbvlnstr5b65mrerwfnvjxig2jvq", "")
 617         self._testknown(hashutil.uri_extension_hash, "wthsu45q7zewac2mnivoaa4ulh5xvbzdmsbuyztq2a5fzxdrnkka", "")
 618         self._testknown(hashutil.plaintext_hash, "5lz5hwz3qj3af7n6e3arblw7xzutvnd3p3fjsngqjcb7utf3x3da", "")
 619         self._testknown(hashutil.crypttext_hash, "itdj6e4njtkoiavlrmxkvpreosscssklunhwtvxn6ggho4rkqwga", "")
 620         self._testknown(hashutil.crypttext_segment_hash, "aovy5aa7jej6ym5ikgwyoi4pxawnoj3wtaludjz7e2nb5xijb7aa", "")
 621         self._testknown(hashutil.plaintext_segment_hash, "4fdgf6qruaisyukhqcmoth4t3li6bkolbxvjy4awwcpprdtva7za", "")
 622         self._testknown(hashutil.convergence_hash, "3mo6ni7xweplycin6nowynw2we", 3, 10, 100, "", "converge")
 623         self._testknown(hashutil.my_renewal_secret_hash, "ujhr5k5f7ypkp67jkpx6jl4p47pyta7hu5m527cpcgvkafsefm6q", "")
 624         self._testknown(hashutil.my_cancel_secret_hash, "rjwzmafe2duixvqy6h47f5wfrokdziry6zhx4smew4cj6iocsfaa", "")
 625         self._testknown(hashutil.file_renewal_secret_hash, "hzshk2kf33gzbd5n3a6eszkf6q6o6kixmnag25pniusyaulqjnia", "", "si")
 626         self._testknown(hashutil.file_cancel_secret_hash, "bfciwvr6w7wcavsngxzxsxxaszj72dej54n4tu2idzp6b74g255q", "", "si")
 627         self._testknown(hashutil.bucket_renewal_secret_hash, "e7imrzgzaoashsncacvy3oysdd2m5yvtooo4gmj4mjlopsazmvuq", "", "\x00"*20)
 628         self._testknown(hashutil.bucket_cancel_secret_hash, "dvdujeyxeirj6uux6g7xcf4lvesk632aulwkzjar7srildvtqwma", "", "\x00"*20)
 629         self._testknown(hashutil.hmac, "c54ypfi6pevb3nvo6ba42jtglpkry2kbdopqsi7dgrm4r7tw5sra", "tag", "")
 630         self._testknown(hashutil.mutable_rwcap_key_hash, "6rvn2iqrghii5n4jbbwwqqsnqu", "iv", "wk")
 631         self._testknown(hashutil.ssk_writekey_hash, "ykpgmdbpgbb6yqz5oluw2q26ye", "")
 632         self._testknown(hashutil.ssk_write_enabler_master_hash, "izbfbfkoait4dummruol3gy2bnixrrrslgye6ycmkuyujnenzpia", "")
 633         self._testknown(hashutil.ssk_write_enabler_hash, "fuu2dvx7g6gqu5x22vfhtyed7p4pd47y5hgxbqzgrlyvxoev62tq", "wk", "\x00"*20)
 634         self._testknown(hashutil.ssk_pubkey_fingerprint_hash, "3opzw4hhm2sgncjx224qmt5ipqgagn7h5zivnfzqycvgqgmgz35q", "")
 635         self._testknown(hashutil.ssk_readkey_hash, "vugid4as6qbqgeq2xczvvcedai", "")
 636         self._testknown(hashutil.ssk_readkey_data_hash, "73wsaldnvdzqaf7v4pzbr2ae5a", "iv", "rk")
 637         self._testknown(hashutil.ssk_storage_index_hash, "j7icz6kigb6hxrej3tv4z7ayym", "")
 638
 639
 640 class Abbreviate(unittest.TestCase):
 641     def test_time(self):
 642         a = abbreviate.abbreviate_time
 643         self.failUnlessEqual(a(None), "unknown")
 644         self.failUnlessEqual(a(0), "0 seconds")
 645         self.failUnlessEqual(a(1), "1 second")
 646         self.failUnlessEqual(a(2), "2 seconds")
 647         self.failUnlessEqual(a(119), "119 seconds")
 648         MIN = 60
 649         self.failUnlessEqual(a(2*MIN), "2 minutes")
 650         self.failUnlessEqual(a(60*MIN), "60 minutes")
 651         self.failUnlessEqual(a(179*MIN), "179 minutes")
 652         HOUR = 60*MIN
 653         self.failUnlessEqual(a(180*MIN), "3 hours")
 654         self.failUnlessEqual(a(4*HOUR), "4 hours")
 655         DAY = 24*HOUR
 656         MONTH = 30*DAY
 657         self.failUnlessEqual(a(2*DAY), "2 days")
 658         self.failUnlessEqual(a(2*MONTH), "2 months")
 659         YEAR = 365*DAY
 660         self.failUnlessEqual(a(5*YEAR), "5 years")
 661
 662     def test_space(self):
 663         tests_si = [(None, "unknown"),
 664                     (0, "0 B"),
 665                     (1, "1 B"),
 666                     (999, "999 B"),
 667                     (1000, "1000 B"),
 668                     (1023, "1023 B"),
 669                     (1024, "1.02 kB"),
 670                     (20*1000, "20.00 kB"),
 671                     (1024*1024, "1.05 MB"),
 672                     (1000*1000, "1.00 MB"),
 673                     (1000*1000*1000, "1.00 GB"),
 674                     (1000*1000*1000*1000, "1.00 TB"),
 675                     (1000*1000*1000*1000*1000, "1.00 PB"),
 676                     (1234567890123456, "1.23 PB"),
 677                     ]
 678         for (x, expected) in tests_si:
 679             got = abbreviate.abbreviate_space(x, SI=True)
 680             self.failUnlessEqual(got, expected)
 681
 682         tests_base1024 = [(None, "unknown"),
 683                           (0, "0 B"),
 684                           (1, "1 B"),
 685                           (999, "999 B"),
 686                           (1000, "1000 B"),
 687                           (1023, "1023 B"),
 688                           (1024, "1.00 kiB"),
 689                           (20*1024, "20.00 kiB"),
 690                           (1000*1000, "976.56 kiB"),
 691                           (1024*1024, "1.00 MiB"),
 692                           (1024*1024*1024, "1.00 GiB"),
 693                           (1024*1024*1024*1024, "1.00 TiB"),
 694                           (1000*1000*1000*1000*1000, "909.49 TiB"),
 695                           (1024*1024*1024*1024*1024, "1.00 PiB"),
 696                           (1234567890123456, "1.10 PiB"),
 697                     ]
 698         for (x, expected) in tests_base1024:
 699             got = abbreviate.abbreviate_space(x, SI=False)
 700             self.failUnlessEqual(got, expected)
 701
 702         self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567),
 703                              "(1.23 MB, 1.18 MiB)")
 704
 705     def test_parse_space(self):
 706         p = abbreviate.parse_abbreviated_size
 707         self.failUnlessEqual(p(""), None)
 708         self.failUnlessEqual(p(None), None)
 709         self.failUnlessEqual(p("123"), 123)
 710         self.failUnlessEqual(p("123B"), 123)
 711         self.failUnlessEqual(p("2K"), 2000)
 712         self.failUnlessEqual(p("2kb"), 2000)
 713         self.failUnlessEqual(p("2KiB"), 2048)
 714         self.failUnlessEqual(p("10MB"), 10*1000*1000)
 715         self.failUnlessEqual(p("10MiB"), 10*1024*1024)
 716         self.failUnlessEqual(p("5G"), 5*1000*1000*1000)
 717         self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024)
 718         e = self.failUnlessRaises(ValueError, p, "12 cubits")
 719         self.failUnless("12 cubits" in str(e))
 720
 721 class Limiter(unittest.TestCase):
 722     timeout = 480 # This takes longer than 240 seconds on Francois's arm box.
 723
 724     def job(self, i, foo):
 725         self.calls.append( (i, foo) )
 726         self.simultaneous += 1
 727         self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous)
 728         d = defer.Deferred()
 729         def _done():
 730             self.simultaneous -= 1
 731             d.callback("done %d" % i)
 732         reactor.callLater(1.0, _done)
 733         return d
 734
 735     def bad_job(self, i, foo):
 736         raise ValueError("bad_job %d" % i)
 737
 738     def test_limiter(self):
 739         self.calls = []
 740         self.simultaneous = 0
 741         self.peak_simultaneous = 0
 742         l = limiter.ConcurrencyLimiter()
 743         dl = []
 744         for i in range(20):
 745             dl.append(l.add(self.job, i, foo=str(i)))
 746         d = defer.DeferredList(dl, fireOnOneErrback=True)
 747         def _done(res):
 748             self.failUnlessEqual(self.simultaneous, 0)
 749             self.failUnless(self.peak_simultaneous <= 10)
 750             self.failUnlessEqual(len(self.calls), 20)
 751             for i in range(20):
 752                 self.failUnless( (i, str(i)) in self.calls)
 753         d.addCallback(_done)
 754         return d
 755
 756     def test_errors(self):
 757         self.calls = []
 758         self.simultaneous = 0
 759         self.peak_simultaneous = 0
 760         l = limiter.ConcurrencyLimiter()
 761         dl = []
 762         for i in range(20):
 763             dl.append(l.add(self.job, i, foo=str(i)))
 764         d2 = l.add(self.bad_job, 21, "21")
 765         d = defer.DeferredList(dl, fireOnOneErrback=True)
 766         def _most_done(res):
 767             results = []
 768             for (success, result) in res:
 769                 self.failUnlessEqual(success, True)
 770                 results.append(result)
 771             results.sort()
 772             expected_results = ["done %d" % i for i in range(20)]
 773             expected_results.sort()
 774             self.failUnlessEqual(results, expected_results)
 775             self.failUnless(self.peak_simultaneous <= 10)
 776             self.failUnlessEqual(len(self.calls), 20)
 777             for i in range(20):
 778                 self.failUnless( (i, str(i)) in self.calls)
 779             def _good(res):
 780                 self.fail("should have failed, not got %s" % (res,))
 781             def _err(f):
 782                 f.trap(ValueError)
 783                 self.failUnless("bad_job 21" in str(f))
 784             d2.addCallbacks(_good, _err)
 785             return d2
 786         d.addCallback(_most_done)
 787         def _all_done(res):
 788             self.failUnlessEqual(self.simultaneous, 0)
 789             self.failUnless(self.peak_simultaneous <= 10)
 790             self.failUnlessEqual(len(self.calls), 20)
 791             for i in range(20):
 792                 self.failUnless( (i, str(i)) in self.calls)
 793         d.addCallback(_all_done)
 794         return d
 795
 796 class TimeFormat(unittest.TestCase):
 797     def test_epoch(self):
 798         return self._help_test_epoch()
 799
 800     def test_epoch_in_London(self):
 801         # Europe/London is a particularly troublesome timezone.  Nowadays, its
 802         # offset from GMT is 0.  But in 1970, its offset from GMT was 1.
 803         # (Apparently in 1970 Britain had redefined standard time to be GMT+1
 804         # and stayed in standard time all year round, whereas today
 805         # Europe/London standard time is GMT and Europe/London Daylight
 806         # Savings Time is GMT+1.)  The current implementation of
 807         # time_format.iso_utc_time_to_localseconds() breaks if the timezone is
 808         # Europe/London.  (As soon as this unit test is done then I'll change
 809         # that implementation to something that works even in this case...)
 810         origtz = os.environ.get('TZ')
 811         os.environ['TZ'] = "Europe/London"
 812         if hasattr(time, 'tzset'):
 813             time.tzset()
 814         try:
 815             return self._help_test_epoch()
 816         finally:
 817             if origtz is None:
 818                 del os.environ['TZ']
 819             else:
 820                 os.environ['TZ'] = origtz
 821             if hasattr(time, 'tzset'):
 822                 time.tzset()
 823
 824     def _help_test_epoch(self):
 825         origtzname = time.tzname
 826         s = time_format.iso_utc_time_to_seconds("1970-01-01T00:00:01")
 827         self.failUnlessEqual(s, 1.0)
 828         s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01")
 829         self.failUnlessEqual(s, 1.0)
 830         s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01")
 831         self.failUnlessEqual(s, 1.0)
 832
 833         self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01")
 834         self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "),
 835                              "1970-01-01 00:00:01")
 836
 837         now = time.time()
 838         isostr = time_format.iso_utc(now)
 839         timestamp = time_format.iso_utc_time_to_seconds(isostr)
 840         self.failUnlessEqual(int(timestamp), int(now))
 841
 842         def my_time():
 843             return 1.0
 844         self.failUnlessEqual(time_format.iso_utc(t=my_time),
 845                              "1970-01-01_00:00:01")
 846         e = self.failUnlessRaises(ValueError,
 847                                   time_format.iso_utc_time_to_seconds,
 848                                   "invalid timestring")
 849         self.failUnless("not a complete ISO8601 timestamp" in str(e))
 850         s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01.500")
 851         self.failUnlessEqual(s, 1.5)
 852
 853         # Look for daylight-savings-related errors.
 854         thatmomentinmarch = time_format.iso_utc_time_to_seconds("2009-03-20 21:49:02.226536")
 855         self.failUnlessEqual(thatmomentinmarch, 1237585742.226536)
 856         self.failUnlessEqual(origtzname, time.tzname)
 857
 858     def test_iso_utc(self):
 859         when = 1266760143.7841301
 860         out = time_format.iso_utc_date(when)
 861         self.failUnlessEqual(out, "2010-02-21")
 862         out = time_format.iso_utc_date(t=lambda: when)
 863         self.failUnlessEqual(out, "2010-02-21")
 864         out = time_format.iso_utc(when)
 865         self.failUnlessEqual(out, "2010-02-21_13:49:03.784130")
 866         out = time_format.iso_utc(when, sep="-")
 867         self.failUnlessEqual(out, "2010-02-21-13:49:03.784130")
 868
 869     def test_parse_duration(self):
 870         p = time_format.parse_duration
 871         DAY = 24*60*60
 872         self.failUnlessEqual(p("1 day"), DAY)
 873         self.failUnlessEqual(p("2 days"), 2*DAY)
 874         self.failUnlessEqual(p("3 months"), 3*31*DAY)
 875         self.failUnlessEqual(p("4 mo"), 4*31*DAY)
 876         self.failUnlessEqual(p("5 years"), 5*365*DAY)
 877         e = self.failUnlessRaises(ValueError, p, "123")
 878         self.failUnlessIn("no unit (like day, month, or year) in '123'",
 879                           str(e))
 880
 881     def test_parse_date(self):
 882         self.failUnlessEqual(time_format.parse_date("2010-02-21"), 1266710400)
 883
 884 class CacheDir(unittest.TestCase):
 885     def test_basic(self):
 886         basedir = "test_util/CacheDir/test_basic"
 887
 888         def _failIfExists(name):
 889             absfn = os.path.join(basedir, name)
 890             self.failIf(os.path.exists(absfn),
 891                         "%s exists but it shouldn't" % absfn)
 892
 893         def _failUnlessExists(name):
 894             absfn = os.path.join(basedir, name)
 895             self.failUnless(os.path.exists(absfn),
 896                             "%s doesn't exist but it should" % absfn)
 897
 898         cdm = cachedir.CacheDirectoryManager(basedir)
 899         a = cdm.get_file("a")
 900         b = cdm.get_file("b")
 901         c = cdm.get_file("c")
 902         f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f
 903         f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f
 904         f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f
 905
 906         _failUnlessExists("a")
 907         _failUnlessExists("b")
 908         _failUnlessExists("c")
 909
 910         cdm.check()
 911
 912         _failUnlessExists("a")
 913         _failUnlessExists("b")
 914         _failUnlessExists("c")
 915
 916         del a
 917         # this file won't be deleted yet, because it isn't old enough
 918         cdm.check()
 919         _failUnlessExists("a")
 920         _failUnlessExists("b")
 921         _failUnlessExists("c")
 922
 923         # we change the definition of "old" to make everything old
 924         cdm.old = -10
 925
 926         cdm.check()
 927         _failIfExists("a")
 928         _failUnlessExists("b")
 929         _failUnlessExists("c")
 930
 931         cdm.old = 60*60
 932
 933         del b
 934
 935         cdm.check()
 936         _failIfExists("a")
 937         _failUnlessExists("b")
 938         _failUnlessExists("c")
 939
 940         b2 = cdm.get_file("b")
 941
 942         cdm.check()
 943         _failIfExists("a")
 944         _failUnlessExists("b")
 945         _failUnlessExists("c")
 946         del b2
 947
 948 ctr = [0]
 949 class EqButNotIs:
 950     def __init__(self, x):
 951         self.x = x
 952         self.hash = ctr[0]
 953         ctr[0] += 1
 954     def __repr__(self):
 955         return "<%s %s>" % (self.__class__.__name__, self.x,)
 956     def __hash__(self):
 957         return self.hash
 958     def __le__(self, other):
 959         return self.x <= other
 960     def __lt__(self, other):
 961         return self.x < other
 962     def __ge__(self, other):
 963         return self.x >= other
 964     def __gt__(self, other):
 965         return self.x > other
 966     def __ne__(self, other):
 967         return self.x != other
 968     def __eq__(self, other):
 969         return self.x == other
 970
 971 class DictUtil(unittest.TestCase):
 972     def _help_test_empty_dict(self, klass):
 973         d1 = klass()
 974         d2 = klass({})
 975
 976         self.failUnless(d1 == d2, "d1: %r, d2: %r" % (d1, d2,))
 977         self.failUnless(len(d1) == 0)
 978         self.failUnless(len(d2) == 0)
 979
 980     def _help_test_nonempty_dict(self, klass):
 981         d1 = klass({'a': 1, 'b': "eggs", 3: "spam",})
 982         d2 = klass({'a': 1, 'b': "eggs", 3: "spam",})
 983
 984         self.failUnless(d1 == d2)
 985         self.failUnless(len(d1) == 3, "%s, %s" % (len(d1), d1,))
 986         self.failUnless(len(d2) == 3)
 987
 988     def _help_test_eq_but_notis(self, klass):
 989         d = klass({'a': 3, 'b': EqButNotIs(3), 'c': 3})
 990         d.pop('b')
 991
 992         d.clear()
 993         d['a'] = 3
 994         d['b'] = EqButNotIs(3)
 995         d['c'] = 3
 996         d.pop('b')
 997
 998         d.clear()
 999         d['b'] = EqButNotIs(3)
1000         d['a'] = 3
1001         d['c'] = 3
1002         d.pop('b')
1003
1004         d.clear()
1005         d['a'] = EqButNotIs(3)
1006         d['c'] = 3
1007         d['a'] = 3
1008
1009         d.clear()
1010         fake3 = EqButNotIs(3)
1011         fake7 = EqButNotIs(7)
1012         d[fake3] = fake7
1013         d[3] = 7
1014         d[3] = 8
1015         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
1016         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
1017         # The real 7 should have been ejected by the d[3] = 8.
1018         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
1019         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
1020         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
1021         d[fake3] = 8
1022
1023         d.clear()
1024         d[3] = 7
1025         fake3 = EqButNotIs(3)
1026         fake7 = EqButNotIs(7)
1027         d[fake3] = fake7
1028         d[3] = 8
1029         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
1030         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
1031         # The real 7 should have been ejected by the d[3] = 8.
1032         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
1033         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
1034         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
1035         d[fake3] = 8
1036
1037     def test_all(self):
1038         self._help_test_eq_but_notis(dictutil.UtilDict)
1039         self._help_test_eq_but_notis(dictutil.NumDict)
1040         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
1041         self._help_test_nonempty_dict(dictutil.UtilDict)
1042         self._help_test_nonempty_dict(dictutil.NumDict)
1043         self._help_test_nonempty_dict(dictutil.ValueOrderedDict)
1044         self._help_test_eq_but_notis(dictutil.UtilDict)
1045         self._help_test_eq_but_notis(dictutil.NumDict)
1046         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
1047
1048     def test_dict_of_sets(self):
1049         ds = dictutil.DictOfSets()
1050         ds.add(1, "a")
1051         ds.add(2, "b")
1052         ds.add(2, "b")
1053         ds.add(2, "c")
1054         self.failUnlessEqual(ds[1], set(["a"]))
1055         self.failUnlessEqual(ds[2], set(["b", "c"]))
1056         ds.discard(3, "d") # should not raise an exception
1057         ds.discard(2, "b")
1058         self.failUnlessEqual(ds[2], set(["c"]))
1059         ds.discard(2, "c")
1060         self.failIf(2 in ds)
1061
1062         ds.union(1, ["a", "e"])
1063         ds.union(3, ["f"])
1064         self.failUnlessEqual(ds[1], set(["a","e"]))
1065         self.failUnlessEqual(ds[3], set(["f"]))
1066         ds2 = dictutil.DictOfSets()
1067         ds2.add(3, "f")
1068         ds2.add(3, "g")
1069         ds2.add(4, "h")
1070         ds.update(ds2)
1071         self.failUnlessEqual(ds[1], set(["a","e"]))
1072         self.failUnlessEqual(ds[3], set(["f", "g"]))
1073         self.failUnlessEqual(ds[4], set(["h"]))
1074
1075     def test_move(self):
1076         d1 = {1: "a", 2: "b"}
1077         d2 = {2: "c", 3: "d"}
1078         dictutil.move(1, d1, d2)
1079         self.failUnlessEqual(d1, {2: "b"})
1080         self.failUnlessEqual(d2, {1: "a", 2: "c", 3: "d"})
1081
1082         d1 = {1: "a", 2: "b"}
1083         d2 = {2: "c", 3: "d"}
1084         dictutil.move(2, d1, d2)
1085         self.failUnlessEqual(d1, {1: "a"})
1086         self.failUnlessEqual(d2, {2: "b", 3: "d"})
1087
1088         d1 = {1: "a", 2: "b"}
1089         d2 = {2: "c", 3: "d"}
1090         self.failUnlessRaises(KeyError, dictutil.move, 5, d1, d2, strict=True)
1091
1092     def test_subtract(self):
1093         d1 = {1: "a", 2: "b"}
1094         d2 = {2: "c", 3: "d"}
1095         d3 = dictutil.subtract(d1, d2)
1096         self.failUnlessEqual(d3, {1: "a"})
1097
1098         d1 = {1: "a", 2: "b"}
1099         d2 = {2: "c"}
1100         d3 = dictutil.subtract(d1, d2)
1101         self.failUnlessEqual(d3, {1: "a"})
1102
1103     def test_utildict(self):
1104         d = dictutil.UtilDict({1: "a", 2: "b"})
1105         d.del_if_present(1)
1106         d.del_if_present(3)
1107         self.failUnlessEqual(d, {2: "b"})
1108         def eq(a, b):
1109             return a == b
1110         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1111
1112         d = dictutil.UtilDict({1: "b", 2: "a"})
1113         self.failUnlessEqual(d.items_sorted_by_value(),
1114                              [(2, "a"), (1, "b")])
1115         self.failUnlessEqual(d.items_sorted_by_key(),
1116                              [(1, "b"), (2, "a")])
1117         self.failUnlessEqual(repr(d), "{1: 'b', 2: 'a'}")
1118         self.failUnless(1 in d)
1119
1120         d2 = dictutil.UtilDict({3: "c", 4: "d"})
1121         self.failUnless(d != d2)
1122         self.failUnless(d2 > d)
1123         self.failUnless(d2 >= d)
1124         self.failUnless(d <= d2)
1125         self.failUnless(d < d2)
1126         self.failUnlessEqual(d[1], "b")
1127         self.failUnlessEqual(sorted(list([k for k in d])), [1,2])
1128
1129         d3 = d.copy()
1130         self.failUnlessEqual(d, d3)
1131         self.failUnless(isinstance(d3, dictutil.UtilDict))
1132
1133         d4 = d.fromkeys([3,4], "e")
1134         self.failUnlessEqual(d4, {3: "e", 4: "e"})
1135
1136         self.failUnlessEqual(d.get(1), "b")
1137         self.failUnlessEqual(d.get(3), None)
1138         self.failUnlessEqual(d.get(3, "default"), "default")
1139         self.failUnlessEqual(sorted(list(d.items())),
1140                              [(1, "b"), (2, "a")])
1141         self.failUnlessEqual(sorted(list(d.iteritems())),
1142                              [(1, "b"), (2, "a")])
1143         self.failUnlessEqual(sorted(d.keys()), [1, 2])
1144         self.failUnlessEqual(sorted(d.values()), ["a", "b"])
1145         x = d.setdefault(1, "new")
1146         self.failUnlessEqual(x, "b")
1147         self.failUnlessEqual(d[1], "b")
1148         x = d.setdefault(3, "new")
1149         self.failUnlessEqual(x, "new")
1150         self.failUnlessEqual(d[3], "new")
1151         del d[3]
1152
1153         x = d.popitem()
1154         self.failUnless(x in [(1, "b"), (2, "a")])
1155         x = d.popitem()
1156         self.failUnless(x in [(1, "b"), (2, "a")])
1157         self.failUnlessRaises(KeyError, d.popitem)
1158
1159     def test_numdict(self):
1160         d = dictutil.NumDict({"a": 1, "b": 2})
1161
1162         d.add_num("a", 10, 5)
1163         d.add_num("c", 20, 5)
1164         d.add_num("d", 30)
1165         self.failUnlessEqual(d, {"a": 11, "b": 2, "c": 25, "d": 30})
1166
1167         d.subtract_num("a", 10)
1168         d.subtract_num("e", 10)
1169         d.subtract_num("f", 10, 15)
1170         self.failUnlessEqual(d, {"a": 1, "b": 2, "c": 25, "d": 30,
1171                                  "e": -10, "f": 5})
1172
1173         self.failUnlessEqual(d.sum(), sum([1, 2, 25, 30, -10, 5]))
1174
1175         d = dictutil.NumDict()
1176         d.inc("a")
1177         d.inc("a")
1178         d.inc("b", 5)
1179         self.failUnlessEqual(d, {"a": 2, "b": 6})
1180         d.dec("a")
1181         d.dec("c")
1182         d.dec("d", 5)
1183         self.failUnlessEqual(d, {"a": 1, "b": 6, "c": -1, "d": 4})
1184         self.failUnlessEqual(d.items_sorted_by_key(),
1185                              [("a", 1), ("b", 6), ("c", -1), ("d", 4)])
1186         self.failUnlessEqual(d.items_sorted_by_value(),
1187                              [("c", -1), ("a", 1), ("d", 4), ("b", 6)])
1188         self.failUnlessEqual(d.item_with_largest_value(), ("b", 6))
1189
1190         d = dictutil.NumDict({"a": 1, "b": 2})
1191         self.failUnlessEqual(repr(d), "{'a': 1, 'b': 2}")
1192         self.failUnless("a" in d)
1193
1194         d2 = dictutil.NumDict({"c": 3, "d": 4})
1195         self.failUnless(d != d2)
1196         self.failUnless(d2 > d)
1197         self.failUnless(d2 >= d)
1198         self.failUnless(d <= d2)
1199         self.failUnless(d < d2)
1200         self.failUnlessEqual(d["a"], 1)
1201         self.failUnlessEqual(sorted(list([k for k in d])), ["a","b"])
1202         def eq(a, b):
1203             return a == b
1204         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1205
1206         d3 = d.copy()
1207         self.failUnlessEqual(d, d3)
1208         self.failUnless(isinstance(d3, dictutil.NumDict))
1209
1210         d4 = d.fromkeys(["a","b"], 5)
1211         self.failUnlessEqual(d4, {"a": 5, "b": 5})
1212
1213         self.failUnlessEqual(d.get("a"), 1)
1214         self.failUnlessEqual(d.get("c"), 0)
1215         self.failUnlessEqual(d.get("c", 5), 5)
1216         self.failUnlessEqual(sorted(list(d.items())),
1217                              [("a", 1), ("b", 2)])
1218         self.failUnlessEqual(sorted(list(d.iteritems())),
1219                              [("a", 1), ("b", 2)])
1220         self.failUnlessEqual(sorted(d.keys()), ["a", "b"])
1221         self.failUnlessEqual(sorted(d.values()), [1, 2])
1222         self.failUnless(d.has_key("a"))
1223         self.failIf(d.has_key("c"))
1224
1225         x = d.setdefault("c", 3)
1226         self.failUnlessEqual(x, 3)
1227         self.failUnlessEqual(d["c"], 3)
1228         x = d.setdefault("c", 5)
1229         self.failUnlessEqual(x, 3)
1230         self.failUnlessEqual(d["c"], 3)
1231         del d["c"]
1232
1233         x = d.popitem()
1234         self.failUnless(x in [("a", 1), ("b", 2)])
1235         x = d.popitem()
1236         self.failUnless(x in [("a", 1), ("b", 2)])
1237         self.failUnlessRaises(KeyError, d.popitem)
1238
1239         d.update({"c": 3})
1240         d.update({"c": 4, "d": 5})
1241         self.failUnlessEqual(d, {"c": 4, "d": 5})
1242
1243     def test_del_if_present(self):
1244         d = {1: "a", 2: "b"}
1245         dictutil.del_if_present(d, 1)
1246         dictutil.del_if_present(d, 3)
1247         self.failUnlessEqual(d, {2: "b"})
1248
1249     def test_valueordereddict(self):
1250         d = dictutil.ValueOrderedDict()
1251         d["a"] = 3
1252         d["b"] = 2
1253         d["c"] = 1
1254
1255         self.failUnlessEqual(d, {"a": 3, "b": 2, "c": 1})
1256         self.failUnlessEqual(d.items(), [("c", 1), ("b", 2), ("a", 3)])
1257         self.failUnlessEqual(d.values(), [1, 2, 3])
1258         self.failUnlessEqual(d.keys(), ["c", "b", "a"])
1259         self.failUnlessEqual(repr(d), "<ValueOrderedDict {c: 1, b: 2, a: 3}>")
1260         def eq(a, b):
1261             return a == b
1262         self.failIf(d == {"a": 4})
1263         self.failUnless(d != {"a": 4})
1264
1265         x = d.setdefault("d", 0)
1266         self.failUnlessEqual(x, 0)
1267         self.failUnlessEqual(d["d"], 0)
1268         x = d.setdefault("d", -1)
1269         self.failUnlessEqual(x, 0)
1270         self.failUnlessEqual(d["d"], 0)
1271
1272         x = d.remove("e", "default", False)
1273         self.failUnlessEqual(x, "default")
1274         self.failUnlessRaises(KeyError, d.remove, "e", "default", True)
1275         x = d.remove("d", 5)
1276         self.failUnlessEqual(x, 0)
1277
1278         x = d.__getitem__("c")
1279         self.failUnlessEqual(x, 1)
1280         x = d.__getitem__("e", "default", False)
1281         self.failUnlessEqual(x, "default")
1282         self.failUnlessRaises(KeyError, d.__getitem__, "e", "default", True)
1283
1284         self.failUnlessEqual(d.popitem(), ("c", 1))
1285         self.failUnlessEqual(d.popitem(), ("b", 2))
1286         self.failUnlessEqual(d.popitem(), ("a", 3))
1287         self.failUnlessRaises(KeyError, d.popitem)
1288
1289         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1290         x = d.pop("d", "default", False)
1291         self.failUnlessEqual(x, "default")
1292         self.failUnlessRaises(KeyError, d.pop, "d", "default", True)
1293         x = d.pop("b")
1294         self.failUnlessEqual(x, 2)
1295         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1296
1297         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1298         x = d.pop_from_list(1) # pop the second item, b/2
1299         self.failUnlessEqual(x, "b")
1300         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1301
1302     def test_auxdict(self):
1303         d = dictutil.AuxValueDict()
1304         # we put the serialized form in the auxdata
1305         d.set_with_aux("key", ("filecap", "metadata"), "serialized")
1306
1307         self.failUnlessEqual(d.keys(), ["key"])
1308         self.failUnlessEqual(d["key"], ("filecap", "metadata"))
1309         self.failUnlessEqual(d.get_aux("key"), "serialized")
1310         def _get_missing(key):
1311             return d[key]
1312         self.failUnlessRaises(KeyError, _get_missing, "nonkey")
1313         self.failUnlessEqual(d.get("nonkey"), None)
1314         self.failUnlessEqual(d.get("nonkey", "nonvalue"), "nonvalue")
1315         self.failUnlessEqual(d.get_aux("nonkey"), None)
1316         self.failUnlessEqual(d.get_aux("nonkey", "nonvalue"), "nonvalue")
1317
1318         d["key"] = ("filecap2", "metadata2")
1319         self.failUnlessEqual(d["key"], ("filecap2", "metadata2"))
1320         self.failUnlessEqual(d.get_aux("key"), None)
1321
1322         d.set_with_aux("key2", "value2", "aux2")
1323         self.failUnlessEqual(sorted(d.keys()), ["key", "key2"])
1324         del d["key2"]
1325         self.failUnlessEqual(d.keys(), ["key"])
1326         self.failIf("key2" in d)
1327         self.failUnlessRaises(KeyError, _get_missing, "key2")
1328         self.failUnlessEqual(d.get("key2"), None)
1329         self.failUnlessEqual(d.get_aux("key2"), None)
1330         d["key2"] = "newvalue2"
1331         self.failUnlessEqual(d.get("key2"), "newvalue2")
1332         self.failUnlessEqual(d.get_aux("key2"), None)
1333
1334         d = dictutil.AuxValueDict({1:2,3:4})
1335         self.failUnlessEqual(sorted(d.keys()), [1,3])
1336         self.failUnlessEqual(d[1], 2)
1337         self.failUnlessEqual(d.get_aux(1), None)
1338
1339         d = dictutil.AuxValueDict([ (1,2), (3,4) ])
1340         self.failUnlessEqual(sorted(d.keys()), [1,3])
1341         self.failUnlessEqual(d[1], 2)
1342         self.failUnlessEqual(d.get_aux(1), None)
1343
1344         d = dictutil.AuxValueDict(one=1, two=2)
1345         self.failUnlessEqual(sorted(d.keys()), ["one","two"])
1346         self.failUnlessEqual(d["one"], 1)
1347         self.failUnlessEqual(d.get_aux("one"), None)
1348
1349 class Pipeline(unittest.TestCase):
1350     def pause(self, *args, **kwargs):
1351         d = defer.Deferred()
1352         self.calls.append( (d, args, kwargs) )
1353         return d
1354
1355     def failUnlessCallsAre(self, expected):
1356         #print self.calls
1357         #print expected
1358         self.failUnlessEqual(len(self.calls), len(expected), self.calls)
1359         for i,c in enumerate(self.calls):
1360             self.failUnlessEqual(c[1:], expected[i], str(i))
1361
1362     def test_basic(self):
1363         self.calls = []
1364         finished = []
1365         p = pipeline.Pipeline(100)
1366
1367         d = p.flush() # fires immediately
1368         d.addCallbacks(finished.append, log.err)
1369         self.failUnlessEqual(len(finished), 1)
1370         finished = []
1371
1372         d = p.add(10, self.pause, "one")
1373         # the call should start right away, and our return Deferred should
1374         # fire right away
1375         d.addCallbacks(finished.append, log.err)
1376         self.failUnlessEqual(len(finished), 1)
1377         self.failUnlessEqual(finished[0], None)
1378         self.failUnlessCallsAre([ ( ("one",) , {} ) ])
1379         self.failUnlessEqual(p.gauge, 10)
1380
1381         # pipeline: [one]
1382
1383         finished = []
1384         d = p.add(20, self.pause, "two", kw=2)
1385         # pipeline: [one, two]
1386
1387         # the call and the Deferred should fire right away
1388         d.addCallbacks(finished.append, log.err)
1389         self.failUnlessEqual(len(finished), 1)
1390         self.failUnlessEqual(finished[0], None)
1391         self.failUnlessCallsAre([ ( ("one",) , {} ),
1392                                   ( ("two",) , {"kw": 2} ),
1393                                   ])
1394         self.failUnlessEqual(p.gauge, 30)
1395
1396         self.calls[0][0].callback("one-result")
1397         # pipeline: [two]
1398         self.failUnlessEqual(p.gauge, 20)
1399
1400         finished = []
1401         d = p.add(90, self.pause, "three", "posarg1")
1402         # pipeline: [two, three]
1403         flushed = []
1404         fd = p.flush()
1405         fd.addCallbacks(flushed.append, log.err)
1406         self.failUnlessEqual(flushed, [])
1407
1408         # the call will be made right away, but the return Deferred will not,
1409         # because the pipeline is now full.
1410         d.addCallbacks(finished.append, log.err)
1411         self.failUnlessEqual(len(finished), 0)
1412         self.failUnlessCallsAre([ ( ("one",) , {} ),
1413                                   ( ("two",) , {"kw": 2} ),
1414                                   ( ("three", "posarg1"), {} ),
1415                                   ])
1416         self.failUnlessEqual(p.gauge, 110)
1417
1418         self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause)
1419
1420         # retiring either call will unblock the pipeline, causing the #3
1421         # Deferred to fire
1422         self.calls[2][0].callback("three-result")
1423         # pipeline: [two]
1424
1425         self.failUnlessEqual(len(finished), 1)
1426         self.failUnlessEqual(finished[0], None)
1427         self.failUnlessEqual(flushed, [])
1428
1429         # retiring call#2 will finally allow the flush() Deferred to fire
1430         self.calls[1][0].callback("two-result")
1431         self.failUnlessEqual(len(flushed), 1)
1432
1433     def test_errors(self):
1434         self.calls = []
1435         p = pipeline.Pipeline(100)
1436
1437         d1 = p.add(200, self.pause, "one")
1438         d2 = p.flush()
1439
1440         finished = []
1441         d1.addBoth(finished.append)
1442         self.failUnlessEqual(finished, [])
1443
1444         flushed = []
1445         d2.addBoth(flushed.append)
1446         self.failUnlessEqual(flushed, [])
1447
1448         self.calls[0][0].errback(ValueError("oops"))
1449
1450         self.failUnlessEqual(len(finished), 1)
1451         f = finished[0]
1452         self.failUnless(isinstance(f, Failure))
1453         self.failUnless(f.check(pipeline.PipelineError))
1454         self.failUnlessIn("PipelineError", str(f.value))
1455         self.failUnlessIn("ValueError", str(f.value))
1456         r = repr(f.value)
1457         self.failUnless("ValueError" in r, r)
1458         f2 = f.value.error
1459         self.failUnless(f2.check(ValueError))
1460
1461         self.failUnlessEqual(len(flushed), 1)
1462         f = flushed[0]
1463         self.failUnless(isinstance(f, Failure))
1464         self.failUnless(f.check(pipeline.PipelineError))
1465         f2 = f.value.error
1466         self.failUnless(f2.check(ValueError))
1467
1468         # now that the pipeline is in the failed state, any new calls will
1469         # fail immediately
1470
1471         d3 = p.add(20, self.pause, "two")
1472
1473         finished = []
1474         d3.addBoth(finished.append)
1475         self.failUnlessEqual(len(finished), 1)
1476         f = finished[0]
1477         self.failUnless(isinstance(f, Failure))
1478         self.failUnless(f.check(pipeline.PipelineError))
1479         r = repr(f.value)
1480         self.failUnless("ValueError" in r, r)
1481         f2 = f.value.error
1482         self.failUnless(f2.check(ValueError))
1483
1484         d4 = p.flush()
1485         flushed = []
1486         d4.addBoth(flushed.append)
1487         self.failUnlessEqual(len(flushed), 1)
1488         f = flushed[0]
1489         self.failUnless(isinstance(f, Failure))
1490         self.failUnless(f.check(pipeline.PipelineError))
1491         f2 = f.value.error
1492         self.failUnless(f2.check(ValueError))
1493
1494     def test_errors2(self):
1495         self.calls = []
1496         p = pipeline.Pipeline(100)
1497
1498         d1 = p.add(10, self.pause, "one")
1499         d2 = p.add(20, self.pause, "two")
1500         d3 = p.add(30, self.pause, "three")
1501         d4 = p.flush()
1502
1503         # one call fails, then the second one succeeds: make sure
1504         # ExpandableDeferredList tolerates the second one
1505
1506         flushed = []
1507         d4.addBoth(flushed.append)
1508         self.failUnlessEqual(flushed, [])
1509
1510         self.calls[0][0].errback(ValueError("oops"))
1511         self.failUnlessEqual(len(flushed), 1)
1512         f = flushed[0]
1513         self.failUnless(isinstance(f, Failure))
1514         self.failUnless(f.check(pipeline.PipelineError))
1515         f2 = f.value.error
1516         self.failUnless(f2.check(ValueError))
1517
1518         self.calls[1][0].callback("two-result")
1519         self.calls[2][0].errback(ValueError("three-error"))
1520
1521         del d1,d2,d3,d4
1522
1523 class SampleError(Exception):
1524     pass
1525
1526 class Log(unittest.TestCase):
1527     def test_err(self):
1528         if not hasattr(self, "flushLoggedErrors"):
1529             # without flushLoggedErrors, we can't get rid of the
1530             # twisted.log.err that tahoe_log records, so we can't keep this
1531             # test from [ERROR]ing
1532             raise unittest.SkipTest("needs flushLoggedErrors from Twisted-2.5.0")
1533         try:
1534             raise SampleError("simple sample")
1535         except:
1536             f = Failure()
1537         tahoe_log.err(format="intentional sample error",
1538                       failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ")
1539         self.flushLoggedErrors(SampleError)