src/allmydata/test/test_util.py

   1
   2 def foo(): pass # keep the line number constant
   3
   4 import os, time, sys
   5 from StringIO import StringIO
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.python.failure import Failure
   9 from twisted.python import log
  10
  11 from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
  12 from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
  13 from allmydata.util import limiter, time_format, pollmixin, cachedir
  14 from allmydata.util import statistics, dictutil, pipeline
  15 from allmydata.util import log as tahoe_log
  16
  17 class Base32(unittest.TestCase):
  18     def test_b2a_matches_Pythons(self):
  19         import base64
  20         y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0"
  21         x = base64.b32encode(y)
  22         while x and x[-1] == '=':
  23             x = x[:-1]
  24         x = x.lower()
  25         self.failUnlessEqual(base32.b2a(y), x)
  26     def test_b2a(self):
  27         self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a")
  28     def test_b2a_or_none(self):
  29         self.failUnlessEqual(base32.b2a_or_none(None), None)
  30         self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a")
  31     def test_a2b(self):
  32         self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34")
  33         self.failUnlessRaises(AssertionError, base32.a2b, "b0gus")
  34
  35 class IDLib(unittest.TestCase):
  36     def test_nodeid_b2a(self):
  37         self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
  38
  39 class NoArgumentException(Exception):
  40     def __init__(self):
  41         pass
  42
  43 class HumanReadable(unittest.TestCase):
  44     def test_repr(self):
  45         hr = humanreadable.hr
  46         self.failUnlessEqual(hr(foo), "<foo() at test_util.py:2>")
  47         self.failUnlessEqual(hr(self.test_repr),
  48                              "<bound method HumanReadable.test_repr of <allmydata.test.test_util.HumanReadable testMethod=test_repr>>")
  49         self.failUnlessEqual(hr(1L), "1")
  50         self.failUnlessEqual(hr(10**40),
  51                              "100000000000000000...000000000000000000")
  52         self.failUnlessEqual(hr(self), "<allmydata.test.test_util.HumanReadable testMethod=test_repr>")
  53         self.failUnlessEqual(hr([1,2]), "[1, 2]")
  54         self.failUnlessEqual(hr({1:2}), "{1:2}")
  55         try:
  56             raise ValueError
  57         except Exception, e:
  58             self.failUnless(
  59                 hr(e) == "<ValueError: ()>" # python-2.4
  60                 or hr(e) == "ValueError()") # python-2.5
  61         try:
  62             raise ValueError("oops")
  63         except Exception, e:
  64             self.failUnless(
  65                 hr(e) == "<ValueError: 'oops'>" # python-2.4
  66                 or hr(e) == "ValueError('oops',)") # python-2.5
  67         try:
  68             raise NoArgumentException
  69         except Exception, e:
  70             self.failUnless(
  71                 hr(e) == "<NoArgumentException>" # python-2.4
  72                 or hr(e) == "NoArgumentException()") # python-2.5
  73
  74
  75 class MyList(list):
  76     pass
  77
  78 class Math(unittest.TestCase):
  79     def test_div_ceil(self):
  80         f = mathutil.div_ceil
  81         self.failUnlessEqual(f(0, 1), 0)
  82         self.failUnlessEqual(f(0, 2), 0)
  83         self.failUnlessEqual(f(0, 3), 0)
  84         self.failUnlessEqual(f(1, 3), 1)
  85         self.failUnlessEqual(f(2, 3), 1)
  86         self.failUnlessEqual(f(3, 3), 1)
  87         self.failUnlessEqual(f(4, 3), 2)
  88         self.failUnlessEqual(f(5, 3), 2)
  89         self.failUnlessEqual(f(6, 3), 2)
  90         self.failUnlessEqual(f(7, 3), 3)
  91
  92     def test_next_multiple(self):
  93         f = mathutil.next_multiple
  94         self.failUnlessEqual(f(5, 1), 5)
  95         self.failUnlessEqual(f(5, 2), 6)
  96         self.failUnlessEqual(f(5, 3), 6)
  97         self.failUnlessEqual(f(5, 4), 8)
  98         self.failUnlessEqual(f(5, 5), 5)
  99         self.failUnlessEqual(f(5, 6), 6)
 100         self.failUnlessEqual(f(32, 1), 32)
 101         self.failUnlessEqual(f(32, 2), 32)
 102         self.failUnlessEqual(f(32, 3), 33)
 103         self.failUnlessEqual(f(32, 4), 32)
 104         self.failUnlessEqual(f(32, 5), 35)
 105         self.failUnlessEqual(f(32, 6), 36)
 106         self.failUnlessEqual(f(32, 7), 35)
 107         self.failUnlessEqual(f(32, 8), 32)
 108         self.failUnlessEqual(f(32, 9), 36)
 109         self.failUnlessEqual(f(32, 10), 40)
 110         self.failUnlessEqual(f(32, 11), 33)
 111         self.failUnlessEqual(f(32, 12), 36)
 112         self.failUnlessEqual(f(32, 13), 39)
 113         self.failUnlessEqual(f(32, 14), 42)
 114         self.failUnlessEqual(f(32, 15), 45)
 115         self.failUnlessEqual(f(32, 16), 32)
 116         self.failUnlessEqual(f(32, 17), 34)
 117         self.failUnlessEqual(f(32, 18), 36)
 118         self.failUnlessEqual(f(32, 589), 589)
 119
 120     def test_pad_size(self):
 121         f = mathutil.pad_size
 122         self.failUnlessEqual(f(0, 4), 0)
 123         self.failUnlessEqual(f(1, 4), 3)
 124         self.failUnlessEqual(f(2, 4), 2)
 125         self.failUnlessEqual(f(3, 4), 1)
 126         self.failUnlessEqual(f(4, 4), 0)
 127         self.failUnlessEqual(f(5, 4), 3)
 128
 129     def test_is_power_of_k(self):
 130         f = mathutil.is_power_of_k
 131         for i in range(1, 100):
 132             if i in (1, 2, 4, 8, 16, 32, 64):
 133                 self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
 134             else:
 135                 self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
 136         for i in range(1, 100):
 137             if i in (1, 3, 9, 27, 81):
 138                 self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
 139             else:
 140                 self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
 141
 142     def test_next_power_of_k(self):
 143         f = mathutil.next_power_of_k
 144         self.failUnlessEqual(f(0,2), 1)
 145         self.failUnlessEqual(f(1,2), 1)
 146         self.failUnlessEqual(f(2,2), 2)
 147         self.failUnlessEqual(f(3,2), 4)
 148         self.failUnlessEqual(f(4,2), 4)
 149         for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
 150         for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
 151         for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
 152         for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
 153         for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
 154
 155         self.failUnlessEqual(f(0,3), 1)
 156         self.failUnlessEqual(f(1,3), 1)
 157         self.failUnlessEqual(f(2,3), 3)
 158         self.failUnlessEqual(f(3,3), 3)
 159         for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
 160         for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
 161         for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
 162         for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
 163
 164     def test_ave(self):
 165         f = mathutil.ave
 166         self.failUnlessEqual(f([1,2,3]), 2)
 167         self.failUnlessEqual(f([0,0,0,4]), 1)
 168         self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
 169
 170     def test_round_sigfigs(self):
 171         f = mathutil.round_sigfigs
 172         self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002)
 173
 174 class Statistics(unittest.TestCase):
 175     def should_assert(self, msg, func, *args, **kwargs):
 176         try:
 177             func(*args, **kwargs)
 178             self.fail(msg)
 179         except AssertionError:
 180             pass
 181
 182     def failUnlessListEqual(self, a, b, msg = None):
 183         self.failUnlessEqual(len(a), len(b))
 184         for i in range(len(a)):
 185             self.failUnlessEqual(a[i], b[i], msg)
 186
 187     def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None):
 188         self.failUnlessEqual(len(a), len(b))
 189         for i in range(len(a)):
 190             self.failUnlessAlmostEqual(a[i], b[i], places, msg)
 191
 192     def test_binomial_coeff(self):
 193         f = statistics.binomial_coeff
 194         self.failUnlessEqual(f(20, 0), 1)
 195         self.failUnlessEqual(f(20, 1), 20)
 196         self.failUnlessEqual(f(20, 2), 190)
 197         self.failUnlessEqual(f(20, 8), f(20, 12))
 198         self.should_assert("Should assert if n < k", f, 2, 3)
 199
 200     def test_binomial_distribution_pmf(self):
 201         f = statistics.binomial_distribution_pmf
 202
 203         pmf_comp = f(2, .1)
 204         pmf_stat = [0.81, 0.18, 0.01]
 205         self.failUnlessListAlmostEqual(pmf_comp, pmf_stat)
 206
 207         # Summing across a PMF should give the total probability 1
 208         self.failUnlessAlmostEqual(sum(pmf_comp), 1)
 209         self.should_assert("Should assert if not 0<=p<=1", f, 1, -1)
 210         self.should_assert("Should assert if n < 1", f, 0, .1)
 211
 212         out = StringIO()
 213         statistics.print_pmf(pmf_comp, out=out)
 214         lines = out.getvalue().splitlines()
 215         self.failUnlessEqual(lines[0], "i=0: 0.81")
 216         self.failUnlessEqual(lines[1], "i=1: 0.18")
 217         self.failUnlessEqual(lines[2], "i=2: 0.01")
 218
 219     def test_survival_pmf(self):
 220         f = statistics.survival_pmf
 221         # Cross-check binomial-distribution method against convolution
 222         # method.
 223         p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20
 224         pmf1 = statistics.survival_pmf_via_conv(p_list)
 225         pmf2 = statistics.survival_pmf_via_bd(p_list)
 226         self.failUnlessListAlmostEqual(pmf1, pmf2)
 227         self.failUnlessTrue(statistics.valid_pmf(pmf1))
 228         self.should_assert("Should assert if p_i > 1", f, [1.1]);
 229         self.should_assert("Should assert if p_i < 0", f, [-.1]);
 230
 231     def test_repair_count_pmf(self):
 232         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 233         repair_pmf = statistics.repair_count_pmf(survival_pmf, 3)
 234         # repair_pmf[0] == sum(survival_pmf[0,1,2,5])
 235         # repair_pmf[1] == survival_pmf[4]
 236         # repair_pmf[2] = survival_pmf[3]
 237         self.failUnlessListAlmostEqual(repair_pmf,
 238                                        [0.00001 + 0.00045 + 0.0081 + 0.59049,
 239                                         .32805,
 240                                         .0729,
 241                                         0, 0, 0])
 242
 243     def test_repair_cost(self):
 244         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 245         bwcost = statistics.bandwidth_cost_function
 246         cost = statistics.mean_repair_cost(bwcost, 1000,
 247                                            survival_pmf, 3, ul_dl_ratio=1.0)
 248         self.failUnlessAlmostEqual(cost, 558.90)
 249         cost = statistics.mean_repair_cost(bwcost, 1000,
 250                                            survival_pmf, 3, ul_dl_ratio=8.0)
 251         self.failUnlessAlmostEqual(cost, 1664.55)
 252
 253         # I haven't manually checked the math beyond here -warner
 254         cost = statistics.eternal_repair_cost(bwcost, 1000,
 255                                               survival_pmf, 3,
 256                                               discount_rate=0, ul_dl_ratio=1.0)
 257         self.failUnlessAlmostEqual(cost, 65292.056074766246)
 258         cost = statistics.eternal_repair_cost(bwcost, 1000,
 259                                               survival_pmf, 3,
 260                                               discount_rate=0.05,
 261                                               ul_dl_ratio=1.0)
 262         self.failUnlessAlmostEqual(cost, 9133.6097158191551)
 263
 264     def test_convolve(self):
 265         f = statistics.convolve
 266         v1 = [ 1, 2, 3 ]
 267         v2 = [ 4, 5, 6 ]
 268         v3 = [ 7, 8 ]
 269         v1v2result = [ 4, 13, 28, 27, 18 ]
 270         # Convolution is commutative
 271         r1 = f(v1, v2)
 272         r2 = f(v2, v1)
 273         self.failUnlessListEqual(r1, r2, "Convolution should be commutative")
 274         self.failUnlessListEqual(r1, v1v2result, "Didn't match known result")
 275         # Convolution is associative
 276         r1 = f(f(v1, v2), v3)
 277         r2 = f(v1, f(v2, v3))
 278         self.failUnlessListEqual(r1, r2, "Convolution should be associative")
 279         # Convolution is distributive
 280         r1 = f(v3, [ a + b for a, b in zip(v1, v2) ])
 281         tmp1 = f(v3, v1)
 282         tmp2 = f(v3, v2)
 283         r2 = [ a + b for a, b in zip(tmp1, tmp2) ]
 284         self.failUnlessListEqual(r1, r2, "Convolution should be distributive")
 285         # Convolution is scalar multiplication associative
 286         tmp1 = f(v1, v2)
 287         r1 = [ a * 4 for a in tmp1 ]
 288         tmp2 = [ a * 4 for a in v1 ]
 289         r2 = f(tmp2, v2)
 290         self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative")
 291
 292     def test_find_k(self):
 293         f = statistics.find_k
 294         g = statistics.pr_file_loss
 295         plist = [.9] * 10 + [.8] * 10 # N=20
 296         t = .0001
 297         k = f(plist, t)
 298         self.failUnlessEqual(k, 10)
 299         self.failUnless(g(plist, k) < t)
 300
 301     def test_pr_file_loss(self):
 302         f = statistics.pr_file_loss
 303         plist = [.5] * 10
 304         self.failUnlessEqual(f(plist, 3), .0546875)
 305
 306     def test_pr_backup_file_loss(self):
 307         f = statistics.pr_backup_file_loss
 308         plist = [.5] * 10
 309         self.failUnlessEqual(f(plist, .5, 3), .02734375)
 310
 311
 312 class Asserts(unittest.TestCase):
 313     def should_assert(self, func, *args, **kwargs):
 314         try:
 315             func(*args, **kwargs)
 316         except AssertionError, e:
 317             return str(e)
 318         except Exception, e:
 319             self.fail("assert failed with non-AssertionError: %s" % e)
 320         self.fail("assert was not caught")
 321
 322     def should_not_assert(self, func, *args, **kwargs):
 323         try:
 324             func(*args, **kwargs)
 325         except AssertionError, e:
 326             self.fail("assertion fired when it should not have: %s" % e)
 327         except Exception, e:
 328             self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e)
 329         return # we're happy
 330
 331
 332     def test_assert(self):
 333         f = assertutil._assert
 334         self.should_assert(f)
 335         self.should_assert(f, False)
 336         self.should_not_assert(f, True)
 337
 338         m = self.should_assert(f, False, "message")
 339         self.failUnlessEqual(m, "'message' <type 'str'>", m)
 340         m = self.should_assert(f, False, "message1", othermsg=12)
 341         self.failUnlessEqual("'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 342         m = self.should_assert(f, False, othermsg="message2")
 343         self.failUnlessEqual("othermsg: 'message2' <type 'str'>", m)
 344
 345     def test_precondition(self):
 346         f = assertutil.precondition
 347         self.should_assert(f)
 348         self.should_assert(f, False)
 349         self.should_not_assert(f, True)
 350
 351         m = self.should_assert(f, False, "message")
 352         self.failUnlessEqual("precondition: 'message' <type 'str'>", m)
 353         m = self.should_assert(f, False, "message1", othermsg=12)
 354         self.failUnlessEqual("precondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 355         m = self.should_assert(f, False, othermsg="message2")
 356         self.failUnlessEqual("precondition: othermsg: 'message2' <type 'str'>", m)
 357
 358     def test_postcondition(self):
 359         f = assertutil.postcondition
 360         self.should_assert(f)
 361         self.should_assert(f, False)
 362         self.should_not_assert(f, True)
 363
 364         m = self.should_assert(f, False, "message")
 365         self.failUnlessEqual("postcondition: 'message' <type 'str'>", m)
 366         m = self.should_assert(f, False, "message1", othermsg=12)
 367         self.failUnlessEqual("postcondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 368         m = self.should_assert(f, False, othermsg="message2")
 369         self.failUnlessEqual("postcondition: othermsg: 'message2' <type 'str'>", m)
 370
 371 class FileUtil(unittest.TestCase):
 372     def mkdir(self, basedir, path, mode=0777):
 373         fn = os.path.join(basedir, path)
 374         fileutil.make_dirs(fn, mode)
 375
 376     def touch(self, basedir, path, mode=None, data="touch\n"):
 377         fn = os.path.join(basedir, path)
 378         f = open(fn, "w")
 379         f.write(data)
 380         f.close()
 381         if mode is not None:
 382             os.chmod(fn, mode)
 383
 384     def test_rm_dir(self):
 385         basedir = "util/FileUtil/test_rm_dir"
 386         fileutil.make_dirs(basedir)
 387         # create it again to test idempotency
 388         fileutil.make_dirs(basedir)
 389         d = os.path.join(basedir, "doomed")
 390         self.mkdir(d, "a/b")
 391         self.touch(d, "a/b/1.txt")
 392         self.touch(d, "a/b/2.txt", 0444)
 393         self.touch(d, "a/b/3.txt", 0)
 394         self.mkdir(d, "a/c")
 395         self.touch(d, "a/c/1.txt")
 396         self.touch(d, "a/c/2.txt", 0444)
 397         self.touch(d, "a/c/3.txt", 0)
 398         os.chmod(os.path.join(d, "a/c"), 0444)
 399         self.mkdir(d, "a/d")
 400         self.touch(d, "a/d/1.txt")
 401         self.touch(d, "a/d/2.txt", 0444)
 402         self.touch(d, "a/d/3.txt", 0)
 403         os.chmod(os.path.join(d, "a/d"), 0)
 404
 405         fileutil.rm_dir(d)
 406         self.failIf(os.path.exists(d))
 407         # remove it again to test idempotency
 408         fileutil.rm_dir(d)
 409
 410     def test_remove_if_possible(self):
 411         basedir = "util/FileUtil/test_remove_if_possible"
 412         fileutil.make_dirs(basedir)
 413         self.touch(basedir, "here")
 414         fn = os.path.join(basedir, "here")
 415         fileutil.remove_if_possible(fn)
 416         self.failIf(os.path.exists(fn))
 417         fileutil.remove_if_possible(fn) # should be idempotent
 418         fileutil.rm_dir(basedir)
 419         fileutil.remove_if_possible(fn) # should survive errors
 420
 421     def test_open_or_create(self):
 422         basedir = "util/FileUtil/test_open_or_create"
 423         fileutil.make_dirs(basedir)
 424         fn = os.path.join(basedir, "here")
 425         f = fileutil.open_or_create(fn)
 426         f.write("stuff.")
 427         f.close()
 428         f = fileutil.open_or_create(fn)
 429         f.seek(0, 2)
 430         f.write("more.")
 431         f.close()
 432         f = open(fn, "r")
 433         data = f.read()
 434         f.close()
 435         self.failUnlessEqual(data, "stuff.more.")
 436
 437     def test_NamedTemporaryDirectory(self):
 438         basedir = "util/FileUtil/test_NamedTemporaryDirectory"
 439         fileutil.make_dirs(basedir)
 440         td = fileutil.NamedTemporaryDirectory(dir=basedir)
 441         name = td.name
 442         self.failUnless(basedir in name)
 443         self.failUnless(basedir in repr(td))
 444         self.failUnless(os.path.isdir(name))
 445         del td
 446         # it is conceivable that we need to force gc here, but I'm not sure
 447         self.failIf(os.path.isdir(name))
 448
 449     def test_rename(self):
 450         basedir = "util/FileUtil/test_rename"
 451         fileutil.make_dirs(basedir)
 452         self.touch(basedir, "here")
 453         fn = os.path.join(basedir, "here")
 454         fn2 = os.path.join(basedir, "there")
 455         fileutil.rename(fn, fn2)
 456         self.failIf(os.path.exists(fn))
 457         self.failUnless(os.path.exists(fn2))
 458
 459     def test_du(self):
 460         basedir = "util/FileUtil/test_du"
 461         fileutil.make_dirs(basedir)
 462         d = os.path.join(basedir, "space-consuming")
 463         self.mkdir(d, "a/b")
 464         self.touch(d, "a/b/1.txt", data="a"*10)
 465         self.touch(d, "a/b/2.txt", data="b"*11)
 466         self.mkdir(d, "a/c")
 467         self.touch(d, "a/c/1.txt", data="c"*12)
 468         self.touch(d, "a/c/2.txt", data="d"*13)
 469
 470         used = fileutil.du(basedir)
 471         self.failUnlessEqual(10+11+12+13, used)
 472
 473     def test_abspath_expanduser_unicode(self):
 474         self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring")
 475
 476         saved_cwd = os.path.normpath(os.getcwdu())
 477         abspath_cwd = fileutil.abspath_expanduser_unicode(u".")
 478         self.failUnless(isinstance(saved_cwd, unicode), saved_cwd)
 479         self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd)
 480         self.failUnlessEqual(abspath_cwd, saved_cwd)
 481
 482         # adapted from <http://svn.python.org/view/python/branches/release26-maint/Lib/test/test_posixpath.py?view=markup&pathrev=78279#test_abspath>
 483
 484         self.failUnlessIn(u"foo", fileutil.abspath_expanduser_unicode(u"foo"))
 485         self.failIfIn(u"~", fileutil.abspath_expanduser_unicode(u"~"))
 486
 487         cwds = ['cwd']
 488         try:
 489             cwds.append(u'\xe7w\xf0'.encode(sys.getfilesystemencoding()
 490                                             or 'ascii'))
 491         except UnicodeEncodeError:
 492             pass # the cwd can't be encoded -- test with ascii cwd only
 493
 494         for cwd in cwds:
 495             try:
 496                 os.mkdir(cwd)
 497                 os.chdir(cwd)
 498                 for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'):
 499                     uabspath = fileutil.abspath_expanduser_unicode(upath)
 500                     self.failUnless(isinstance(uabspath, unicode), uabspath)
 501             finally:
 502                 os.chdir(saved_cwd)
 503
 504 class PollMixinTests(unittest.TestCase):
 505     def setUp(self):
 506         self.pm = pollmixin.PollMixin()
 507
 508     def test_PollMixin_True(self):
 509         d = self.pm.poll(check_f=lambda : True,
 510                          pollinterval=0.1)
 511         return d
 512
 513     def test_PollMixin_False_then_True(self):
 514         i = iter([False, True])
 515         d = self.pm.poll(check_f=i.next,
 516                          pollinterval=0.1)
 517         return d
 518
 519     def test_timeout(self):
 520         d = self.pm.poll(check_f=lambda: False,
 521                          pollinterval=0.01,
 522                          timeout=1)
 523         def _suc(res):
 524             self.fail("poll should have failed, not returned %s" % (res,))
 525         def _err(f):
 526             f.trap(pollmixin.TimeoutError)
 527             return None # success
 528         d.addCallbacks(_suc, _err)
 529         return d
 530
 531 class DeferredUtilTests(unittest.TestCase):
 532     def test_gather_results(self):
 533         d1 = defer.Deferred()
 534         d2 = defer.Deferred()
 535         res = deferredutil.gatherResults([d1, d2])
 536         d1.errback(ValueError("BAD"))
 537         def _callb(res):
 538             self.fail("Should have errbacked, not resulted in %s" % (res,))
 539         def _errb(thef):
 540             thef.trap(ValueError)
 541         res.addCallbacks(_callb, _errb)
 542         return res
 543
 544     def test_success(self):
 545         d1, d2 = defer.Deferred(), defer.Deferred()
 546         good = []
 547         bad = []
 548         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 549         dlss.addCallbacks(good.append, bad.append)
 550         d1.callback(1)
 551         d2.callback(2)
 552         self.failUnlessEqual(good, [[1,2]])
 553         self.failUnlessEqual(bad, [])
 554
 555     def test_failure(self):
 556         d1, d2 = defer.Deferred(), defer.Deferred()
 557         good = []
 558         bad = []
 559         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 560         dlss.addCallbacks(good.append, bad.append)
 561         d1.addErrback(lambda _ignore: None)
 562         d2.addErrback(lambda _ignore: None)
 563         d1.callback(1)
 564         d2.errback(ValueError())
 565         self.failUnlessEqual(good, [])
 566         self.failUnlessEqual(len(bad), 1)
 567         f = bad[0]
 568         self.failUnless(isinstance(f, Failure))
 569         self.failUnless(f.check(ValueError))
 570
 571 class HashUtilTests(unittest.TestCase):
 572
 573     def test_random_key(self):
 574         k = hashutil.random_key()
 575         self.failUnlessEqual(len(k), hashutil.KEYLEN)
 576
 577     def test_sha256d(self):
 578         h1 = hashutil.tagged_hash("tag1", "value")
 579         h2 = hashutil.tagged_hasher("tag1")
 580         h2.update("value")
 581         h2a = h2.digest()
 582         h2b = h2.digest()
 583         self.failUnlessEqual(h1, h2a)
 584         self.failUnlessEqual(h2a, h2b)
 585
 586     def test_sha256d_truncated(self):
 587         h1 = hashutil.tagged_hash("tag1", "value", 16)
 588         h2 = hashutil.tagged_hasher("tag1", 16)
 589         h2.update("value")
 590         h2 = h2.digest()
 591         self.failUnlessEqual(len(h1), 16)
 592         self.failUnlessEqual(len(h2), 16)
 593         self.failUnlessEqual(h1, h2)
 594
 595     def test_chk(self):
 596         h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret")
 597         h2 = hashutil.convergence_hasher(3, 10, 1000, "secret")
 598         h2.update("data")
 599         h2 = h2.digest()
 600         self.failUnlessEqual(h1, h2)
 601
 602     def test_hashers(self):
 603         h1 = hashutil.block_hash("foo")
 604         h2 = hashutil.block_hasher()
 605         h2.update("foo")
 606         self.failUnlessEqual(h1, h2.digest())
 607
 608         h1 = hashutil.uri_extension_hash("foo")
 609         h2 = hashutil.uri_extension_hasher()
 610         h2.update("foo")
 611         self.failUnlessEqual(h1, h2.digest())
 612
 613         h1 = hashutil.plaintext_hash("foo")
 614         h2 = hashutil.plaintext_hasher()
 615         h2.update("foo")
 616         self.failUnlessEqual(h1, h2.digest())
 617
 618         h1 = hashutil.crypttext_hash("foo")
 619         h2 = hashutil.crypttext_hasher()
 620         h2.update("foo")
 621         self.failUnlessEqual(h1, h2.digest())
 622
 623         h1 = hashutil.crypttext_segment_hash("foo")
 624         h2 = hashutil.crypttext_segment_hasher()
 625         h2.update("foo")
 626         self.failUnlessEqual(h1, h2.digest())
 627
 628         h1 = hashutil.plaintext_segment_hash("foo")
 629         h2 = hashutil.plaintext_segment_hasher()
 630         h2.update("foo")
 631         self.failUnlessEqual(h1, h2.digest())
 632
 633     def test_constant_time_compare(self):
 634         self.failUnless(hashutil.constant_time_compare("a", "a"))
 635         self.failUnless(hashutil.constant_time_compare("ab", "ab"))
 636         self.failIf(hashutil.constant_time_compare("a", "b"))
 637         self.failIf(hashutil.constant_time_compare("a", "aa"))
 638
 639     def _testknown(self, hashf, expected_a, *args):
 640         got = hashf(*args)
 641         got_a = base32.b2a(got)
 642         self.failUnlessEqual(got_a, expected_a)
 643
 644     def test_known_answers(self):
 645         # assert backwards compatibility
 646         self._testknown(hashutil.storage_index_hash, "qb5igbhcc5esa6lwqorsy7e6am", "")
 647         self._testknown(hashutil.block_hash, "msjr5bh4evuh7fa3zw7uovixfbvlnstr5b65mrerwfnvjxig2jvq", "")
 648         self._testknown(hashutil.uri_extension_hash, "wthsu45q7zewac2mnivoaa4ulh5xvbzdmsbuyztq2a5fzxdrnkka", "")
 649         self._testknown(hashutil.plaintext_hash, "5lz5hwz3qj3af7n6e3arblw7xzutvnd3p3fjsngqjcb7utf3x3da", "")
 650         self._testknown(hashutil.crypttext_hash, "itdj6e4njtkoiavlrmxkvpreosscssklunhwtvxn6ggho4rkqwga", "")
 651         self._testknown(hashutil.crypttext_segment_hash, "aovy5aa7jej6ym5ikgwyoi4pxawnoj3wtaludjz7e2nb5xijb7aa", "")
 652         self._testknown(hashutil.plaintext_segment_hash, "4fdgf6qruaisyukhqcmoth4t3li6bkolbxvjy4awwcpprdtva7za", "")
 653         self._testknown(hashutil.convergence_hash, "3mo6ni7xweplycin6nowynw2we", 3, 10, 100, "", "converge")
 654         self._testknown(hashutil.my_renewal_secret_hash, "ujhr5k5f7ypkp67jkpx6jl4p47pyta7hu5m527cpcgvkafsefm6q", "")
 655         self._testknown(hashutil.my_cancel_secret_hash, "rjwzmafe2duixvqy6h47f5wfrokdziry6zhx4smew4cj6iocsfaa", "")
 656         self._testknown(hashutil.file_renewal_secret_hash, "hzshk2kf33gzbd5n3a6eszkf6q6o6kixmnag25pniusyaulqjnia", "", "si")
 657         self._testknown(hashutil.file_cancel_secret_hash, "bfciwvr6w7wcavsngxzxsxxaszj72dej54n4tu2idzp6b74g255q", "", "si")
 658         self._testknown(hashutil.bucket_renewal_secret_hash, "e7imrzgzaoashsncacvy3oysdd2m5yvtooo4gmj4mjlopsazmvuq", "", "\x00"*20)
 659         self._testknown(hashutil.bucket_cancel_secret_hash, "dvdujeyxeirj6uux6g7xcf4lvesk632aulwkzjar7srildvtqwma", "", "\x00"*20)
 660         self._testknown(hashutil.hmac, "c54ypfi6pevb3nvo6ba42jtglpkry2kbdopqsi7dgrm4r7tw5sra", "tag", "")
 661         self._testknown(hashutil.mutable_rwcap_key_hash, "6rvn2iqrghii5n4jbbwwqqsnqu", "iv", "wk")
 662         self._testknown(hashutil.ssk_writekey_hash, "ykpgmdbpgbb6yqz5oluw2q26ye", "")
 663         self._testknown(hashutil.ssk_write_enabler_master_hash, "izbfbfkoait4dummruol3gy2bnixrrrslgye6ycmkuyujnenzpia", "")
 664         self._testknown(hashutil.ssk_write_enabler_hash, "fuu2dvx7g6gqu5x22vfhtyed7p4pd47y5hgxbqzgrlyvxoev62tq", "wk", "\x00"*20)
 665         self._testknown(hashutil.ssk_pubkey_fingerprint_hash, "3opzw4hhm2sgncjx224qmt5ipqgagn7h5zivnfzqycvgqgmgz35q", "")
 666         self._testknown(hashutil.ssk_readkey_hash, "vugid4as6qbqgeq2xczvvcedai", "")
 667         self._testknown(hashutil.ssk_readkey_data_hash, "73wsaldnvdzqaf7v4pzbr2ae5a", "iv", "rk")
 668         self._testknown(hashutil.ssk_storage_index_hash, "j7icz6kigb6hxrej3tv4z7ayym", "")
 669
 670
 671 class Abbreviate(unittest.TestCase):
 672     def test_time(self):
 673         a = abbreviate.abbreviate_time
 674         self.failUnlessEqual(a(None), "unknown")
 675         self.failUnlessEqual(a(0), "0 seconds")
 676         self.failUnlessEqual(a(1), "1 second")
 677         self.failUnlessEqual(a(2), "2 seconds")
 678         self.failUnlessEqual(a(119), "119 seconds")
 679         MIN = 60
 680         self.failUnlessEqual(a(2*MIN), "2 minutes")
 681         self.failUnlessEqual(a(60*MIN), "60 minutes")
 682         self.failUnlessEqual(a(179*MIN), "179 minutes")
 683         HOUR = 60*MIN
 684         self.failUnlessEqual(a(180*MIN), "3 hours")
 685         self.failUnlessEqual(a(4*HOUR), "4 hours")
 686         DAY = 24*HOUR
 687         MONTH = 30*DAY
 688         self.failUnlessEqual(a(2*DAY), "2 days")
 689         self.failUnlessEqual(a(2*MONTH), "2 months")
 690         YEAR = 365*DAY
 691         self.failUnlessEqual(a(5*YEAR), "5 years")
 692
 693     def test_space(self):
 694         tests_si = [(None, "unknown"),
 695                     (0, "0 B"),
 696                     (1, "1 B"),
 697                     (999, "999 B"),
 698                     (1000, "1000 B"),
 699                     (1023, "1023 B"),
 700                     (1024, "1.02 kB"),
 701                     (20*1000, "20.00 kB"),
 702                     (1024*1024, "1.05 MB"),
 703                     (1000*1000, "1.00 MB"),
 704                     (1000*1000*1000, "1.00 GB"),
 705                     (1000*1000*1000*1000, "1.00 TB"),
 706                     (1000*1000*1000*1000*1000, "1.00 PB"),
 707                     (1234567890123456, "1.23 PB"),
 708                     ]
 709         for (x, expected) in tests_si:
 710             got = abbreviate.abbreviate_space(x, SI=True)
 711             self.failUnlessEqual(got, expected)
 712
 713         tests_base1024 = [(None, "unknown"),
 714                           (0, "0 B"),
 715                           (1, "1 B"),
 716                           (999, "999 B"),
 717                           (1000, "1000 B"),
 718                           (1023, "1023 B"),
 719                           (1024, "1.00 kiB"),
 720                           (20*1024, "20.00 kiB"),
 721                           (1000*1000, "976.56 kiB"),
 722                           (1024*1024, "1.00 MiB"),
 723                           (1024*1024*1024, "1.00 GiB"),
 724                           (1024*1024*1024*1024, "1.00 TiB"),
 725                           (1000*1000*1000*1000*1000, "909.49 TiB"),
 726                           (1024*1024*1024*1024*1024, "1.00 PiB"),
 727                           (1234567890123456, "1.10 PiB"),
 728                     ]
 729         for (x, expected) in tests_base1024:
 730             got = abbreviate.abbreviate_space(x, SI=False)
 731             self.failUnlessEqual(got, expected)
 732
 733         self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567),
 734                              "(1.23 MB, 1.18 MiB)")
 735
 736     def test_parse_space(self):
 737         p = abbreviate.parse_abbreviated_size
 738         self.failUnlessEqual(p(""), None)
 739         self.failUnlessEqual(p(None), None)
 740         self.failUnlessEqual(p("123"), 123)
 741         self.failUnlessEqual(p("123B"), 123)
 742         self.failUnlessEqual(p("2K"), 2000)
 743         self.failUnlessEqual(p("2kb"), 2000)
 744         self.failUnlessEqual(p("2KiB"), 2048)
 745         self.failUnlessEqual(p("10MB"), 10*1000*1000)
 746         self.failUnlessEqual(p("10MiB"), 10*1024*1024)
 747         self.failUnlessEqual(p("5G"), 5*1000*1000*1000)
 748         self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024)
 749         e = self.failUnlessRaises(ValueError, p, "12 cubits")
 750         self.failUnless("12 cubits" in str(e))
 751
 752 class Limiter(unittest.TestCase):
 753     timeout = 480 # This takes longer than 240 seconds on Francois's arm box.
 754
 755     def job(self, i, foo):
 756         self.calls.append( (i, foo) )
 757         self.simultaneous += 1
 758         self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous)
 759         d = defer.Deferred()
 760         def _done():
 761             self.simultaneous -= 1
 762             d.callback("done %d" % i)
 763         reactor.callLater(1.0, _done)
 764         return d
 765
 766     def bad_job(self, i, foo):
 767         raise ValueError("bad_job %d" % i)
 768
 769     def test_limiter(self):
 770         self.calls = []
 771         self.simultaneous = 0
 772         self.peak_simultaneous = 0
 773         l = limiter.ConcurrencyLimiter()
 774         dl = []
 775         for i in range(20):
 776             dl.append(l.add(self.job, i, foo=str(i)))
 777         d = defer.DeferredList(dl, fireOnOneErrback=True)
 778         def _done(res):
 779             self.failUnlessEqual(self.simultaneous, 0)
 780             self.failUnless(self.peak_simultaneous <= 10)
 781             self.failUnlessEqual(len(self.calls), 20)
 782             for i in range(20):
 783                 self.failUnless( (i, str(i)) in self.calls)
 784         d.addCallback(_done)
 785         return d
 786
 787     def test_errors(self):
 788         self.calls = []
 789         self.simultaneous = 0
 790         self.peak_simultaneous = 0
 791         l = limiter.ConcurrencyLimiter()
 792         dl = []
 793         for i in range(20):
 794             dl.append(l.add(self.job, i, foo=str(i)))
 795         d2 = l.add(self.bad_job, 21, "21")
 796         d = defer.DeferredList(dl, fireOnOneErrback=True)
 797         def _most_done(res):
 798             results = []
 799             for (success, result) in res:
 800                 self.failUnlessEqual(success, True)
 801                 results.append(result)
 802             results.sort()
 803             expected_results = ["done %d" % i for i in range(20)]
 804             expected_results.sort()
 805             self.failUnlessEqual(results, expected_results)
 806             self.failUnless(self.peak_simultaneous <= 10)
 807             self.failUnlessEqual(len(self.calls), 20)
 808             for i in range(20):
 809                 self.failUnless( (i, str(i)) in self.calls)
 810             def _good(res):
 811                 self.fail("should have failed, not got %s" % (res,))
 812             def _err(f):
 813                 f.trap(ValueError)
 814                 self.failUnless("bad_job 21" in str(f))
 815             d2.addCallbacks(_good, _err)
 816             return d2
 817         d.addCallback(_most_done)
 818         def _all_done(res):
 819             self.failUnlessEqual(self.simultaneous, 0)
 820             self.failUnless(self.peak_simultaneous <= 10)
 821             self.failUnlessEqual(len(self.calls), 20)
 822             for i in range(20):
 823                 self.failUnless( (i, str(i)) in self.calls)
 824         d.addCallback(_all_done)
 825         return d
 826
 827 class TimeFormat(unittest.TestCase):
 828     def test_epoch(self):
 829         return self._help_test_epoch()
 830
 831     def test_epoch_in_London(self):
 832         # Europe/London is a particularly troublesome timezone.  Nowadays, its
 833         # offset from GMT is 0.  But in 1970, its offset from GMT was 1.
 834         # (Apparently in 1970 Britain had redefined standard time to be GMT+1
 835         # and stayed in standard time all year round, whereas today
 836         # Europe/London standard time is GMT and Europe/London Daylight
 837         # Savings Time is GMT+1.)  The current implementation of
 838         # time_format.iso_utc_time_to_localseconds() breaks if the timezone is
 839         # Europe/London.  (As soon as this unit test is done then I'll change
 840         # that implementation to something that works even in this case...)
 841         origtz = os.environ.get('TZ')
 842         os.environ['TZ'] = "Europe/London"
 843         if hasattr(time, 'tzset'):
 844             time.tzset()
 845         try:
 846             return self._help_test_epoch()
 847         finally:
 848             if origtz is None:
 849                 del os.environ['TZ']
 850             else:
 851                 os.environ['TZ'] = origtz
 852             if hasattr(time, 'tzset'):
 853                 time.tzset()
 854
 855     def _help_test_epoch(self):
 856         origtzname = time.tzname
 857         s = time_format.iso_utc_time_to_seconds("1970-01-01T00:00:01")
 858         self.failUnlessEqual(s, 1.0)
 859         s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01")
 860         self.failUnlessEqual(s, 1.0)
 861         s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01")
 862         self.failUnlessEqual(s, 1.0)
 863
 864         self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01")
 865         self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "),
 866                              "1970-01-01 00:00:01")
 867
 868         now = time.time()
 869         isostr = time_format.iso_utc(now)
 870         timestamp = time_format.iso_utc_time_to_seconds(isostr)
 871         self.failUnlessEqual(int(timestamp), int(now))
 872
 873         def my_time():
 874             return 1.0
 875         self.failUnlessEqual(time_format.iso_utc(t=my_time),
 876                              "1970-01-01_00:00:01")
 877         e = self.failUnlessRaises(ValueError,
 878                                   time_format.iso_utc_time_to_seconds,
 879                                   "invalid timestring")
 880         self.failUnless("not a complete ISO8601 timestamp" in str(e))
 881         s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01.500")
 882         self.failUnlessEqual(s, 1.5)
 883
 884         # Look for daylight-savings-related errors.
 885         thatmomentinmarch = time_format.iso_utc_time_to_seconds("2009-03-20 21:49:02.226536")
 886         self.failUnlessEqual(thatmomentinmarch, 1237585742.226536)
 887         self.failUnlessEqual(origtzname, time.tzname)
 888
 889     def test_iso_utc(self):
 890         when = 1266760143.7841301
 891         out = time_format.iso_utc_date(when)
 892         self.failUnlessEqual(out, "2010-02-21")
 893         out = time_format.iso_utc_date(t=lambda: when)
 894         self.failUnlessEqual(out, "2010-02-21")
 895         out = time_format.iso_utc(when)
 896         self.failUnlessEqual(out, "2010-02-21_13:49:03.784130")
 897         out = time_format.iso_utc(when, sep="-")
 898         self.failUnlessEqual(out, "2010-02-21-13:49:03.784130")
 899
 900     def test_parse_duration(self):
 901         p = time_format.parse_duration
 902         DAY = 24*60*60
 903         self.failUnlessEqual(p("1 day"), DAY)
 904         self.failUnlessEqual(p("2 days"), 2*DAY)
 905         self.failUnlessEqual(p("3 months"), 3*31*DAY)
 906         self.failUnlessEqual(p("4 mo"), 4*31*DAY)
 907         self.failUnlessEqual(p("5 years"), 5*365*DAY)
 908         e = self.failUnlessRaises(ValueError, p, "123")
 909         self.failUnlessIn("no unit (like day, month, or year) in '123'",
 910                           str(e))
 911
 912     def test_parse_date(self):
 913         self.failUnlessEqual(time_format.parse_date("2010-02-21"), 1266710400)
 914
 915 class CacheDir(unittest.TestCase):
 916     def test_basic(self):
 917         basedir = "test_util/CacheDir/test_basic"
 918
 919         def _failIfExists(name):
 920             absfn = os.path.join(basedir, name)
 921             self.failIf(os.path.exists(absfn),
 922                         "%s exists but it shouldn't" % absfn)
 923
 924         def _failUnlessExists(name):
 925             absfn = os.path.join(basedir, name)
 926             self.failUnless(os.path.exists(absfn),
 927                             "%s doesn't exist but it should" % absfn)
 928
 929         cdm = cachedir.CacheDirectoryManager(basedir)
 930         a = cdm.get_file("a")
 931         b = cdm.get_file("b")
 932         c = cdm.get_file("c")
 933         f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f
 934         f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f
 935         f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f
 936
 937         _failUnlessExists("a")
 938         _failUnlessExists("b")
 939         _failUnlessExists("c")
 940
 941         cdm.check()
 942
 943         _failUnlessExists("a")
 944         _failUnlessExists("b")
 945         _failUnlessExists("c")
 946
 947         del a
 948         # this file won't be deleted yet, because it isn't old enough
 949         cdm.check()
 950         _failUnlessExists("a")
 951         _failUnlessExists("b")
 952         _failUnlessExists("c")
 953
 954         # we change the definition of "old" to make everything old
 955         cdm.old = -10
 956
 957         cdm.check()
 958         _failIfExists("a")
 959         _failUnlessExists("b")
 960         _failUnlessExists("c")
 961
 962         cdm.old = 60*60
 963
 964         del b
 965
 966         cdm.check()
 967         _failIfExists("a")
 968         _failUnlessExists("b")
 969         _failUnlessExists("c")
 970
 971         b2 = cdm.get_file("b")
 972
 973         cdm.check()
 974         _failIfExists("a")
 975         _failUnlessExists("b")
 976         _failUnlessExists("c")
 977         del b2
 978
 979 ctr = [0]
 980 class EqButNotIs:
 981     def __init__(self, x):
 982         self.x = x
 983         self.hash = ctr[0]
 984         ctr[0] += 1
 985     def __repr__(self):
 986         return "<%s %s>" % (self.__class__.__name__, self.x,)
 987     def __hash__(self):
 988         return self.hash
 989     def __le__(self, other):
 990         return self.x <= other
 991     def __lt__(self, other):
 992         return self.x < other
 993     def __ge__(self, other):
 994         return self.x >= other
 995     def __gt__(self, other):
 996         return self.x > other
 997     def __ne__(self, other):
 998         return self.x != other
 999     def __eq__(self, other):
1000         return self.x == other
1001
1002 class DictUtil(unittest.TestCase):
1003     def _help_test_empty_dict(self, klass):
1004         d1 = klass()
1005         d2 = klass({})
1006
1007         self.failUnless(d1 == d2, "d1: %r, d2: %r" % (d1, d2,))
1008         self.failUnless(len(d1) == 0)
1009         self.failUnless(len(d2) == 0)
1010
1011     def _help_test_nonempty_dict(self, klass):
1012         d1 = klass({'a': 1, 'b': "eggs", 3: "spam",})
1013         d2 = klass({'a': 1, 'b': "eggs", 3: "spam",})
1014
1015         self.failUnless(d1 == d2)
1016         self.failUnless(len(d1) == 3, "%s, %s" % (len(d1), d1,))
1017         self.failUnless(len(d2) == 3)
1018
1019     def _help_test_eq_but_notis(self, klass):
1020         d = klass({'a': 3, 'b': EqButNotIs(3), 'c': 3})
1021         d.pop('b')
1022
1023         d.clear()
1024         d['a'] = 3
1025         d['b'] = EqButNotIs(3)
1026         d['c'] = 3
1027         d.pop('b')
1028
1029         d.clear()
1030         d['b'] = EqButNotIs(3)
1031         d['a'] = 3
1032         d['c'] = 3
1033         d.pop('b')
1034
1035         d.clear()
1036         d['a'] = EqButNotIs(3)
1037         d['c'] = 3
1038         d['a'] = 3
1039
1040         d.clear()
1041         fake3 = EqButNotIs(3)
1042         fake7 = EqButNotIs(7)
1043         d[fake3] = fake7
1044         d[3] = 7
1045         d[3] = 8
1046         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
1047         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
1048         # The real 7 should have been ejected by the d[3] = 8.
1049         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
1050         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
1051         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
1052         d[fake3] = 8
1053
1054         d.clear()
1055         d[3] = 7
1056         fake3 = EqButNotIs(3)
1057         fake7 = EqButNotIs(7)
1058         d[fake3] = fake7
1059         d[3] = 8
1060         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
1061         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
1062         # The real 7 should have been ejected by the d[3] = 8.
1063         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
1064         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
1065         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
1066         d[fake3] = 8
1067
1068     def test_all(self):
1069         self._help_test_eq_but_notis(dictutil.UtilDict)
1070         self._help_test_eq_but_notis(dictutil.NumDict)
1071         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
1072         self._help_test_nonempty_dict(dictutil.UtilDict)
1073         self._help_test_nonempty_dict(dictutil.NumDict)
1074         self._help_test_nonempty_dict(dictutil.ValueOrderedDict)
1075         self._help_test_eq_but_notis(dictutil.UtilDict)
1076         self._help_test_eq_but_notis(dictutil.NumDict)
1077         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
1078
1079     def test_dict_of_sets(self):
1080         ds = dictutil.DictOfSets()
1081         ds.add(1, "a")
1082         ds.add(2, "b")
1083         ds.add(2, "b")
1084         ds.add(2, "c")
1085         self.failUnlessEqual(ds[1], set(["a"]))
1086         self.failUnlessEqual(ds[2], set(["b", "c"]))
1087         ds.discard(3, "d") # should not raise an exception
1088         ds.discard(2, "b")
1089         self.failUnlessEqual(ds[2], set(["c"]))
1090         ds.discard(2, "c")
1091         self.failIf(2 in ds)
1092
1093         ds.union(1, ["a", "e"])
1094         ds.union(3, ["f"])
1095         self.failUnlessEqual(ds[1], set(["a","e"]))
1096         self.failUnlessEqual(ds[3], set(["f"]))
1097         ds2 = dictutil.DictOfSets()
1098         ds2.add(3, "f")
1099         ds2.add(3, "g")
1100         ds2.add(4, "h")
1101         ds.update(ds2)
1102         self.failUnlessEqual(ds[1], set(["a","e"]))
1103         self.failUnlessEqual(ds[3], set(["f", "g"]))
1104         self.failUnlessEqual(ds[4], set(["h"]))
1105
1106     def test_move(self):
1107         d1 = {1: "a", 2: "b"}
1108         d2 = {2: "c", 3: "d"}
1109         dictutil.move(1, d1, d2)
1110         self.failUnlessEqual(d1, {2: "b"})
1111         self.failUnlessEqual(d2, {1: "a", 2: "c", 3: "d"})
1112
1113         d1 = {1: "a", 2: "b"}
1114         d2 = {2: "c", 3: "d"}
1115         dictutil.move(2, d1, d2)
1116         self.failUnlessEqual(d1, {1: "a"})
1117         self.failUnlessEqual(d2, {2: "b", 3: "d"})
1118
1119         d1 = {1: "a", 2: "b"}
1120         d2 = {2: "c", 3: "d"}
1121         self.failUnlessRaises(KeyError, dictutil.move, 5, d1, d2, strict=True)
1122
1123     def test_subtract(self):
1124         d1 = {1: "a", 2: "b"}
1125         d2 = {2: "c", 3: "d"}
1126         d3 = dictutil.subtract(d1, d2)
1127         self.failUnlessEqual(d3, {1: "a"})
1128
1129         d1 = {1: "a", 2: "b"}
1130         d2 = {2: "c"}
1131         d3 = dictutil.subtract(d1, d2)
1132         self.failUnlessEqual(d3, {1: "a"})
1133
1134     def test_utildict(self):
1135         d = dictutil.UtilDict({1: "a", 2: "b"})
1136         d.del_if_present(1)
1137         d.del_if_present(3)
1138         self.failUnlessEqual(d, {2: "b"})
1139         def eq(a, b):
1140             return a == b
1141         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1142
1143         d = dictutil.UtilDict({1: "b", 2: "a"})
1144         self.failUnlessEqual(d.items_sorted_by_value(),
1145                              [(2, "a"), (1, "b")])
1146         self.failUnlessEqual(d.items_sorted_by_key(),
1147                              [(1, "b"), (2, "a")])
1148         self.failUnlessEqual(repr(d), "{1: 'b', 2: 'a'}")
1149         self.failUnless(1 in d)
1150
1151         d2 = dictutil.UtilDict({3: "c", 4: "d"})
1152         self.failUnless(d != d2)
1153         self.failUnless(d2 > d)
1154         self.failUnless(d2 >= d)
1155         self.failUnless(d <= d2)
1156         self.failUnless(d < d2)
1157         self.failUnlessEqual(d[1], "b")
1158         self.failUnlessEqual(sorted(list([k for k in d])), [1,2])
1159
1160         d3 = d.copy()
1161         self.failUnlessEqual(d, d3)
1162         self.failUnless(isinstance(d3, dictutil.UtilDict))
1163
1164         d4 = d.fromkeys([3,4], "e")
1165         self.failUnlessEqual(d4, {3: "e", 4: "e"})
1166
1167         self.failUnlessEqual(d.get(1), "b")
1168         self.failUnlessEqual(d.get(3), None)
1169         self.failUnlessEqual(d.get(3, "default"), "default")
1170         self.failUnlessEqual(sorted(list(d.items())),
1171                              [(1, "b"), (2, "a")])
1172         self.failUnlessEqual(sorted(list(d.iteritems())),
1173                              [(1, "b"), (2, "a")])
1174         self.failUnlessEqual(sorted(d.keys()), [1, 2])
1175         self.failUnlessEqual(sorted(d.values()), ["a", "b"])
1176         x = d.setdefault(1, "new")
1177         self.failUnlessEqual(x, "b")
1178         self.failUnlessEqual(d[1], "b")
1179         x = d.setdefault(3, "new")
1180         self.failUnlessEqual(x, "new")
1181         self.failUnlessEqual(d[3], "new")
1182         del d[3]
1183
1184         x = d.popitem()
1185         self.failUnless(x in [(1, "b"), (2, "a")])
1186         x = d.popitem()
1187         self.failUnless(x in [(1, "b"), (2, "a")])
1188         self.failUnlessRaises(KeyError, d.popitem)
1189
1190     def test_numdict(self):
1191         d = dictutil.NumDict({"a": 1, "b": 2})
1192
1193         d.add_num("a", 10, 5)
1194         d.add_num("c", 20, 5)
1195         d.add_num("d", 30)
1196         self.failUnlessEqual(d, {"a": 11, "b": 2, "c": 25, "d": 30})
1197
1198         d.subtract_num("a", 10)
1199         d.subtract_num("e", 10)
1200         d.subtract_num("f", 10, 15)
1201         self.failUnlessEqual(d, {"a": 1, "b": 2, "c": 25, "d": 30,
1202                                  "e": -10, "f": 5})
1203
1204         self.failUnlessEqual(d.sum(), sum([1, 2, 25, 30, -10, 5]))
1205
1206         d = dictutil.NumDict()
1207         d.inc("a")
1208         d.inc("a")
1209         d.inc("b", 5)
1210         self.failUnlessEqual(d, {"a": 2, "b": 6})
1211         d.dec("a")
1212         d.dec("c")
1213         d.dec("d", 5)
1214         self.failUnlessEqual(d, {"a": 1, "b": 6, "c": -1, "d": 4})
1215         self.failUnlessEqual(d.items_sorted_by_key(),
1216                              [("a", 1), ("b", 6), ("c", -1), ("d", 4)])
1217         self.failUnlessEqual(d.items_sorted_by_value(),
1218                              [("c", -1), ("a", 1), ("d", 4), ("b", 6)])
1219         self.failUnlessEqual(d.item_with_largest_value(), ("b", 6))
1220
1221         d = dictutil.NumDict({"a": 1, "b": 2})
1222         self.failUnlessEqual(repr(d), "{'a': 1, 'b': 2}")
1223         self.failUnless("a" in d)
1224
1225         d2 = dictutil.NumDict({"c": 3, "d": 4})
1226         self.failUnless(d != d2)
1227         self.failUnless(d2 > d)
1228         self.failUnless(d2 >= d)
1229         self.failUnless(d <= d2)
1230         self.failUnless(d < d2)
1231         self.failUnlessEqual(d["a"], 1)
1232         self.failUnlessEqual(sorted(list([k for k in d])), ["a","b"])
1233         def eq(a, b):
1234             return a == b
1235         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1236
1237         d3 = d.copy()
1238         self.failUnlessEqual(d, d3)
1239         self.failUnless(isinstance(d3, dictutil.NumDict))
1240
1241         d4 = d.fromkeys(["a","b"], 5)
1242         self.failUnlessEqual(d4, {"a": 5, "b": 5})
1243
1244         self.failUnlessEqual(d.get("a"), 1)
1245         self.failUnlessEqual(d.get("c"), 0)
1246         self.failUnlessEqual(d.get("c", 5), 5)
1247         self.failUnlessEqual(sorted(list(d.items())),
1248                              [("a", 1), ("b", 2)])
1249         self.failUnlessEqual(sorted(list(d.iteritems())),
1250                              [("a", 1), ("b", 2)])
1251         self.failUnlessEqual(sorted(d.keys()), ["a", "b"])
1252         self.failUnlessEqual(sorted(d.values()), [1, 2])
1253         self.failUnless(d.has_key("a"))
1254         self.failIf(d.has_key("c"))
1255
1256         x = d.setdefault("c", 3)
1257         self.failUnlessEqual(x, 3)
1258         self.failUnlessEqual(d["c"], 3)
1259         x = d.setdefault("c", 5)
1260         self.failUnlessEqual(x, 3)
1261         self.failUnlessEqual(d["c"], 3)
1262         del d["c"]
1263
1264         x = d.popitem()
1265         self.failUnless(x in [("a", 1), ("b", 2)])
1266         x = d.popitem()
1267         self.failUnless(x in [("a", 1), ("b", 2)])
1268         self.failUnlessRaises(KeyError, d.popitem)
1269
1270         d.update({"c": 3})
1271         d.update({"c": 4, "d": 5})
1272         self.failUnlessEqual(d, {"c": 4, "d": 5})
1273
1274     def test_del_if_present(self):
1275         d = {1: "a", 2: "b"}
1276         dictutil.del_if_present(d, 1)
1277         dictutil.del_if_present(d, 3)
1278         self.failUnlessEqual(d, {2: "b"})
1279
1280     def test_valueordereddict(self):
1281         d = dictutil.ValueOrderedDict()
1282         d["a"] = 3
1283         d["b"] = 2
1284         d["c"] = 1
1285
1286         self.failUnlessEqual(d, {"a": 3, "b": 2, "c": 1})
1287         self.failUnlessEqual(d.items(), [("c", 1), ("b", 2), ("a", 3)])
1288         self.failUnlessEqual(d.values(), [1, 2, 3])
1289         self.failUnlessEqual(d.keys(), ["c", "b", "a"])
1290         self.failUnlessEqual(repr(d), "<ValueOrderedDict {c: 1, b: 2, a: 3}>")
1291         def eq(a, b):
1292             return a == b
1293         self.failIf(d == {"a": 4})
1294         self.failUnless(d != {"a": 4})
1295
1296         x = d.setdefault("d", 0)
1297         self.failUnlessEqual(x, 0)
1298         self.failUnlessEqual(d["d"], 0)
1299         x = d.setdefault("d", -1)
1300         self.failUnlessEqual(x, 0)
1301         self.failUnlessEqual(d["d"], 0)
1302
1303         x = d.remove("e", "default", False)
1304         self.failUnlessEqual(x, "default")
1305         self.failUnlessRaises(KeyError, d.remove, "e", "default", True)
1306         x = d.remove("d", 5)
1307         self.failUnlessEqual(x, 0)
1308
1309         x = d.__getitem__("c")
1310         self.failUnlessEqual(x, 1)
1311         x = d.__getitem__("e", "default", False)
1312         self.failUnlessEqual(x, "default")
1313         self.failUnlessRaises(KeyError, d.__getitem__, "e", "default", True)
1314
1315         self.failUnlessEqual(d.popitem(), ("c", 1))
1316         self.failUnlessEqual(d.popitem(), ("b", 2))
1317         self.failUnlessEqual(d.popitem(), ("a", 3))
1318         self.failUnlessRaises(KeyError, d.popitem)
1319
1320         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1321         x = d.pop("d", "default", False)
1322         self.failUnlessEqual(x, "default")
1323         self.failUnlessRaises(KeyError, d.pop, "d", "default", True)
1324         x = d.pop("b")
1325         self.failUnlessEqual(x, 2)
1326         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1327
1328         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1329         x = d.pop_from_list(1) # pop the second item, b/2
1330         self.failUnlessEqual(x, "b")
1331         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1332
1333     def test_auxdict(self):
1334         d = dictutil.AuxValueDict()
1335         # we put the serialized form in the auxdata
1336         d.set_with_aux("key", ("filecap", "metadata"), "serialized")
1337
1338         self.failUnlessEqual(d.keys(), ["key"])
1339         self.failUnlessEqual(d["key"], ("filecap", "metadata"))
1340         self.failUnlessEqual(d.get_aux("key"), "serialized")
1341         def _get_missing(key):
1342             return d[key]
1343         self.failUnlessRaises(KeyError, _get_missing, "nonkey")
1344         self.failUnlessEqual(d.get("nonkey"), None)
1345         self.failUnlessEqual(d.get("nonkey", "nonvalue"), "nonvalue")
1346         self.failUnlessEqual(d.get_aux("nonkey"), None)
1347         self.failUnlessEqual(d.get_aux("nonkey", "nonvalue"), "nonvalue")
1348
1349         d["key"] = ("filecap2", "metadata2")
1350         self.failUnlessEqual(d["key"], ("filecap2", "metadata2"))
1351         self.failUnlessEqual(d.get_aux("key"), None)
1352
1353         d.set_with_aux("key2", "value2", "aux2")
1354         self.failUnlessEqual(sorted(d.keys()), ["key", "key2"])
1355         del d["key2"]
1356         self.failUnlessEqual(d.keys(), ["key"])
1357         self.failIf("key2" in d)
1358         self.failUnlessRaises(KeyError, _get_missing, "key2")
1359         self.failUnlessEqual(d.get("key2"), None)
1360         self.failUnlessEqual(d.get_aux("key2"), None)
1361         d["key2"] = "newvalue2"
1362         self.failUnlessEqual(d.get("key2"), "newvalue2")
1363         self.failUnlessEqual(d.get_aux("key2"), None)
1364
1365         d = dictutil.AuxValueDict({1:2,3:4})
1366         self.failUnlessEqual(sorted(d.keys()), [1,3])
1367         self.failUnlessEqual(d[1], 2)
1368         self.failUnlessEqual(d.get_aux(1), None)
1369
1370         d = dictutil.AuxValueDict([ (1,2), (3,4) ])
1371         self.failUnlessEqual(sorted(d.keys()), [1,3])
1372         self.failUnlessEqual(d[1], 2)
1373         self.failUnlessEqual(d.get_aux(1), None)
1374
1375         d = dictutil.AuxValueDict(one=1, two=2)
1376         self.failUnlessEqual(sorted(d.keys()), ["one","two"])
1377         self.failUnlessEqual(d["one"], 1)
1378         self.failUnlessEqual(d.get_aux("one"), None)
1379
1380 class Pipeline(unittest.TestCase):
1381     def pause(self, *args, **kwargs):
1382         d = defer.Deferred()
1383         self.calls.append( (d, args, kwargs) )
1384         return d
1385
1386     def failUnlessCallsAre(self, expected):
1387         #print self.calls
1388         #print expected
1389         self.failUnlessEqual(len(self.calls), len(expected), self.calls)
1390         for i,c in enumerate(self.calls):
1391             self.failUnlessEqual(c[1:], expected[i], str(i))
1392
1393     def test_basic(self):
1394         self.calls = []
1395         finished = []
1396         p = pipeline.Pipeline(100)
1397
1398         d = p.flush() # fires immediately
1399         d.addCallbacks(finished.append, log.err)
1400         self.failUnlessEqual(len(finished), 1)
1401         finished = []
1402
1403         d = p.add(10, self.pause, "one")
1404         # the call should start right away, and our return Deferred should
1405         # fire right away
1406         d.addCallbacks(finished.append, log.err)
1407         self.failUnlessEqual(len(finished), 1)
1408         self.failUnlessEqual(finished[0], None)
1409         self.failUnlessCallsAre([ ( ("one",) , {} ) ])
1410         self.failUnlessEqual(p.gauge, 10)
1411
1412         # pipeline: [one]
1413
1414         finished = []
1415         d = p.add(20, self.pause, "two", kw=2)
1416         # pipeline: [one, two]
1417
1418         # the call and the Deferred should fire right away
1419         d.addCallbacks(finished.append, log.err)
1420         self.failUnlessEqual(len(finished), 1)
1421         self.failUnlessEqual(finished[0], None)
1422         self.failUnlessCallsAre([ ( ("one",) , {} ),
1423                                   ( ("two",) , {"kw": 2} ),
1424                                   ])
1425         self.failUnlessEqual(p.gauge, 30)
1426
1427         self.calls[0][0].callback("one-result")
1428         # pipeline: [two]
1429         self.failUnlessEqual(p.gauge, 20)
1430
1431         finished = []
1432         d = p.add(90, self.pause, "three", "posarg1")
1433         # pipeline: [two, three]
1434         flushed = []
1435         fd = p.flush()
1436         fd.addCallbacks(flushed.append, log.err)
1437         self.failUnlessEqual(flushed, [])
1438
1439         # the call will be made right away, but the return Deferred will not,
1440         # because the pipeline is now full.
1441         d.addCallbacks(finished.append, log.err)
1442         self.failUnlessEqual(len(finished), 0)
1443         self.failUnlessCallsAre([ ( ("one",) , {} ),
1444                                   ( ("two",) , {"kw": 2} ),
1445                                   ( ("three", "posarg1"), {} ),
1446                                   ])
1447         self.failUnlessEqual(p.gauge, 110)
1448
1449         self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause)
1450
1451         # retiring either call will unblock the pipeline, causing the #3
1452         # Deferred to fire
1453         self.calls[2][0].callback("three-result")
1454         # pipeline: [two]
1455
1456         self.failUnlessEqual(len(finished), 1)
1457         self.failUnlessEqual(finished[0], None)
1458         self.failUnlessEqual(flushed, [])
1459
1460         # retiring call#2 will finally allow the flush() Deferred to fire
1461         self.calls[1][0].callback("two-result")
1462         self.failUnlessEqual(len(flushed), 1)
1463
1464     def test_errors(self):
1465         self.calls = []
1466         p = pipeline.Pipeline(100)
1467
1468         d1 = p.add(200, self.pause, "one")
1469         d2 = p.flush()
1470
1471         finished = []
1472         d1.addBoth(finished.append)
1473         self.failUnlessEqual(finished, [])
1474
1475         flushed = []
1476         d2.addBoth(flushed.append)
1477         self.failUnlessEqual(flushed, [])
1478
1479         self.calls[0][0].errback(ValueError("oops"))
1480
1481         self.failUnlessEqual(len(finished), 1)
1482         f = finished[0]
1483         self.failUnless(isinstance(f, Failure))
1484         self.failUnless(f.check(pipeline.PipelineError))
1485         self.failUnlessIn("PipelineError", str(f.value))
1486         self.failUnlessIn("ValueError", str(f.value))
1487         r = repr(f.value)
1488         self.failUnless("ValueError" in r, r)
1489         f2 = f.value.error
1490         self.failUnless(f2.check(ValueError))
1491
1492         self.failUnlessEqual(len(flushed), 1)
1493         f = flushed[0]
1494         self.failUnless(isinstance(f, Failure))
1495         self.failUnless(f.check(pipeline.PipelineError))
1496         f2 = f.value.error
1497         self.failUnless(f2.check(ValueError))
1498
1499         # now that the pipeline is in the failed state, any new calls will
1500         # fail immediately
1501
1502         d3 = p.add(20, self.pause, "two")
1503
1504         finished = []
1505         d3.addBoth(finished.append)
1506         self.failUnlessEqual(len(finished), 1)
1507         f = finished[0]
1508         self.failUnless(isinstance(f, Failure))
1509         self.failUnless(f.check(pipeline.PipelineError))
1510         r = repr(f.value)
1511         self.failUnless("ValueError" in r, r)
1512         f2 = f.value.error
1513         self.failUnless(f2.check(ValueError))
1514
1515         d4 = p.flush()
1516         flushed = []
1517         d4.addBoth(flushed.append)
1518         self.failUnlessEqual(len(flushed), 1)
1519         f = flushed[0]
1520         self.failUnless(isinstance(f, Failure))
1521         self.failUnless(f.check(pipeline.PipelineError))
1522         f2 = f.value.error
1523         self.failUnless(f2.check(ValueError))
1524
1525     def test_errors2(self):
1526         self.calls = []
1527         p = pipeline.Pipeline(100)
1528
1529         d1 = p.add(10, self.pause, "one")
1530         d2 = p.add(20, self.pause, "two")
1531         d3 = p.add(30, self.pause, "three")
1532         d4 = p.flush()
1533
1534         # one call fails, then the second one succeeds: make sure
1535         # ExpandableDeferredList tolerates the second one
1536
1537         flushed = []
1538         d4.addBoth(flushed.append)
1539         self.failUnlessEqual(flushed, [])
1540
1541         self.calls[0][0].errback(ValueError("oops"))
1542         self.failUnlessEqual(len(flushed), 1)
1543         f = flushed[0]
1544         self.failUnless(isinstance(f, Failure))
1545         self.failUnless(f.check(pipeline.PipelineError))
1546         f2 = f.value.error
1547         self.failUnless(f2.check(ValueError))
1548
1549         self.calls[1][0].callback("two-result")
1550         self.calls[2][0].errback(ValueError("three-error"))
1551
1552         del d1,d2,d3,d4
1553
1554 class SampleError(Exception):
1555     pass
1556
1557 class Log(unittest.TestCase):
1558     def test_err(self):
1559         if not hasattr(self, "flushLoggedErrors"):
1560             # without flushLoggedErrors, we can't get rid of the
1561             # twisted.log.err that tahoe_log records, so we can't keep this
1562             # test from [ERROR]ing
1563             raise unittest.SkipTest("needs flushLoggedErrors from Twisted-2.5.0")
1564         try:
1565             raise SampleError("simple sample")
1566         except:
1567             f = Failure()
1568         tahoe_log.err(format="intentional sample error",
1569                       failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ")
1570         self.flushLoggedErrors(SampleError)