src/allmydata/test/test_util.py

   1
   2 def foo(): pass # keep the line number constant
   3
   4 import os, time
   5 from StringIO import StringIO
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.python.failure import Failure
   9 from twisted.python import log
  10
  11 from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
  12 from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
  13 from allmydata.util import limiter, time_format, pollmixin, cachedir
  14 from allmydata.util import statistics, dictutil, rrefutil, pipeline
  15 from allmydata.util.rrefutil import ServerFailure
  16
  17 class Base32(unittest.TestCase):
  18     def test_b2a_matches_Pythons(self):
  19         import base64
  20         y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0"
  21         x = base64.b32encode(y)
  22         while x and x[-1] == '=':
  23             x = x[:-1]
  24         x = x.lower()
  25         self.failUnlessEqual(base32.b2a(y), x)
  26     def test_b2a(self):
  27         self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a")
  28     def test_b2a_or_none(self):
  29         self.failUnlessEqual(base32.b2a_or_none(None), None)
  30         self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a")
  31     def test_a2b(self):
  32         self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34")
  33         self.failUnlessRaises(AssertionError, base32.a2b, "b0gus")
  34
  35 class IDLib(unittest.TestCase):
  36     def test_nodeid_b2a(self):
  37         self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
  38
  39 class NoArgumentException(Exception):
  40     def __init__(self):
  41         pass
  42
  43 class HumanReadable(unittest.TestCase):
  44     def test_repr(self):
  45         hr = humanreadable.hr
  46         self.failUnlessEqual(hr(foo), "<foo() at test_util.py:2>")
  47         self.failUnlessEqual(hr(self.test_repr),
  48                              "<bound method HumanReadable.test_repr of <allmydata.test.test_util.HumanReadable testMethod=test_repr>>")
  49         self.failUnlessEqual(hr(1L), "1")
  50         self.failUnlessEqual(hr(10**40),
  51                              "100000000000000000...000000000000000000")
  52         self.failUnlessEqual(hr(self), "<allmydata.test.test_util.HumanReadable testMethod=test_repr>")
  53         self.failUnlessEqual(hr([1,2]), "[1, 2]")
  54         self.failUnlessEqual(hr({1:2}), "{1:2}")
  55         try:
  56             raise ValueError
  57         except Exception, e:
  58             self.failUnless(
  59                 hr(e) == "<ValueError: ()>" # python-2.4
  60                 or hr(e) == "ValueError()") # python-2.5
  61         try:
  62             raise ValueError("oops")
  63         except Exception, e:
  64             self.failUnless(
  65                 hr(e) == "<ValueError: 'oops'>" # python-2.4
  66                 or hr(e) == "ValueError('oops',)") # python-2.5
  67         try:
  68             raise NoArgumentException
  69         except Exception, e:
  70             self.failUnless(
  71                 hr(e) == "<NoArgumentException>" # python-2.4
  72                 or hr(e) == "NoArgumentException()") # python-2.5
  73
  74
  75 class MyList(list):
  76     pass
  77
  78 class Math(unittest.TestCase):
  79     def test_div_ceil(self):
  80         f = mathutil.div_ceil
  81         self.failUnlessEqual(f(0, 1), 0)
  82         self.failUnlessEqual(f(0, 2), 0)
  83         self.failUnlessEqual(f(0, 3), 0)
  84         self.failUnlessEqual(f(1, 3), 1)
  85         self.failUnlessEqual(f(2, 3), 1)
  86         self.failUnlessEqual(f(3, 3), 1)
  87         self.failUnlessEqual(f(4, 3), 2)
  88         self.failUnlessEqual(f(5, 3), 2)
  89         self.failUnlessEqual(f(6, 3), 2)
  90         self.failUnlessEqual(f(7, 3), 3)
  91
  92     def test_next_multiple(self):
  93         f = mathutil.next_multiple
  94         self.failUnlessEqual(f(5, 1), 5)
  95         self.failUnlessEqual(f(5, 2), 6)
  96         self.failUnlessEqual(f(5, 3), 6)
  97         self.failUnlessEqual(f(5, 4), 8)
  98         self.failUnlessEqual(f(5, 5), 5)
  99         self.failUnlessEqual(f(5, 6), 6)
 100         self.failUnlessEqual(f(32, 1), 32)
 101         self.failUnlessEqual(f(32, 2), 32)
 102         self.failUnlessEqual(f(32, 3), 33)
 103         self.failUnlessEqual(f(32, 4), 32)
 104         self.failUnlessEqual(f(32, 5), 35)
 105         self.failUnlessEqual(f(32, 6), 36)
 106         self.failUnlessEqual(f(32, 7), 35)
 107         self.failUnlessEqual(f(32, 8), 32)
 108         self.failUnlessEqual(f(32, 9), 36)
 109         self.failUnlessEqual(f(32, 10), 40)
 110         self.failUnlessEqual(f(32, 11), 33)
 111         self.failUnlessEqual(f(32, 12), 36)
 112         self.failUnlessEqual(f(32, 13), 39)
 113         self.failUnlessEqual(f(32, 14), 42)
 114         self.failUnlessEqual(f(32, 15), 45)
 115         self.failUnlessEqual(f(32, 16), 32)
 116         self.failUnlessEqual(f(32, 17), 34)
 117         self.failUnlessEqual(f(32, 18), 36)
 118         self.failUnlessEqual(f(32, 589), 589)
 119
 120     def test_pad_size(self):
 121         f = mathutil.pad_size
 122         self.failUnlessEqual(f(0, 4), 0)
 123         self.failUnlessEqual(f(1, 4), 3)
 124         self.failUnlessEqual(f(2, 4), 2)
 125         self.failUnlessEqual(f(3, 4), 1)
 126         self.failUnlessEqual(f(4, 4), 0)
 127         self.failUnlessEqual(f(5, 4), 3)
 128
 129     def test_is_power_of_k(self):
 130         f = mathutil.is_power_of_k
 131         for i in range(1, 100):
 132             if i in (1, 2, 4, 8, 16, 32, 64):
 133                 self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
 134             else:
 135                 self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
 136         for i in range(1, 100):
 137             if i in (1, 3, 9, 27, 81):
 138                 self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
 139             else:
 140                 self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
 141
 142     def test_next_power_of_k(self):
 143         f = mathutil.next_power_of_k
 144         self.failUnlessEqual(f(0,2), 1)
 145         self.failUnlessEqual(f(1,2), 1)
 146         self.failUnlessEqual(f(2,2), 2)
 147         self.failUnlessEqual(f(3,2), 4)
 148         self.failUnlessEqual(f(4,2), 4)
 149         for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
 150         for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
 151         for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
 152         for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
 153         for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
 154
 155         self.failUnlessEqual(f(0,3), 1)
 156         self.failUnlessEqual(f(1,3), 1)
 157         self.failUnlessEqual(f(2,3), 3)
 158         self.failUnlessEqual(f(3,3), 3)
 159         for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
 160         for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
 161         for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
 162         for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
 163
 164     def test_ave(self):
 165         f = mathutil.ave
 166         self.failUnlessEqual(f([1,2,3]), 2)
 167         self.failUnlessEqual(f([0,0,0,4]), 1)
 168         self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
 169
 170     def test_round_sigfigs(self):
 171         f = mathutil.round_sigfigs
 172         self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002)
 173
 174 class Statistics(unittest.TestCase):
 175     def should_assert(self, msg, func, *args, **kwargs):
 176         try:
 177             func(*args, **kwargs)
 178             self.fail(msg)
 179         except AssertionError, e:
 180             pass
 181
 182     def failUnlessListEqual(self, a, b, msg = None):
 183         self.failUnlessEqual(len(a), len(b))
 184         for i in range(len(a)):
 185             self.failUnlessEqual(a[i], b[i], msg)
 186
 187     def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None):
 188         self.failUnlessEqual(len(a), len(b))
 189         for i in range(len(a)):
 190             self.failUnlessAlmostEqual(a[i], b[i], places, msg)
 191
 192     def test_binomial_coeff(self):
 193         f = statistics.binomial_coeff
 194         self.failUnlessEqual(f(20, 0), 1)
 195         self.failUnlessEqual(f(20, 1), 20)
 196         self.failUnlessEqual(f(20, 2), 190)
 197         self.failUnlessEqual(f(20, 8), f(20, 12))
 198         self.should_assert("Should assert if n < k", f, 2, 3)
 199
 200     def test_binomial_distribution_pmf(self):
 201         f = statistics.binomial_distribution_pmf
 202
 203         pmf_comp = f(2, .1)
 204         pmf_stat = [0.81, 0.18, 0.01]
 205         self.failUnlessListAlmostEqual(pmf_comp, pmf_stat)
 206
 207         # Summing across a PMF should give the total probability 1
 208         self.failUnlessAlmostEqual(sum(pmf_comp), 1)
 209         self.should_assert("Should assert if not 0<=p<=1", f, 1, -1)
 210         self.should_assert("Should assert if n < 1", f, 0, .1)
 211
 212         out = StringIO()
 213         statistics.print_pmf(pmf_comp, out=out)
 214         lines = out.getvalue().splitlines()
 215         self.failUnlessEqual(lines[0], "i=0: 0.81")
 216         self.failUnlessEqual(lines[1], "i=1: 0.18")
 217         self.failUnlessEqual(lines[2], "i=2: 0.01")
 218
 219     def test_survival_pmf(self):
 220         f = statistics.survival_pmf
 221         # Cross-check binomial-distribution method against convolution
 222         # method.
 223         p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20
 224         pmf1 = statistics.survival_pmf_via_conv(p_list)
 225         pmf2 = statistics.survival_pmf_via_bd(p_list)
 226         self.failUnlessListAlmostEqual(pmf1, pmf2)
 227         self.failUnlessTrue(statistics.valid_pmf(pmf1))
 228         self.should_assert("Should assert if p_i > 1", f, [1.1]);
 229         self.should_assert("Should assert if p_i < 0", f, [-.1]);
 230
 231     def test_repair_count_pmf(self):
 232         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 233         repair_pmf = statistics.repair_count_pmf(survival_pmf, 3)
 234         # repair_pmf[0] == sum(survival_pmf[0,1,2,5])
 235         # repair_pmf[1] == survival_pmf[4]
 236         # repair_pmf[2] = survival_pmf[3]
 237         self.failUnlessListAlmostEqual(repair_pmf,
 238                                        [0.00001 + 0.00045 + 0.0081 + 0.59049,
 239                                         .32805,
 240                                         .0729,
 241                                         0, 0, 0])
 242
 243     def test_repair_cost(self):
 244         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 245         bwcost = statistics.bandwidth_cost_function
 246         cost = statistics.mean_repair_cost(bwcost, 1000,
 247                                            survival_pmf, 3, ul_dl_ratio=1.0)
 248         self.failUnlessAlmostEqual(cost, 558.90)
 249         cost = statistics.mean_repair_cost(bwcost, 1000,
 250                                            survival_pmf, 3, ul_dl_ratio=8.0)
 251         self.failUnlessAlmostEqual(cost, 1664.55)
 252
 253         # I haven't manually checked the math beyond here -warner
 254         cost = statistics.eternal_repair_cost(bwcost, 1000,
 255                                               survival_pmf, 3,
 256                                               discount_rate=0, ul_dl_ratio=1.0)
 257         self.failUnlessAlmostEqual(cost, 65292.056074766246)
 258         cost = statistics.eternal_repair_cost(bwcost, 1000,
 259                                               survival_pmf, 3,
 260                                               discount_rate=0.05,
 261                                               ul_dl_ratio=1.0)
 262         self.failUnlessAlmostEqual(cost, 9133.6097158191551)
 263
 264     def test_convolve(self):
 265         f = statistics.convolve
 266         v1 = [ 1, 2, 3 ]
 267         v2 = [ 4, 5, 6 ]
 268         v3 = [ 7, 8 ]
 269         v1v2result = [ 4, 13, 28, 27, 18 ]
 270         # Convolution is commutative
 271         r1 = f(v1, v2)
 272         r2 = f(v2, v1)
 273         self.failUnlessListEqual(r1, r2, "Convolution should be commutative")
 274         self.failUnlessListEqual(r1, v1v2result, "Didn't match known result")
 275         # Convolution is associative
 276         r1 = f(f(v1, v2), v3)
 277         r2 = f(v1, f(v2, v3))
 278         self.failUnlessListEqual(r1, r2, "Convolution should be associative")
 279         # Convolution is distributive
 280         r1 = f(v3, [ a + b for a, b in zip(v1, v2) ])
 281         tmp1 = f(v3, v1)
 282         tmp2 = f(v3, v2)
 283         r2 = [ a + b for a, b in zip(tmp1, tmp2) ]
 284         self.failUnlessListEqual(r1, r2, "Convolution should be distributive")
 285         # Convolution is scalar multiplication associative
 286         tmp1 = f(v1, v2)
 287         r1 = [ a * 4 for a in tmp1 ]
 288         tmp2 = [ a * 4 for a in v1 ]
 289         r2 = f(tmp2, v2)
 290         self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative")
 291
 292     def test_find_k(self):
 293         f = statistics.find_k
 294         g = statistics.pr_file_loss
 295         plist = [.9] * 10 + [.8] * 10 # N=20
 296         t = .0001
 297         k = f(plist, t)
 298         self.failUnlessEqual(k, 10)
 299         self.failUnless(g(plist, k) < t)
 300
 301     def test_pr_file_loss(self):
 302         f = statistics.pr_file_loss
 303         plist = [.5] * 10
 304         self.failUnlessEqual(f(plist, 3), .0546875)
 305
 306     def test_pr_backup_file_loss(self):
 307         f = statistics.pr_backup_file_loss
 308         plist = [.5] * 10
 309         self.failUnlessEqual(f(plist, .5, 3), .02734375)
 310
 311
 312 class Asserts(unittest.TestCase):
 313     def should_assert(self, func, *args, **kwargs):
 314         try:
 315             func(*args, **kwargs)
 316         except AssertionError, e:
 317             return str(e)
 318         except Exception, e:
 319             self.fail("assert failed with non-AssertionError: %s" % e)
 320         self.fail("assert was not caught")
 321
 322     def should_not_assert(self, func, *args, **kwargs):
 323         if "re" in kwargs:
 324             regexp = kwargs["re"]
 325             del kwargs["re"]
 326         try:
 327             func(*args, **kwargs)
 328         except AssertionError, e:
 329             self.fail("assertion fired when it should not have: %s" % e)
 330         except Exception, e:
 331             self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e)
 332         return # we're happy
 333
 334
 335     def test_assert(self):
 336         f = assertutil._assert
 337         self.should_assert(f)
 338         self.should_assert(f, False)
 339         self.should_not_assert(f, True)
 340
 341         m = self.should_assert(f, False, "message")
 342         self.failUnlessEqual(m, "'message' <type 'str'>", m)
 343         m = self.should_assert(f, False, "message1", othermsg=12)
 344         self.failUnlessEqual("'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 345         m = self.should_assert(f, False, othermsg="message2")
 346         self.failUnlessEqual("othermsg: 'message2' <type 'str'>", m)
 347
 348     def test_precondition(self):
 349         f = assertutil.precondition
 350         self.should_assert(f)
 351         self.should_assert(f, False)
 352         self.should_not_assert(f, True)
 353
 354         m = self.should_assert(f, False, "message")
 355         self.failUnlessEqual("precondition: 'message' <type 'str'>", m)
 356         m = self.should_assert(f, False, "message1", othermsg=12)
 357         self.failUnlessEqual("precondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 358         m = self.should_assert(f, False, othermsg="message2")
 359         self.failUnlessEqual("precondition: othermsg: 'message2' <type 'str'>", m)
 360
 361     def test_postcondition(self):
 362         f = assertutil.postcondition
 363         self.should_assert(f)
 364         self.should_assert(f, False)
 365         self.should_not_assert(f, True)
 366
 367         m = self.should_assert(f, False, "message")
 368         self.failUnlessEqual("postcondition: 'message' <type 'str'>", m)
 369         m = self.should_assert(f, False, "message1", othermsg=12)
 370         self.failUnlessEqual("postcondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 371         m = self.should_assert(f, False, othermsg="message2")
 372         self.failUnlessEqual("postcondition: othermsg: 'message2' <type 'str'>", m)
 373
 374 class FileUtil(unittest.TestCase):
 375     def mkdir(self, basedir, path, mode=0777):
 376         fn = os.path.join(basedir, path)
 377         fileutil.make_dirs(fn, mode)
 378
 379     def touch(self, basedir, path, mode=None, data="touch\n"):
 380         fn = os.path.join(basedir, path)
 381         f = open(fn, "w")
 382         f.write(data)
 383         f.close()
 384         if mode is not None:
 385             os.chmod(fn, mode)
 386
 387     def test_rm_dir(self):
 388         basedir = "util/FileUtil/test_rm_dir"
 389         fileutil.make_dirs(basedir)
 390         # create it again to test idempotency
 391         fileutil.make_dirs(basedir)
 392         d = os.path.join(basedir, "doomed")
 393         self.mkdir(d, "a/b")
 394         self.touch(d, "a/b/1.txt")
 395         self.touch(d, "a/b/2.txt", 0444)
 396         self.touch(d, "a/b/3.txt", 0)
 397         self.mkdir(d, "a/c")
 398         self.touch(d, "a/c/1.txt")
 399         self.touch(d, "a/c/2.txt", 0444)
 400         self.touch(d, "a/c/3.txt", 0)
 401         os.chmod(os.path.join(d, "a/c"), 0444)
 402         self.mkdir(d, "a/d")
 403         self.touch(d, "a/d/1.txt")
 404         self.touch(d, "a/d/2.txt", 0444)
 405         self.touch(d, "a/d/3.txt", 0)
 406         os.chmod(os.path.join(d, "a/d"), 0)
 407
 408         fileutil.rm_dir(d)
 409         self.failIf(os.path.exists(d))
 410         # remove it again to test idempotency
 411         fileutil.rm_dir(d)
 412
 413     def test_remove_if_possible(self):
 414         basedir = "util/FileUtil/test_remove_if_possible"
 415         fileutil.make_dirs(basedir)
 416         self.touch(basedir, "here")
 417         fn = os.path.join(basedir, "here")
 418         fileutil.remove_if_possible(fn)
 419         self.failIf(os.path.exists(fn))
 420         fileutil.remove_if_possible(fn) # should be idempotent
 421         fileutil.rm_dir(basedir)
 422         fileutil.remove_if_possible(fn) # should survive errors
 423
 424     def test_open_or_create(self):
 425         basedir = "util/FileUtil/test_open_or_create"
 426         fileutil.make_dirs(basedir)
 427         fn = os.path.join(basedir, "here")
 428         f = fileutil.open_or_create(fn)
 429         f.write("stuff.")
 430         f.close()
 431         f = fileutil.open_or_create(fn)
 432         f.seek(0, 2)
 433         f.write("more.")
 434         f.close()
 435         f = open(fn, "r")
 436         data = f.read()
 437         f.close()
 438         self.failUnlessEqual(data, "stuff.more.")
 439
 440     def test_NamedTemporaryDirectory(self):
 441         basedir = "util/FileUtil/test_NamedTemporaryDirectory"
 442         fileutil.make_dirs(basedir)
 443         td = fileutil.NamedTemporaryDirectory(dir=basedir)
 444         name = td.name
 445         self.failUnless(basedir in name)
 446         self.failUnless(basedir in repr(td))
 447         self.failUnless(os.path.isdir(name))
 448         del td
 449         # it is conceivable that we need to force gc here, but I'm not sure
 450         self.failIf(os.path.isdir(name))
 451
 452     def test_rename(self):
 453         basedir = "util/FileUtil/test_rename"
 454         fileutil.make_dirs(basedir)
 455         self.touch(basedir, "here")
 456         fn = os.path.join(basedir, "here")
 457         fn2 = os.path.join(basedir, "there")
 458         fileutil.rename(fn, fn2)
 459         self.failIf(os.path.exists(fn))
 460         self.failUnless(os.path.exists(fn2))
 461
 462     def test_du(self):
 463         basedir = "util/FileUtil/test_du"
 464         fileutil.make_dirs(basedir)
 465         d = os.path.join(basedir, "space-consuming")
 466         self.mkdir(d, "a/b")
 467         self.touch(d, "a/b/1.txt", data="a"*10)
 468         self.touch(d, "a/b/2.txt", data="b"*11)
 469         self.mkdir(d, "a/c")
 470         self.touch(d, "a/c/1.txt", data="c"*12)
 471         self.touch(d, "a/c/2.txt", data="d"*13)
 472
 473         used = fileutil.du(basedir)
 474         self.failUnlessEqual(10+11+12+13, used)
 475
 476 class PollMixinTests(unittest.TestCase):
 477     def setUp(self):
 478         self.pm = pollmixin.PollMixin()
 479
 480     def test_PollMixin_True(self):
 481         d = self.pm.poll(check_f=lambda : True,
 482                          pollinterval=0.1)
 483         return d
 484
 485     def test_PollMixin_False_then_True(self):
 486         i = iter([False, True])
 487         d = self.pm.poll(check_f=i.next,
 488                          pollinterval=0.1)
 489         return d
 490
 491     def test_timeout(self):
 492         d = self.pm.poll(check_f=lambda: False,
 493                          pollinterval=0.01,
 494                          timeout=1)
 495         def _suc(res):
 496             self.fail("poll should have failed, not returned %s" % (res,))
 497         def _err(f):
 498             f.trap(pollmixin.TimeoutError)
 499             return None # success
 500         d.addCallbacks(_suc, _err)
 501         return d
 502
 503 class DeferredUtilTests(unittest.TestCase):
 504     def test_gather_results(self):
 505         d1 = defer.Deferred()
 506         d2 = defer.Deferred()
 507         res = deferredutil.gatherResults([d1, d2])
 508         d1.errback(ValueError("BAD"))
 509         def _callb(res):
 510             self.fail("Should have errbacked, not resulted in %s" % (res,))
 511         def _errb(thef):
 512             thef.trap(ValueError)
 513         res.addCallbacks(_callb, _errb)
 514         return res
 515
 516     def test_success(self):
 517         d1, d2 = defer.Deferred(), defer.Deferred()
 518         good = []
 519         bad = []
 520         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 521         dlss.addCallbacks(good.append, bad.append)
 522         d1.callback(1)
 523         d2.callback(2)
 524         self.failUnlessEqual(good, [[1,2]])
 525         self.failUnlessEqual(bad, [])
 526
 527     def test_failure(self):
 528         d1, d2 = defer.Deferred(), defer.Deferred()
 529         good = []
 530         bad = []
 531         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 532         dlss.addCallbacks(good.append, bad.append)
 533         d1.addErrback(lambda _ignore: None)
 534         d2.addErrback(lambda _ignore: None)
 535         d1.callback(1)
 536         d2.errback(ValueError())
 537         self.failUnlessEqual(good, [])
 538         self.failUnlessEqual(len(bad), 1)
 539         f = bad[0]
 540         self.failUnless(isinstance(f, Failure))
 541         self.failUnless(f.check(ValueError))
 542
 543 class HashUtilTests(unittest.TestCase):
 544
 545     def test_random_key(self):
 546         k = hashutil.random_key()
 547         self.failUnlessEqual(len(k), hashutil.KEYLEN)
 548
 549     def test_sha256d(self):
 550         h1 = hashutil.tagged_hash("tag1", "value")
 551         h2 = hashutil.tagged_hasher("tag1")
 552         h2.update("value")
 553         h2a = h2.digest()
 554         h2b = h2.digest()
 555         self.failUnlessEqual(h1, h2a)
 556         self.failUnlessEqual(h2a, h2b)
 557
 558     def test_sha256d_truncated(self):
 559         h1 = hashutil.tagged_hash("tag1", "value", 16)
 560         h2 = hashutil.tagged_hasher("tag1", 16)
 561         h2.update("value")
 562         h2 = h2.digest()
 563         self.failUnlessEqual(len(h1), 16)
 564         self.failUnlessEqual(len(h2), 16)
 565         self.failUnlessEqual(h1, h2)
 566
 567     def test_chk(self):
 568         h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret")
 569         h2 = hashutil.convergence_hasher(3, 10, 1000, "secret")
 570         h2.update("data")
 571         h2 = h2.digest()
 572         self.failUnlessEqual(h1, h2)
 573
 574     def test_hashers(self):
 575         h1 = hashutil.block_hash("foo")
 576         h2 = hashutil.block_hasher()
 577         h2.update("foo")
 578         self.failUnlessEqual(h1, h2.digest())
 579
 580         h1 = hashutil.uri_extension_hash("foo")
 581         h2 = hashutil.uri_extension_hasher()
 582         h2.update("foo")
 583         self.failUnlessEqual(h1, h2.digest())
 584
 585         h1 = hashutil.plaintext_hash("foo")
 586         h2 = hashutil.plaintext_hasher()
 587         h2.update("foo")
 588         self.failUnlessEqual(h1, h2.digest())
 589
 590         h1 = hashutil.crypttext_hash("foo")
 591         h2 = hashutil.crypttext_hasher()
 592         h2.update("foo")
 593         self.failUnlessEqual(h1, h2.digest())
 594
 595         h1 = hashutil.crypttext_segment_hash("foo")
 596         h2 = hashutil.crypttext_segment_hasher()
 597         h2.update("foo")
 598         self.failUnlessEqual(h1, h2.digest())
 599
 600         h1 = hashutil.plaintext_segment_hash("foo")
 601         h2 = hashutil.plaintext_segment_hasher()
 602         h2.update("foo")
 603         self.failUnlessEqual(h1, h2.digest())
 604
 605     def test_constant_time_compare(self):
 606         self.failUnless(hashutil.constant_time_compare("a", "a"))
 607         self.failUnless(hashutil.constant_time_compare("ab", "ab"))
 608         self.failIf(hashutil.constant_time_compare("a", "b"))
 609         self.failIf(hashutil.constant_time_compare("a", "aa"))
 610
 611 class Abbreviate(unittest.TestCase):
 612     def test_time(self):
 613         a = abbreviate.abbreviate_time
 614         self.failUnlessEqual(a(None), "unknown")
 615         self.failUnlessEqual(a(0), "0 seconds")
 616         self.failUnlessEqual(a(1), "1 second")
 617         self.failUnlessEqual(a(2), "2 seconds")
 618         self.failUnlessEqual(a(119), "119 seconds")
 619         MIN = 60
 620         self.failUnlessEqual(a(2*MIN), "2 minutes")
 621         self.failUnlessEqual(a(60*MIN), "60 minutes")
 622         self.failUnlessEqual(a(179*MIN), "179 minutes")
 623         HOUR = 60*MIN
 624         self.failUnlessEqual(a(180*MIN), "3 hours")
 625         self.failUnlessEqual(a(4*HOUR), "4 hours")
 626         DAY = 24*HOUR
 627         MONTH = 30*DAY
 628         self.failUnlessEqual(a(2*DAY), "2 days")
 629         self.failUnlessEqual(a(2*MONTH), "2 months")
 630         YEAR = 365*DAY
 631         self.failUnlessEqual(a(5*YEAR), "5 years")
 632
 633     def test_space(self):
 634         tests_si = [(None, "unknown"),
 635                     (0, "0 B"),
 636                     (1, "1 B"),
 637                     (999, "999 B"),
 638                     (1000, "1000 B"),
 639                     (1023, "1023 B"),
 640                     (1024, "1.02 kB"),
 641                     (20*1000, "20.00 kB"),
 642                     (1024*1024, "1.05 MB"),
 643                     (1000*1000, "1.00 MB"),
 644                     (1000*1000*1000, "1.00 GB"),
 645                     (1000*1000*1000*1000, "1.00 TB"),
 646                     (1000*1000*1000*1000*1000, "1.00 PB"),
 647                     (1234567890123456, "1.23 PB"),
 648                     ]
 649         for (x, expected) in tests_si:
 650             got = abbreviate.abbreviate_space(x, SI=True)
 651             self.failUnlessEqual(got, expected)
 652
 653         tests_base1024 = [(None, "unknown"),
 654                           (0, "0 B"),
 655                           (1, "1 B"),
 656                           (999, "999 B"),
 657                           (1000, "1000 B"),
 658                           (1023, "1023 B"),
 659                           (1024, "1.00 kiB"),
 660                           (20*1024, "20.00 kiB"),
 661                           (1000*1000, "976.56 kiB"),
 662                           (1024*1024, "1.00 MiB"),
 663                           (1024*1024*1024, "1.00 GiB"),
 664                           (1024*1024*1024*1024, "1.00 TiB"),
 665                           (1000*1000*1000*1000*1000, "909.49 TiB"),
 666                           (1024*1024*1024*1024*1024, "1.00 PiB"),
 667                           (1234567890123456, "1.10 PiB"),
 668                     ]
 669         for (x, expected) in tests_base1024:
 670             got = abbreviate.abbreviate_space(x, SI=False)
 671             self.failUnlessEqual(got, expected)
 672
 673         self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567),
 674                              "(1.23 MB, 1.18 MiB)")
 675
 676     def test_parse_space(self):
 677         p = abbreviate.parse_abbreviated_size
 678         self.failUnlessEqual(p(""), None)
 679         self.failUnlessEqual(p(None), None)
 680         self.failUnlessEqual(p("123"), 123)
 681         self.failUnlessEqual(p("123B"), 123)
 682         self.failUnlessEqual(p("2K"), 2000)
 683         self.failUnlessEqual(p("2kb"), 2000)
 684         self.failUnlessEqual(p("2KiB"), 2048)
 685         self.failUnlessEqual(p("10MB"), 10*1000*1000)
 686         self.failUnlessEqual(p("10MiB"), 10*1024*1024)
 687         self.failUnlessEqual(p("5G"), 5*1000*1000*1000)
 688         self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024)
 689         e = self.failUnlessRaises(ValueError, p, "12 cubits")
 690         self.failUnless("12 cubits" in str(e))
 691
 692 class Limiter(unittest.TestCase):
 693     def job(self, i, foo):
 694         self.calls.append( (i, foo) )
 695         self.simultaneous += 1
 696         self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous)
 697         d = defer.Deferred()
 698         def _done():
 699             self.simultaneous -= 1
 700             d.callback("done %d" % i)
 701         reactor.callLater(1.0, _done)
 702         return d
 703
 704     def bad_job(self, i, foo):
 705         raise ValueError("bad_job %d" % i)
 706
 707     def test_limiter(self):
 708         self.calls = []
 709         self.simultaneous = 0
 710         self.peak_simultaneous = 0
 711         l = limiter.ConcurrencyLimiter()
 712         dl = []
 713         for i in range(20):
 714             dl.append(l.add(self.job, i, foo=str(i)))
 715         d = defer.DeferredList(dl, fireOnOneErrback=True)
 716         def _done(res):
 717             self.failUnlessEqual(self.simultaneous, 0)
 718             self.failUnless(self.peak_simultaneous <= 10)
 719             self.failUnlessEqual(len(self.calls), 20)
 720             for i in range(20):
 721                 self.failUnless( (i, str(i)) in self.calls)
 722         d.addCallback(_done)
 723         return d
 724
 725     def test_errors(self):
 726         self.calls = []
 727         self.simultaneous = 0
 728         self.peak_simultaneous = 0
 729         l = limiter.ConcurrencyLimiter()
 730         dl = []
 731         for i in range(20):
 732             dl.append(l.add(self.job, i, foo=str(i)))
 733         d2 = l.add(self.bad_job, 21, "21")
 734         d = defer.DeferredList(dl, fireOnOneErrback=True)
 735         def _most_done(res):
 736             results = []
 737             for (success, result) in res:
 738                 self.failUnlessEqual(success, True)
 739                 results.append(result)
 740             results.sort()
 741             expected_results = ["done %d" % i for i in range(20)]
 742             expected_results.sort()
 743             self.failUnlessEqual(results, expected_results)
 744             self.failUnless(self.peak_simultaneous <= 10)
 745             self.failUnlessEqual(len(self.calls), 20)
 746             for i in range(20):
 747                 self.failUnless( (i, str(i)) in self.calls)
 748             def _good(res):
 749                 self.fail("should have failed, not got %s" % (res,))
 750             def _err(f):
 751                 f.trap(ValueError)
 752                 self.failUnless("bad_job 21" in str(f))
 753             d2.addCallbacks(_good, _err)
 754             return d2
 755         d.addCallback(_most_done)
 756         def _all_done(res):
 757             self.failUnlessEqual(self.simultaneous, 0)
 758             self.failUnless(self.peak_simultaneous <= 10)
 759             self.failUnlessEqual(len(self.calls), 20)
 760             for i in range(20):
 761                 self.failUnless( (i, str(i)) in self.calls)
 762         d.addCallback(_all_done)
 763         return d
 764
 765 class TimeFormat(unittest.TestCase):
 766     def test_epoch(self):
 767         s = time_format.iso_utc_time_to_seconds("1970-01-01T00:00:01")
 768         self.failUnlessEqual(s, 1.0)
 769         s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01")
 770         self.failUnlessEqual(s, 1.0)
 771         s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01")
 772         self.failUnlessEqual(s, 1.0)
 773
 774         self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01")
 775         self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "),
 776                              "1970-01-01 00:00:01")
 777
 778         now = time.time()
 779         isostr = time_format.iso_utc(now)
 780         timestamp = time_format.iso_utc_time_to_seconds(isostr)
 781         self.failUnlessEqual(int(timestamp), int(now))
 782
 783         def my_time():
 784             return 1.0
 785         self.failUnlessEqual(time_format.iso_utc(t=my_time),
 786                              "1970-01-01_00:00:01")
 787         e = self.failUnlessRaises(ValueError,
 788                                   time_format.iso_utc_time_to_seconds,
 789                                   "invalid timestring")
 790         self.failUnless("not a complete ISO8601 timestamp" in str(e))
 791         s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01.500")
 792         self.failUnlessEqual(s, 1.5)
 793
 794         # Look for daylight-savings-related errors.
 795         thatmomentinmarch = time_format.iso_utc_time_to_seconds("2009-03-20 21:49:02.226536")
 796         self.failUnlessEqual(thatmomentinmarch, 1237585742.226536)
 797
 798 class CacheDir(unittest.TestCase):
 799     def test_basic(self):
 800         basedir = "test_util/CacheDir/test_basic"
 801
 802         def _failIfExists(name):
 803             absfn = os.path.join(basedir, name)
 804             self.failIf(os.path.exists(absfn),
 805                         "%s exists but it shouldn't" % absfn)
 806
 807         def _failUnlessExists(name):
 808             absfn = os.path.join(basedir, name)
 809             self.failUnless(os.path.exists(absfn),
 810                             "%s doesn't exist but it should" % absfn)
 811
 812         cdm = cachedir.CacheDirectoryManager(basedir)
 813         a = cdm.get_file("a")
 814         b = cdm.get_file("b")
 815         c = cdm.get_file("c")
 816         f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f
 817         f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f
 818         f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f
 819
 820         _failUnlessExists("a")
 821         _failUnlessExists("b")
 822         _failUnlessExists("c")
 823
 824         cdm.check()
 825
 826         _failUnlessExists("a")
 827         _failUnlessExists("b")
 828         _failUnlessExists("c")
 829
 830         del a
 831         # this file won't be deleted yet, because it isn't old enough
 832         cdm.check()
 833         _failUnlessExists("a")
 834         _failUnlessExists("b")
 835         _failUnlessExists("c")
 836
 837         # we change the definition of "old" to make everything old
 838         cdm.old = -10
 839
 840         cdm.check()
 841         _failIfExists("a")
 842         _failUnlessExists("b")
 843         _failUnlessExists("c")
 844
 845         cdm.old = 60*60
 846
 847         del b
 848
 849         cdm.check()
 850         _failIfExists("a")
 851         _failUnlessExists("b")
 852         _failUnlessExists("c")
 853
 854         b2 = cdm.get_file("b")
 855
 856         cdm.check()
 857         _failIfExists("a")
 858         _failUnlessExists("b")
 859         _failUnlessExists("c")
 860
 861 ctr = [0]
 862 class EqButNotIs:
 863     def __init__(self, x):
 864         self.x = x
 865         self.hash = ctr[0]
 866         ctr[0] += 1
 867     def __repr__(self):
 868         return "<%s %s>" % (self.__class__.__name__, self.x,)
 869     def __hash__(self):
 870         return self.hash
 871     def __le__(self, other):
 872         return self.x <= other
 873     def __lt__(self, other):
 874         return self.x < other
 875     def __ge__(self, other):
 876         return self.x >= other
 877     def __gt__(self, other):
 878         return self.x > other
 879     def __ne__(self, other):
 880         return self.x != other
 881     def __eq__(self, other):
 882         return self.x == other
 883
 884 class DictUtil(unittest.TestCase):
 885     def _help_test_empty_dict(self, klass):
 886         d1 = klass()
 887         d2 = klass({})
 888
 889         self.failUnless(d1 == d2, "d1: %r, d2: %r" % (d1, d2,))
 890         self.failUnless(len(d1) == 0)
 891         self.failUnless(len(d2) == 0)
 892
 893     def _help_test_nonempty_dict(self, klass):
 894         d1 = klass({'a': 1, 'b': "eggs", 3: "spam",})
 895         d2 = klass({'a': 1, 'b': "eggs", 3: "spam",})
 896
 897         self.failUnless(d1 == d2)
 898         self.failUnless(len(d1) == 3, "%s, %s" % (len(d1), d1,))
 899         self.failUnless(len(d2) == 3)
 900
 901     def _help_test_eq_but_notis(self, klass):
 902         d = klass({'a': 3, 'b': EqButNotIs(3), 'c': 3})
 903         d.pop('b')
 904
 905         d.clear()
 906         d['a'] = 3
 907         d['b'] = EqButNotIs(3)
 908         d['c'] = 3
 909         d.pop('b')
 910
 911         d.clear()
 912         d['b'] = EqButNotIs(3)
 913         d['a'] = 3
 914         d['c'] = 3
 915         d.pop('b')
 916
 917         d.clear()
 918         d['a'] = EqButNotIs(3)
 919         d['c'] = 3
 920         d['a'] = 3
 921
 922         d.clear()
 923         fake3 = EqButNotIs(3)
 924         fake7 = EqButNotIs(7)
 925         d[fake3] = fake7
 926         d[3] = 7
 927         d[3] = 8
 928         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
 929         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
 930         # The real 7 should have been ejected by the d[3] = 8.
 931         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
 932         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
 933         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
 934         d[fake3] = 8
 935
 936         d.clear()
 937         d[3] = 7
 938         fake3 = EqButNotIs(3)
 939         fake7 = EqButNotIs(7)
 940         d[fake3] = fake7
 941         d[3] = 8
 942         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
 943         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
 944         # The real 7 should have been ejected by the d[3] = 8.
 945         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
 946         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
 947         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
 948         d[fake3] = 8
 949
 950     def test_all(self):
 951         self._help_test_eq_but_notis(dictutil.UtilDict)
 952         self._help_test_eq_but_notis(dictutil.NumDict)
 953         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
 954         self._help_test_nonempty_dict(dictutil.UtilDict)
 955         self._help_test_nonempty_dict(dictutil.NumDict)
 956         self._help_test_nonempty_dict(dictutil.ValueOrderedDict)
 957         self._help_test_eq_but_notis(dictutil.UtilDict)
 958         self._help_test_eq_but_notis(dictutil.NumDict)
 959         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
 960
 961     def test_dict_of_sets(self):
 962         ds = dictutil.DictOfSets()
 963         ds.add(1, "a")
 964         ds.add(2, "b")
 965         ds.add(2, "b")
 966         ds.add(2, "c")
 967         self.failUnlessEqual(ds[1], set(["a"]))
 968         self.failUnlessEqual(ds[2], set(["b", "c"]))
 969         ds.discard(3, "d") # should not raise an exception
 970         ds.discard(2, "b")
 971         self.failUnlessEqual(ds[2], set(["c"]))
 972         ds.discard(2, "c")
 973         self.failIf(2 in ds)
 974
 975         ds.union(1, ["a", "e"])
 976         ds.union(3, ["f"])
 977         self.failUnlessEqual(ds[1], set(["a","e"]))
 978         self.failUnlessEqual(ds[3], set(["f"]))
 979         ds2 = dictutil.DictOfSets()
 980         ds2.add(3, "f")
 981         ds2.add(3, "g")
 982         ds2.add(4, "h")
 983         ds.update(ds2)
 984         self.failUnlessEqual(ds[1], set(["a","e"]))
 985         self.failUnlessEqual(ds[3], set(["f", "g"]))
 986         self.failUnlessEqual(ds[4], set(["h"]))
 987
 988     def test_move(self):
 989         d1 = {1: "a", 2: "b"}
 990         d2 = {2: "c", 3: "d"}
 991         dictutil.move(1, d1, d2)
 992         self.failUnlessEqual(d1, {2: "b"})
 993         self.failUnlessEqual(d2, {1: "a", 2: "c", 3: "d"})
 994
 995         d1 = {1: "a", 2: "b"}
 996         d2 = {2: "c", 3: "d"}
 997         dictutil.move(2, d1, d2)
 998         self.failUnlessEqual(d1, {1: "a"})
 999         self.failUnlessEqual(d2, {2: "b", 3: "d"})
1000
1001         d1 = {1: "a", 2: "b"}
1002         d2 = {2: "c", 3: "d"}
1003         self.failUnlessRaises(KeyError, dictutil.move, 5, d1, d2, strict=True)
1004
1005     def test_subtract(self):
1006         d1 = {1: "a", 2: "b"}
1007         d2 = {2: "c", 3: "d"}
1008         d3 = dictutil.subtract(d1, d2)
1009         self.failUnlessEqual(d3, {1: "a"})
1010
1011         d1 = {1: "a", 2: "b"}
1012         d2 = {2: "c"}
1013         d3 = dictutil.subtract(d1, d2)
1014         self.failUnlessEqual(d3, {1: "a"})
1015
1016     def test_utildict(self):
1017         d = dictutil.UtilDict({1: "a", 2: "b"})
1018         d.del_if_present(1)
1019         d.del_if_present(3)
1020         self.failUnlessEqual(d, {2: "b"})
1021         def eq(a, b):
1022             return a == b
1023         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1024
1025         d = dictutil.UtilDict({1: "b", 2: "a"})
1026         self.failUnlessEqual(d.items_sorted_by_value(),
1027                              [(2, "a"), (1, "b")])
1028         self.failUnlessEqual(d.items_sorted_by_key(),
1029                              [(1, "b"), (2, "a")])
1030         self.failUnlessEqual(repr(d), "{1: 'b', 2: 'a'}")
1031         self.failUnless(1 in d)
1032
1033         d2 = dictutil.UtilDict({3: "c", 4: "d"})
1034         self.failUnless(d != d2)
1035         self.failUnless(d2 > d)
1036         self.failUnless(d2 >= d)
1037         self.failUnless(d <= d2)
1038         self.failUnless(d < d2)
1039         self.failUnlessEqual(d[1], "b")
1040         self.failUnlessEqual(sorted(list([k for k in d])), [1,2])
1041
1042         d3 = d.copy()
1043         self.failUnlessEqual(d, d3)
1044         self.failUnless(isinstance(d3, dictutil.UtilDict))
1045
1046         d4 = d.fromkeys([3,4], "e")
1047         self.failUnlessEqual(d4, {3: "e", 4: "e"})
1048
1049         self.failUnlessEqual(d.get(1), "b")
1050         self.failUnlessEqual(d.get(3), None)
1051         self.failUnlessEqual(d.get(3, "default"), "default")
1052         self.failUnlessEqual(sorted(list(d.items())),
1053                              [(1, "b"), (2, "a")])
1054         self.failUnlessEqual(sorted(list(d.iteritems())),
1055                              [(1, "b"), (2, "a")])
1056         self.failUnlessEqual(sorted(d.keys()), [1, 2])
1057         self.failUnlessEqual(sorted(d.values()), ["a", "b"])
1058         x = d.setdefault(1, "new")
1059         self.failUnlessEqual(x, "b")
1060         self.failUnlessEqual(d[1], "b")
1061         x = d.setdefault(3, "new")
1062         self.failUnlessEqual(x, "new")
1063         self.failUnlessEqual(d[3], "new")
1064         del d[3]
1065
1066         x = d.popitem()
1067         self.failUnless(x in [(1, "b"), (2, "a")])
1068         x = d.popitem()
1069         self.failUnless(x in [(1, "b"), (2, "a")])
1070         self.failUnlessRaises(KeyError, d.popitem)
1071
1072     def test_numdict(self):
1073         d = dictutil.NumDict({"a": 1, "b": 2})
1074
1075         d.add_num("a", 10, 5)
1076         d.add_num("c", 20, 5)
1077         d.add_num("d", 30)
1078         self.failUnlessEqual(d, {"a": 11, "b": 2, "c": 25, "d": 30})
1079
1080         d.subtract_num("a", 10)
1081         d.subtract_num("e", 10)
1082         d.subtract_num("f", 10, 15)
1083         self.failUnlessEqual(d, {"a": 1, "b": 2, "c": 25, "d": 30,
1084                                  "e": -10, "f": 5})
1085
1086         self.failUnlessEqual(d.sum(), sum([1, 2, 25, 30, -10, 5]))
1087
1088         d = dictutil.NumDict()
1089         d.inc("a")
1090         d.inc("a")
1091         d.inc("b", 5)
1092         self.failUnlessEqual(d, {"a": 2, "b": 6})
1093         d.dec("a")
1094         d.dec("c")
1095         d.dec("d", 5)
1096         self.failUnlessEqual(d, {"a": 1, "b": 6, "c": -1, "d": 4})
1097         self.failUnlessEqual(d.items_sorted_by_key(),
1098                              [("a", 1), ("b", 6), ("c", -1), ("d", 4)])
1099         self.failUnlessEqual(d.items_sorted_by_value(),
1100                              [("c", -1), ("a", 1), ("d", 4), ("b", 6)])
1101         self.failUnlessEqual(d.item_with_largest_value(), ("b", 6))
1102
1103         d = dictutil.NumDict({"a": 1, "b": 2})
1104         self.failUnlessEqual(repr(d), "{'a': 1, 'b': 2}")
1105         self.failUnless("a" in d)
1106
1107         d2 = dictutil.NumDict({"c": 3, "d": 4})
1108         self.failUnless(d != d2)
1109         self.failUnless(d2 > d)
1110         self.failUnless(d2 >= d)
1111         self.failUnless(d <= d2)
1112         self.failUnless(d < d2)
1113         self.failUnlessEqual(d["a"], 1)
1114         self.failUnlessEqual(sorted(list([k for k in d])), ["a","b"])
1115         def eq(a, b):
1116             return a == b
1117         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1118
1119         d3 = d.copy()
1120         self.failUnlessEqual(d, d3)
1121         self.failUnless(isinstance(d3, dictutil.NumDict))
1122
1123         d4 = d.fromkeys(["a","b"], 5)
1124         self.failUnlessEqual(d4, {"a": 5, "b": 5})
1125
1126         self.failUnlessEqual(d.get("a"), 1)
1127         self.failUnlessEqual(d.get("c"), 0)
1128         self.failUnlessEqual(d.get("c", 5), 5)
1129         self.failUnlessEqual(sorted(list(d.items())),
1130                              [("a", 1), ("b", 2)])
1131         self.failUnlessEqual(sorted(list(d.iteritems())),
1132                              [("a", 1), ("b", 2)])
1133         self.failUnlessEqual(sorted(d.keys()), ["a", "b"])
1134         self.failUnlessEqual(sorted(d.values()), [1, 2])
1135         self.failUnless(d.has_key("a"))
1136         self.failIf(d.has_key("c"))
1137
1138         x = d.setdefault("c", 3)
1139         self.failUnlessEqual(x, 3)
1140         self.failUnlessEqual(d["c"], 3)
1141         x = d.setdefault("c", 5)
1142         self.failUnlessEqual(x, 3)
1143         self.failUnlessEqual(d["c"], 3)
1144         del d["c"]
1145
1146         x = d.popitem()
1147         self.failUnless(x in [("a", 1), ("b", 2)])
1148         x = d.popitem()
1149         self.failUnless(x in [("a", 1), ("b", 2)])
1150         self.failUnlessRaises(KeyError, d.popitem)
1151
1152         d.update({"c": 3})
1153         d.update({"c": 4, "d": 5})
1154         self.failUnlessEqual(d, {"c": 4, "d": 5})
1155
1156     def test_del_if_present(self):
1157         d = {1: "a", 2: "b"}
1158         dictutil.del_if_present(d, 1)
1159         dictutil.del_if_present(d, 3)
1160         self.failUnlessEqual(d, {2: "b"})
1161
1162     def test_valueordereddict(self):
1163         d = dictutil.ValueOrderedDict()
1164         d["a"] = 3
1165         d["b"] = 2
1166         d["c"] = 1
1167
1168         self.failUnlessEqual(d, {"a": 3, "b": 2, "c": 1})
1169         self.failUnlessEqual(d.items(), [("c", 1), ("b", 2), ("a", 3)])
1170         self.failUnlessEqual(d.values(), [1, 2, 3])
1171         self.failUnlessEqual(d.keys(), ["c", "b", "a"])
1172         self.failUnlessEqual(repr(d), "<ValueOrderedDict {c: 1, b: 2, a: 3}>")
1173         def eq(a, b):
1174             return a == b
1175         self.failIf(d == {"a": 4})
1176         self.failUnless(d != {"a": 4})
1177
1178         x = d.setdefault("d", 0)
1179         self.failUnlessEqual(x, 0)
1180         self.failUnlessEqual(d["d"], 0)
1181         x = d.setdefault("d", -1)
1182         self.failUnlessEqual(x, 0)
1183         self.failUnlessEqual(d["d"], 0)
1184
1185         x = d.remove("e", "default", False)
1186         self.failUnlessEqual(x, "default")
1187         self.failUnlessRaises(KeyError, d.remove, "e", "default", True)
1188         x = d.remove("d", 5)
1189         self.failUnlessEqual(x, 0)
1190
1191         x = d.__getitem__("c")
1192         self.failUnlessEqual(x, 1)
1193         x = d.__getitem__("e", "default", False)
1194         self.failUnlessEqual(x, "default")
1195         self.failUnlessRaises(KeyError, d.__getitem__, "e", "default", True)
1196
1197         self.failUnlessEqual(d.popitem(), ("c", 1))
1198         self.failUnlessEqual(d.popitem(), ("b", 2))
1199         self.failUnlessEqual(d.popitem(), ("a", 3))
1200         self.failUnlessRaises(KeyError, d.popitem)
1201
1202         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1203         x = d.pop("d", "default", False)
1204         self.failUnlessEqual(x, "default")
1205         self.failUnlessRaises(KeyError, d.pop, "d", "default", True)
1206         x = d.pop("b")
1207         self.failUnlessEqual(x, 2)
1208         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1209
1210         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1211         x = d.pop_from_list(1) # pop the second item, b/2
1212         self.failUnlessEqual(x, "b")
1213         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1214
1215 class FakeRemoteReference:
1216     def callRemote(self, methname, *args, **kwargs):
1217         return defer.maybeDeferred(self.oops)
1218     def oops(self):
1219         raise IndexError("remote missing key")
1220
1221 class RemoteFailures(unittest.TestCase):
1222     def test_check(self):
1223         check_local = rrefutil.check_local
1224         check_remote = rrefutil.check_remote
1225         try:
1226             raise IndexError("local missing key")
1227         except IndexError:
1228             localf = Failure()
1229
1230         self.failUnlessEqual(localf.check(IndexError, KeyError), IndexError)
1231         self.failUnlessEqual(localf.check(ValueError, KeyError), None)
1232         self.failUnlessEqual(localf.check(ServerFailure), None)
1233         self.failUnlessEqual(check_local(localf, IndexError, KeyError),
1234                              IndexError)
1235         self.failUnlessEqual(check_local(localf, ValueError, KeyError), None)
1236         self.failUnlessEqual(check_remote(localf, IndexError, KeyError), None)
1237         self.failUnlessEqual(check_remote(localf, ValueError, KeyError), None)
1238
1239         frr = FakeRemoteReference()
1240         wrr = rrefutil.WrappedRemoteReference(frr)
1241         d = wrr.callRemote("oops")
1242         def _check(f):
1243             self.failUnlessEqual(f.check(IndexError, KeyError), None)
1244             self.failUnlessEqual(f.check(ServerFailure, KeyError),
1245                                  ServerFailure)
1246             self.failUnlessEqual(check_remote(f, IndexError, KeyError),
1247                                  IndexError)
1248             self.failUnlessEqual(check_remote(f, ValueError, KeyError), None)
1249             self.failUnlessEqual(check_local(f, IndexError, KeyError), None)
1250             self.failUnlessEqual(check_local(f, ValueError, KeyError), None)
1251         d.addErrback(_check)
1252         return d
1253
1254     def test_is_remote(self):
1255         try:
1256             raise IndexError("local missing key")
1257         except IndexError:
1258             localf = Failure()
1259         self.failIf(rrefutil.is_remote(localf))
1260         self.failUnless(rrefutil.is_local(localf))
1261
1262         frr = FakeRemoteReference()
1263         wrr = rrefutil.WrappedRemoteReference(frr)
1264         d = wrr.callRemote("oops")
1265         def _check(f):
1266             self.failUnless(rrefutil.is_remote(f))
1267             self.failIf(rrefutil.is_local(f))
1268         d.addErrback(_check)
1269         return d
1270
1271     def test_trap(self):
1272         try:
1273             raise IndexError("local missing key")
1274         except IndexError:
1275             localf = Failure()
1276
1277         self.failUnlessRaises(Failure, localf.trap, ValueError, KeyError)
1278         self.failUnlessRaises(Failure, localf.trap, ServerFailure)
1279         self.failUnlessEqual(localf.trap(IndexError, KeyError), IndexError)
1280         self.failUnlessEqual(rrefutil.trap_local(localf, IndexError, KeyError),
1281                              IndexError)
1282         self.failUnlessRaises(Failure,
1283                               rrefutil.trap_remote, localf, ValueError, KeyError)
1284
1285         frr = FakeRemoteReference()
1286         wrr = rrefutil.WrappedRemoteReference(frr)
1287         d = wrr.callRemote("oops")
1288         def _check(f):
1289             self.failUnlessRaises(Failure,
1290                                   f.trap, ValueError, KeyError)
1291             self.failUnlessRaises(Failure,
1292                                   f.trap, IndexError)
1293             self.failUnlessEqual(f.trap(ServerFailure), ServerFailure)
1294             self.failUnlessRaises(Failure,
1295                                   rrefutil.trap_remote, f, ValueError, KeyError)
1296             self.failUnlessEqual(rrefutil.trap_remote(f, IndexError, KeyError),
1297                                  IndexError)
1298             self.failUnlessRaises(Failure,
1299                                   rrefutil.trap_local, f, ValueError, KeyError)
1300             self.failUnlessRaises(Failure,
1301                                   rrefutil.trap_local, f, IndexError)
1302         d.addErrback(_check)
1303         return d
1304
1305 class Pipeline(unittest.TestCase):
1306     def pause(self, *args, **kwargs):
1307         d = defer.Deferred()
1308         self.calls.append( (d, args, kwargs) )
1309         return d
1310
1311     def failUnlessCallsAre(self, expected):
1312         #print self.calls
1313         #print expected
1314         self.failUnlessEqual(len(self.calls), len(expected), self.calls)
1315         for i,c in enumerate(self.calls):
1316             self.failUnlessEqual(c[1:], expected[i], str(i))
1317
1318     def test_basic(self):
1319         self.calls = []
1320         finished = []
1321         p = pipeline.Pipeline(100)
1322
1323         d = p.flush() # fires immediately
1324         d.addCallbacks(finished.append, log.err)
1325         self.failUnlessEqual(len(finished), 1)
1326         finished = []
1327
1328         d = p.add(10, self.pause, "one")
1329         # the call should start right away, and our return Deferred should
1330         # fire right away
1331         d.addCallbacks(finished.append, log.err)
1332         self.failUnlessEqual(len(finished), 1)
1333         self.failUnlessEqual(finished[0], None)
1334         self.failUnlessCallsAre([ ( ("one",) , {} ) ])
1335         self.failUnlessEqual(p.gauge, 10)
1336
1337         # pipeline: [one]
1338
1339         finished = []
1340         d = p.add(20, self.pause, "two", kw=2)
1341         # pipeline: [one, two]
1342
1343         # the call and the Deferred should fire right away
1344         d.addCallbacks(finished.append, log.err)
1345         self.failUnlessEqual(len(finished), 1)
1346         self.failUnlessEqual(finished[0], None)
1347         self.failUnlessCallsAre([ ( ("one",) , {} ),
1348                                   ( ("two",) , {"kw": 2} ),
1349                                   ])
1350         self.failUnlessEqual(p.gauge, 30)
1351
1352         self.calls[0][0].callback("one-result")
1353         # pipeline: [two]
1354         self.failUnlessEqual(p.gauge, 20)
1355
1356         finished = []
1357         d = p.add(90, self.pause, "three", "posarg1")
1358         # pipeline: [two, three]
1359         flushed = []
1360         fd = p.flush()
1361         fd.addCallbacks(flushed.append, log.err)
1362         self.failUnlessEqual(flushed, [])
1363
1364         # the call will be made right away, but the return Deferred will not,
1365         # because the pipeline is now full.
1366         d.addCallbacks(finished.append, log.err)
1367         self.failUnlessEqual(len(finished), 0)
1368         self.failUnlessCallsAre([ ( ("one",) , {} ),
1369                                   ( ("two",) , {"kw": 2} ),
1370                                   ( ("three", "posarg1"), {} ),
1371                                   ])
1372         self.failUnlessEqual(p.gauge, 110)
1373
1374         self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause)
1375
1376         # retiring either call will unblock the pipeline, causing the #3
1377         # Deferred to fire
1378         self.calls[2][0].callback("three-result")
1379         # pipeline: [two]
1380
1381         self.failUnlessEqual(len(finished), 1)
1382         self.failUnlessEqual(finished[0], None)
1383         self.failUnlessEqual(flushed, [])
1384
1385         # retiring call#2 will finally allow the flush() Deferred to fire
1386         self.calls[1][0].callback("two-result")
1387         self.failUnlessEqual(len(flushed), 1)
1388
1389     def test_errors(self):
1390         self.calls = []
1391         p = pipeline.Pipeline(100)
1392
1393         d1 = p.add(200, self.pause, "one")
1394         d2 = p.flush()
1395
1396         finished = []
1397         d1.addBoth(finished.append)
1398         self.failUnlessEqual(finished, [])
1399
1400         flushed = []
1401         d2.addBoth(flushed.append)
1402         self.failUnlessEqual(flushed, [])
1403
1404         self.calls[0][0].errback(ValueError("oops"))
1405
1406         self.failUnlessEqual(len(finished), 1)
1407         f = finished[0]
1408         self.failUnless(isinstance(f, Failure))
1409         self.failUnless(f.check(pipeline.PipelineError))
1410         r = repr(f.value)
1411         self.failUnless("ValueError" in r, r)
1412         f2 = f.value.error
1413         self.failUnless(f2.check(ValueError))
1414
1415         self.failUnlessEqual(len(flushed), 1)
1416         f = flushed[0]
1417         self.failUnless(isinstance(f, Failure))
1418         self.failUnless(f.check(pipeline.PipelineError))
1419         f2 = f.value.error
1420         self.failUnless(f2.check(ValueError))
1421
1422         # now that the pipeline is in the failed state, any new calls will
1423         # fail immediately
1424
1425         d3 = p.add(20, self.pause, "two")
1426
1427         finished = []
1428         d3.addBoth(finished.append)
1429         self.failUnlessEqual(len(finished), 1)
1430         f = finished[0]
1431         self.failUnless(isinstance(f, Failure))
1432         self.failUnless(f.check(pipeline.PipelineError))
1433         r = repr(f.value)
1434         self.failUnless("ValueError" in r, r)
1435         f2 = f.value.error
1436         self.failUnless(f2.check(ValueError))
1437
1438         d4 = p.flush()
1439         flushed = []
1440         d4.addBoth(flushed.append)
1441         self.failUnlessEqual(len(flushed), 1)
1442         f = flushed[0]
1443         self.failUnless(isinstance(f, Failure))
1444         self.failUnless(f.check(pipeline.PipelineError))
1445         f2 = f.value.error
1446         self.failUnless(f2.check(ValueError))
1447
1448
1449     def test_errors2(self):
1450         self.calls = []
1451         p = pipeline.Pipeline(100)
1452
1453         d1 = p.add(10, self.pause, "one")
1454         d2 = p.add(20, self.pause, "two")
1455         d3 = p.add(30, self.pause, "three")
1456         d4 = p.flush()
1457
1458         # one call fails, then the second one succeeds: make sure
1459         # ExpandableDeferredList tolerates the second one
1460
1461         flushed = []
1462         d4.addBoth(flushed.append)
1463         self.failUnlessEqual(flushed, [])
1464
1465         self.calls[0][0].errback(ValueError("oops"))
1466         self.failUnlessEqual(len(flushed), 1)
1467         f = flushed[0]
1468         self.failUnless(isinstance(f, Failure))
1469         self.failUnless(f.check(pipeline.PipelineError))
1470         f2 = f.value.error
1471         self.failUnless(f2.check(ValueError))
1472
1473         self.calls[1][0].callback("two-result")
1474         self.calls[2][0].errback(ValueError("three-error"))