src/allmydata/test/test_util.py

   1
   2 def foo(): pass # keep the line number constant
   3
   4 import os, time
   5 from StringIO import StringIO
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.python.failure import Failure
   9 from twisted.python import log
  10
  11 from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
  12 from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
  13 from allmydata.util import limiter, time_format, pollmixin, cachedir
  14 from allmydata.util import statistics, dictutil, pipeline
  15
  16 class Base32(unittest.TestCase):
  17     def test_b2a_matches_Pythons(self):
  18         import base64
  19         y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0"
  20         x = base64.b32encode(y)
  21         while x and x[-1] == '=':
  22             x = x[:-1]
  23         x = x.lower()
  24         self.failUnlessEqual(base32.b2a(y), x)
  25     def test_b2a(self):
  26         self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a")
  27     def test_b2a_or_none(self):
  28         self.failUnlessEqual(base32.b2a_or_none(None), None)
  29         self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a")
  30     def test_a2b(self):
  31         self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34")
  32         self.failUnlessRaises(AssertionError, base32.a2b, "b0gus")
  33
  34 class IDLib(unittest.TestCase):
  35     def test_nodeid_b2a(self):
  36         self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
  37
  38 class NoArgumentException(Exception):
  39     def __init__(self):
  40         pass
  41
  42 class HumanReadable(unittest.TestCase):
  43     def test_repr(self):
  44         hr = humanreadable.hr
  45         self.failUnlessEqual(hr(foo), "<foo() at test_util.py:2>")
  46         self.failUnlessEqual(hr(self.test_repr),
  47                              "<bound method HumanReadable.test_repr of <allmydata.test.test_util.HumanReadable testMethod=test_repr>>")
  48         self.failUnlessEqual(hr(1L), "1")
  49         self.failUnlessEqual(hr(10**40),
  50                              "100000000000000000...000000000000000000")
  51         self.failUnlessEqual(hr(self), "<allmydata.test.test_util.HumanReadable testMethod=test_repr>")
  52         self.failUnlessEqual(hr([1,2]), "[1, 2]")
  53         self.failUnlessEqual(hr({1:2}), "{1:2}")
  54         try:
  55             raise ValueError
  56         except Exception, e:
  57             self.failUnless(
  58                 hr(e) == "<ValueError: ()>" # python-2.4
  59                 or hr(e) == "ValueError()") # python-2.5
  60         try:
  61             raise ValueError("oops")
  62         except Exception, e:
  63             self.failUnless(
  64                 hr(e) == "<ValueError: 'oops'>" # python-2.4
  65                 or hr(e) == "ValueError('oops',)") # python-2.5
  66         try:
  67             raise NoArgumentException
  68         except Exception, e:
  69             self.failUnless(
  70                 hr(e) == "<NoArgumentException>" # python-2.4
  71                 or hr(e) == "NoArgumentException()") # python-2.5
  72
  73
  74 class MyList(list):
  75     pass
  76
  77 class Math(unittest.TestCase):
  78     def test_div_ceil(self):
  79         f = mathutil.div_ceil
  80         self.failUnlessEqual(f(0, 1), 0)
  81         self.failUnlessEqual(f(0, 2), 0)
  82         self.failUnlessEqual(f(0, 3), 0)
  83         self.failUnlessEqual(f(1, 3), 1)
  84         self.failUnlessEqual(f(2, 3), 1)
  85         self.failUnlessEqual(f(3, 3), 1)
  86         self.failUnlessEqual(f(4, 3), 2)
  87         self.failUnlessEqual(f(5, 3), 2)
  88         self.failUnlessEqual(f(6, 3), 2)
  89         self.failUnlessEqual(f(7, 3), 3)
  90
  91     def test_next_multiple(self):
  92         f = mathutil.next_multiple
  93         self.failUnlessEqual(f(5, 1), 5)
  94         self.failUnlessEqual(f(5, 2), 6)
  95         self.failUnlessEqual(f(5, 3), 6)
  96         self.failUnlessEqual(f(5, 4), 8)
  97         self.failUnlessEqual(f(5, 5), 5)
  98         self.failUnlessEqual(f(5, 6), 6)
  99         self.failUnlessEqual(f(32, 1), 32)
 100         self.failUnlessEqual(f(32, 2), 32)
 101         self.failUnlessEqual(f(32, 3), 33)
 102         self.failUnlessEqual(f(32, 4), 32)
 103         self.failUnlessEqual(f(32, 5), 35)
 104         self.failUnlessEqual(f(32, 6), 36)
 105         self.failUnlessEqual(f(32, 7), 35)
 106         self.failUnlessEqual(f(32, 8), 32)
 107         self.failUnlessEqual(f(32, 9), 36)
 108         self.failUnlessEqual(f(32, 10), 40)
 109         self.failUnlessEqual(f(32, 11), 33)
 110         self.failUnlessEqual(f(32, 12), 36)
 111         self.failUnlessEqual(f(32, 13), 39)
 112         self.failUnlessEqual(f(32, 14), 42)
 113         self.failUnlessEqual(f(32, 15), 45)
 114         self.failUnlessEqual(f(32, 16), 32)
 115         self.failUnlessEqual(f(32, 17), 34)
 116         self.failUnlessEqual(f(32, 18), 36)
 117         self.failUnlessEqual(f(32, 589), 589)
 118
 119     def test_pad_size(self):
 120         f = mathutil.pad_size
 121         self.failUnlessEqual(f(0, 4), 0)
 122         self.failUnlessEqual(f(1, 4), 3)
 123         self.failUnlessEqual(f(2, 4), 2)
 124         self.failUnlessEqual(f(3, 4), 1)
 125         self.failUnlessEqual(f(4, 4), 0)
 126         self.failUnlessEqual(f(5, 4), 3)
 127
 128     def test_is_power_of_k(self):
 129         f = mathutil.is_power_of_k
 130         for i in range(1, 100):
 131             if i in (1, 2, 4, 8, 16, 32, 64):
 132                 self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
 133             else:
 134                 self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
 135         for i in range(1, 100):
 136             if i in (1, 3, 9, 27, 81):
 137                 self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
 138             else:
 139                 self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
 140
 141     def test_next_power_of_k(self):
 142         f = mathutil.next_power_of_k
 143         self.failUnlessEqual(f(0,2), 1)
 144         self.failUnlessEqual(f(1,2), 1)
 145         self.failUnlessEqual(f(2,2), 2)
 146         self.failUnlessEqual(f(3,2), 4)
 147         self.failUnlessEqual(f(4,2), 4)
 148         for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
 149         for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
 150         for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
 151         for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
 152         for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
 153
 154         self.failUnlessEqual(f(0,3), 1)
 155         self.failUnlessEqual(f(1,3), 1)
 156         self.failUnlessEqual(f(2,3), 3)
 157         self.failUnlessEqual(f(3,3), 3)
 158         for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
 159         for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
 160         for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
 161         for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
 162
 163     def test_ave(self):
 164         f = mathutil.ave
 165         self.failUnlessEqual(f([1,2,3]), 2)
 166         self.failUnlessEqual(f([0,0,0,4]), 1)
 167         self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
 168
 169     def test_round_sigfigs(self):
 170         f = mathutil.round_sigfigs
 171         self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002)
 172
 173 class Statistics(unittest.TestCase):
 174     def should_assert(self, msg, func, *args, **kwargs):
 175         try:
 176             func(*args, **kwargs)
 177             self.fail(msg)
 178         except AssertionError, e:
 179             pass
 180
 181     def failUnlessListEqual(self, a, b, msg = None):
 182         self.failUnlessEqual(len(a), len(b))
 183         for i in range(len(a)):
 184             self.failUnlessEqual(a[i], b[i], msg)
 185
 186     def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None):
 187         self.failUnlessEqual(len(a), len(b))
 188         for i in range(len(a)):
 189             self.failUnlessAlmostEqual(a[i], b[i], places, msg)
 190
 191     def test_binomial_coeff(self):
 192         f = statistics.binomial_coeff
 193         self.failUnlessEqual(f(20, 0), 1)
 194         self.failUnlessEqual(f(20, 1), 20)
 195         self.failUnlessEqual(f(20, 2), 190)
 196         self.failUnlessEqual(f(20, 8), f(20, 12))
 197         self.should_assert("Should assert if n < k", f, 2, 3)
 198
 199     def test_binomial_distribution_pmf(self):
 200         f = statistics.binomial_distribution_pmf
 201
 202         pmf_comp = f(2, .1)
 203         pmf_stat = [0.81, 0.18, 0.01]
 204         self.failUnlessListAlmostEqual(pmf_comp, pmf_stat)
 205
 206         # Summing across a PMF should give the total probability 1
 207         self.failUnlessAlmostEqual(sum(pmf_comp), 1)
 208         self.should_assert("Should assert if not 0<=p<=1", f, 1, -1)
 209         self.should_assert("Should assert if n < 1", f, 0, .1)
 210
 211         out = StringIO()
 212         statistics.print_pmf(pmf_comp, out=out)
 213         lines = out.getvalue().splitlines()
 214         self.failUnlessEqual(lines[0], "i=0: 0.81")
 215         self.failUnlessEqual(lines[1], "i=1: 0.18")
 216         self.failUnlessEqual(lines[2], "i=2: 0.01")
 217
 218     def test_survival_pmf(self):
 219         f = statistics.survival_pmf
 220         # Cross-check binomial-distribution method against convolution
 221         # method.
 222         p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20
 223         pmf1 = statistics.survival_pmf_via_conv(p_list)
 224         pmf2 = statistics.survival_pmf_via_bd(p_list)
 225         self.failUnlessListAlmostEqual(pmf1, pmf2)
 226         self.failUnlessTrue(statistics.valid_pmf(pmf1))
 227         self.should_assert("Should assert if p_i > 1", f, [1.1]);
 228         self.should_assert("Should assert if p_i < 0", f, [-.1]);
 229
 230     def test_repair_count_pmf(self):
 231         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 232         repair_pmf = statistics.repair_count_pmf(survival_pmf, 3)
 233         # repair_pmf[0] == sum(survival_pmf[0,1,2,5])
 234         # repair_pmf[1] == survival_pmf[4]
 235         # repair_pmf[2] = survival_pmf[3]
 236         self.failUnlessListAlmostEqual(repair_pmf,
 237                                        [0.00001 + 0.00045 + 0.0081 + 0.59049,
 238                                         .32805,
 239                                         .0729,
 240                                         0, 0, 0])
 241
 242     def test_repair_cost(self):
 243         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 244         bwcost = statistics.bandwidth_cost_function
 245         cost = statistics.mean_repair_cost(bwcost, 1000,
 246                                            survival_pmf, 3, ul_dl_ratio=1.0)
 247         self.failUnlessAlmostEqual(cost, 558.90)
 248         cost = statistics.mean_repair_cost(bwcost, 1000,
 249                                            survival_pmf, 3, ul_dl_ratio=8.0)
 250         self.failUnlessAlmostEqual(cost, 1664.55)
 251
 252         # I haven't manually checked the math beyond here -warner
 253         cost = statistics.eternal_repair_cost(bwcost, 1000,
 254                                               survival_pmf, 3,
 255                                               discount_rate=0, ul_dl_ratio=1.0)
 256         self.failUnlessAlmostEqual(cost, 65292.056074766246)
 257         cost = statistics.eternal_repair_cost(bwcost, 1000,
 258                                               survival_pmf, 3,
 259                                               discount_rate=0.05,
 260                                               ul_dl_ratio=1.0)
 261         self.failUnlessAlmostEqual(cost, 9133.6097158191551)
 262
 263     def test_convolve(self):
 264         f = statistics.convolve
 265         v1 = [ 1, 2, 3 ]
 266         v2 = [ 4, 5, 6 ]
 267         v3 = [ 7, 8 ]
 268         v1v2result = [ 4, 13, 28, 27, 18 ]
 269         # Convolution is commutative
 270         r1 = f(v1, v2)
 271         r2 = f(v2, v1)
 272         self.failUnlessListEqual(r1, r2, "Convolution should be commutative")
 273         self.failUnlessListEqual(r1, v1v2result, "Didn't match known result")
 274         # Convolution is associative
 275         r1 = f(f(v1, v2), v3)
 276         r2 = f(v1, f(v2, v3))
 277         self.failUnlessListEqual(r1, r2, "Convolution should be associative")
 278         # Convolution is distributive
 279         r1 = f(v3, [ a + b for a, b in zip(v1, v2) ])
 280         tmp1 = f(v3, v1)
 281         tmp2 = f(v3, v2)
 282         r2 = [ a + b for a, b in zip(tmp1, tmp2) ]
 283         self.failUnlessListEqual(r1, r2, "Convolution should be distributive")
 284         # Convolution is scalar multiplication associative
 285         tmp1 = f(v1, v2)
 286         r1 = [ a * 4 for a in tmp1 ]
 287         tmp2 = [ a * 4 for a in v1 ]
 288         r2 = f(tmp2, v2)
 289         self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative")
 290
 291     def test_find_k(self):
 292         f = statistics.find_k
 293         g = statistics.pr_file_loss
 294         plist = [.9] * 10 + [.8] * 10 # N=20
 295         t = .0001
 296         k = f(plist, t)
 297         self.failUnlessEqual(k, 10)
 298         self.failUnless(g(plist, k) < t)
 299
 300     def test_pr_file_loss(self):
 301         f = statistics.pr_file_loss
 302         plist = [.5] * 10
 303         self.failUnlessEqual(f(plist, 3), .0546875)
 304
 305     def test_pr_backup_file_loss(self):
 306         f = statistics.pr_backup_file_loss
 307         plist = [.5] * 10
 308         self.failUnlessEqual(f(plist, .5, 3), .02734375)
 309
 310
 311 class Asserts(unittest.TestCase):
 312     def should_assert(self, func, *args, **kwargs):
 313         try:
 314             func(*args, **kwargs)
 315         except AssertionError, e:
 316             return str(e)
 317         except Exception, e:
 318             self.fail("assert failed with non-AssertionError: %s" % e)
 319         self.fail("assert was not caught")
 320
 321     def should_not_assert(self, func, *args, **kwargs):
 322         if "re" in kwargs:
 323             regexp = kwargs["re"]
 324             del kwargs["re"]
 325         try:
 326             func(*args, **kwargs)
 327         except AssertionError, e:
 328             self.fail("assertion fired when it should not have: %s" % e)
 329         except Exception, e:
 330             self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e)
 331         return # we're happy
 332
 333
 334     def test_assert(self):
 335         f = assertutil._assert
 336         self.should_assert(f)
 337         self.should_assert(f, False)
 338         self.should_not_assert(f, True)
 339
 340         m = self.should_assert(f, False, "message")
 341         self.failUnlessEqual(m, "'message' <type 'str'>", m)
 342         m = self.should_assert(f, False, "message1", othermsg=12)
 343         self.failUnlessEqual("'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 344         m = self.should_assert(f, False, othermsg="message2")
 345         self.failUnlessEqual("othermsg: 'message2' <type 'str'>", m)
 346
 347     def test_precondition(self):
 348         f = assertutil.precondition
 349         self.should_assert(f)
 350         self.should_assert(f, False)
 351         self.should_not_assert(f, True)
 352
 353         m = self.should_assert(f, False, "message")
 354         self.failUnlessEqual("precondition: 'message' <type 'str'>", m)
 355         m = self.should_assert(f, False, "message1", othermsg=12)
 356         self.failUnlessEqual("precondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 357         m = self.should_assert(f, False, othermsg="message2")
 358         self.failUnlessEqual("precondition: othermsg: 'message2' <type 'str'>", m)
 359
 360     def test_postcondition(self):
 361         f = assertutil.postcondition
 362         self.should_assert(f)
 363         self.should_assert(f, False)
 364         self.should_not_assert(f, True)
 365
 366         m = self.should_assert(f, False, "message")
 367         self.failUnlessEqual("postcondition: 'message' <type 'str'>", m)
 368         m = self.should_assert(f, False, "message1", othermsg=12)
 369         self.failUnlessEqual("postcondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 370         m = self.should_assert(f, False, othermsg="message2")
 371         self.failUnlessEqual("postcondition: othermsg: 'message2' <type 'str'>", m)
 372
 373 class FileUtil(unittest.TestCase):
 374     def mkdir(self, basedir, path, mode=0777):
 375         fn = os.path.join(basedir, path)
 376         fileutil.make_dirs(fn, mode)
 377
 378     def touch(self, basedir, path, mode=None, data="touch\n"):
 379         fn = os.path.join(basedir, path)
 380         f = open(fn, "w")
 381         f.write(data)
 382         f.close()
 383         if mode is not None:
 384             os.chmod(fn, mode)
 385
 386     def test_rm_dir(self):
 387         basedir = "util/FileUtil/test_rm_dir"
 388         fileutil.make_dirs(basedir)
 389         # create it again to test idempotency
 390         fileutil.make_dirs(basedir)
 391         d = os.path.join(basedir, "doomed")
 392         self.mkdir(d, "a/b")
 393         self.touch(d, "a/b/1.txt")
 394         self.touch(d, "a/b/2.txt", 0444)
 395         self.touch(d, "a/b/3.txt", 0)
 396         self.mkdir(d, "a/c")
 397         self.touch(d, "a/c/1.txt")
 398         self.touch(d, "a/c/2.txt", 0444)
 399         self.touch(d, "a/c/3.txt", 0)
 400         os.chmod(os.path.join(d, "a/c"), 0444)
 401         self.mkdir(d, "a/d")
 402         self.touch(d, "a/d/1.txt")
 403         self.touch(d, "a/d/2.txt", 0444)
 404         self.touch(d, "a/d/3.txt", 0)
 405         os.chmod(os.path.join(d, "a/d"), 0)
 406
 407         fileutil.rm_dir(d)
 408         self.failIf(os.path.exists(d))
 409         # remove it again to test idempotency
 410         fileutil.rm_dir(d)
 411
 412     def test_remove_if_possible(self):
 413         basedir = "util/FileUtil/test_remove_if_possible"
 414         fileutil.make_dirs(basedir)
 415         self.touch(basedir, "here")
 416         fn = os.path.join(basedir, "here")
 417         fileutil.remove_if_possible(fn)
 418         self.failIf(os.path.exists(fn))
 419         fileutil.remove_if_possible(fn) # should be idempotent
 420         fileutil.rm_dir(basedir)
 421         fileutil.remove_if_possible(fn) # should survive errors
 422
 423     def test_open_or_create(self):
 424         basedir = "util/FileUtil/test_open_or_create"
 425         fileutil.make_dirs(basedir)
 426         fn = os.path.join(basedir, "here")
 427         f = fileutil.open_or_create(fn)
 428         f.write("stuff.")
 429         f.close()
 430         f = fileutil.open_or_create(fn)
 431         f.seek(0, 2)
 432         f.write("more.")
 433         f.close()
 434         f = open(fn, "r")
 435         data = f.read()
 436         f.close()
 437         self.failUnlessEqual(data, "stuff.more.")
 438
 439     def test_NamedTemporaryDirectory(self):
 440         basedir = "util/FileUtil/test_NamedTemporaryDirectory"
 441         fileutil.make_dirs(basedir)
 442         td = fileutil.NamedTemporaryDirectory(dir=basedir)
 443         name = td.name
 444         self.failUnless(basedir in name)
 445         self.failUnless(basedir in repr(td))
 446         self.failUnless(os.path.isdir(name))
 447         del td
 448         # it is conceivable that we need to force gc here, but I'm not sure
 449         self.failIf(os.path.isdir(name))
 450
 451     def test_rename(self):
 452         basedir = "util/FileUtil/test_rename"
 453         fileutil.make_dirs(basedir)
 454         self.touch(basedir, "here")
 455         fn = os.path.join(basedir, "here")
 456         fn2 = os.path.join(basedir, "there")
 457         fileutil.rename(fn, fn2)
 458         self.failIf(os.path.exists(fn))
 459         self.failUnless(os.path.exists(fn2))
 460
 461     def test_du(self):
 462         basedir = "util/FileUtil/test_du"
 463         fileutil.make_dirs(basedir)
 464         d = os.path.join(basedir, "space-consuming")
 465         self.mkdir(d, "a/b")
 466         self.touch(d, "a/b/1.txt", data="a"*10)
 467         self.touch(d, "a/b/2.txt", data="b"*11)
 468         self.mkdir(d, "a/c")
 469         self.touch(d, "a/c/1.txt", data="c"*12)
 470         self.touch(d, "a/c/2.txt", data="d"*13)
 471
 472         used = fileutil.du(basedir)
 473         self.failUnlessEqual(10+11+12+13, used)
 474
 475 class PollMixinTests(unittest.TestCase):
 476     def setUp(self):
 477         self.pm = pollmixin.PollMixin()
 478
 479     def test_PollMixin_True(self):
 480         d = self.pm.poll(check_f=lambda : True,
 481                          pollinterval=0.1)
 482         return d
 483
 484     def test_PollMixin_False_then_True(self):
 485         i = iter([False, True])
 486         d = self.pm.poll(check_f=i.next,
 487                          pollinterval=0.1)
 488         return d
 489
 490     def test_timeout(self):
 491         d = self.pm.poll(check_f=lambda: False,
 492                          pollinterval=0.01,
 493                          timeout=1)
 494         def _suc(res):
 495             self.fail("poll should have failed, not returned %s" % (res,))
 496         def _err(f):
 497             f.trap(pollmixin.TimeoutError)
 498             return None # success
 499         d.addCallbacks(_suc, _err)
 500         return d
 501
 502 class DeferredUtilTests(unittest.TestCase):
 503     def test_gather_results(self):
 504         d1 = defer.Deferred()
 505         d2 = defer.Deferred()
 506         res = deferredutil.gatherResults([d1, d2])
 507         d1.errback(ValueError("BAD"))
 508         def _callb(res):
 509             self.fail("Should have errbacked, not resulted in %s" % (res,))
 510         def _errb(thef):
 511             thef.trap(ValueError)
 512         res.addCallbacks(_callb, _errb)
 513         return res
 514
 515     def test_success(self):
 516         d1, d2 = defer.Deferred(), defer.Deferred()
 517         good = []
 518         bad = []
 519         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 520         dlss.addCallbacks(good.append, bad.append)
 521         d1.callback(1)
 522         d2.callback(2)
 523         self.failUnlessEqual(good, [[1,2]])
 524         self.failUnlessEqual(bad, [])
 525
 526     def test_failure(self):
 527         d1, d2 = defer.Deferred(), defer.Deferred()
 528         good = []
 529         bad = []
 530         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 531         dlss.addCallbacks(good.append, bad.append)
 532         d1.addErrback(lambda _ignore: None)
 533         d2.addErrback(lambda _ignore: None)
 534         d1.callback(1)
 535         d2.errback(ValueError())
 536         self.failUnlessEqual(good, [])
 537         self.failUnlessEqual(len(bad), 1)
 538         f = bad[0]
 539         self.failUnless(isinstance(f, Failure))
 540         self.failUnless(f.check(ValueError))
 541
 542 class HashUtilTests(unittest.TestCase):
 543
 544     def test_random_key(self):
 545         k = hashutil.random_key()
 546         self.failUnlessEqual(len(k), hashutil.KEYLEN)
 547
 548     def test_sha256d(self):
 549         h1 = hashutil.tagged_hash("tag1", "value")
 550         h2 = hashutil.tagged_hasher("tag1")
 551         h2.update("value")
 552         h2a = h2.digest()
 553         h2b = h2.digest()
 554         self.failUnlessEqual(h1, h2a)
 555         self.failUnlessEqual(h2a, h2b)
 556
 557     def test_sha256d_truncated(self):
 558         h1 = hashutil.tagged_hash("tag1", "value", 16)
 559         h2 = hashutil.tagged_hasher("tag1", 16)
 560         h2.update("value")
 561         h2 = h2.digest()
 562         self.failUnlessEqual(len(h1), 16)
 563         self.failUnlessEqual(len(h2), 16)
 564         self.failUnlessEqual(h1, h2)
 565
 566     def test_chk(self):
 567         h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret")
 568         h2 = hashutil.convergence_hasher(3, 10, 1000, "secret")
 569         h2.update("data")
 570         h2 = h2.digest()
 571         self.failUnlessEqual(h1, h2)
 572
 573     def test_hashers(self):
 574         h1 = hashutil.block_hash("foo")
 575         h2 = hashutil.block_hasher()
 576         h2.update("foo")
 577         self.failUnlessEqual(h1, h2.digest())
 578
 579         h1 = hashutil.uri_extension_hash("foo")
 580         h2 = hashutil.uri_extension_hasher()
 581         h2.update("foo")
 582         self.failUnlessEqual(h1, h2.digest())
 583
 584         h1 = hashutil.plaintext_hash("foo")
 585         h2 = hashutil.plaintext_hasher()
 586         h2.update("foo")
 587         self.failUnlessEqual(h1, h2.digest())
 588
 589         h1 = hashutil.crypttext_hash("foo")
 590         h2 = hashutil.crypttext_hasher()
 591         h2.update("foo")
 592         self.failUnlessEqual(h1, h2.digest())
 593
 594         h1 = hashutil.crypttext_segment_hash("foo")
 595         h2 = hashutil.crypttext_segment_hasher()
 596         h2.update("foo")
 597         self.failUnlessEqual(h1, h2.digest())
 598
 599         h1 = hashutil.plaintext_segment_hash("foo")
 600         h2 = hashutil.plaintext_segment_hasher()
 601         h2.update("foo")
 602         self.failUnlessEqual(h1, h2.digest())
 603
 604     def test_constant_time_compare(self):
 605         self.failUnless(hashutil.constant_time_compare("a", "a"))
 606         self.failUnless(hashutil.constant_time_compare("ab", "ab"))
 607         self.failIf(hashutil.constant_time_compare("a", "b"))
 608         self.failIf(hashutil.constant_time_compare("a", "aa"))
 609
 610 class Abbreviate(unittest.TestCase):
 611     def test_time(self):
 612         a = abbreviate.abbreviate_time
 613         self.failUnlessEqual(a(None), "unknown")
 614         self.failUnlessEqual(a(0), "0 seconds")
 615         self.failUnlessEqual(a(1), "1 second")
 616         self.failUnlessEqual(a(2), "2 seconds")
 617         self.failUnlessEqual(a(119), "119 seconds")
 618         MIN = 60
 619         self.failUnlessEqual(a(2*MIN), "2 minutes")
 620         self.failUnlessEqual(a(60*MIN), "60 minutes")
 621         self.failUnlessEqual(a(179*MIN), "179 minutes")
 622         HOUR = 60*MIN
 623         self.failUnlessEqual(a(180*MIN), "3 hours")
 624         self.failUnlessEqual(a(4*HOUR), "4 hours")
 625         DAY = 24*HOUR
 626         MONTH = 30*DAY
 627         self.failUnlessEqual(a(2*DAY), "2 days")
 628         self.failUnlessEqual(a(2*MONTH), "2 months")
 629         YEAR = 365*DAY
 630         self.failUnlessEqual(a(5*YEAR), "5 years")
 631
 632     def test_space(self):
 633         tests_si = [(None, "unknown"),
 634                     (0, "0 B"),
 635                     (1, "1 B"),
 636                     (999, "999 B"),
 637                     (1000, "1000 B"),
 638                     (1023, "1023 B"),
 639                     (1024, "1.02 kB"),
 640                     (20*1000, "20.00 kB"),
 641                     (1024*1024, "1.05 MB"),
 642                     (1000*1000, "1.00 MB"),
 643                     (1000*1000*1000, "1.00 GB"),
 644                     (1000*1000*1000*1000, "1.00 TB"),
 645                     (1000*1000*1000*1000*1000, "1.00 PB"),
 646                     (1234567890123456, "1.23 PB"),
 647                     ]
 648         for (x, expected) in tests_si:
 649             got = abbreviate.abbreviate_space(x, SI=True)
 650             self.failUnlessEqual(got, expected)
 651
 652         tests_base1024 = [(None, "unknown"),
 653                           (0, "0 B"),
 654                           (1, "1 B"),
 655                           (999, "999 B"),
 656                           (1000, "1000 B"),
 657                           (1023, "1023 B"),
 658                           (1024, "1.00 kiB"),
 659                           (20*1024, "20.00 kiB"),
 660                           (1000*1000, "976.56 kiB"),
 661                           (1024*1024, "1.00 MiB"),
 662                           (1024*1024*1024, "1.00 GiB"),
 663                           (1024*1024*1024*1024, "1.00 TiB"),
 664                           (1000*1000*1000*1000*1000, "909.49 TiB"),
 665                           (1024*1024*1024*1024*1024, "1.00 PiB"),
 666                           (1234567890123456, "1.10 PiB"),
 667                     ]
 668         for (x, expected) in tests_base1024:
 669             got = abbreviate.abbreviate_space(x, SI=False)
 670             self.failUnlessEqual(got, expected)
 671
 672         self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567),
 673                              "(1.23 MB, 1.18 MiB)")
 674
 675     def test_parse_space(self):
 676         p = abbreviate.parse_abbreviated_size
 677         self.failUnlessEqual(p(""), None)
 678         self.failUnlessEqual(p(None), None)
 679         self.failUnlessEqual(p("123"), 123)
 680         self.failUnlessEqual(p("123B"), 123)
 681         self.failUnlessEqual(p("2K"), 2000)
 682         self.failUnlessEqual(p("2kb"), 2000)
 683         self.failUnlessEqual(p("2KiB"), 2048)
 684         self.failUnlessEqual(p("10MB"), 10*1000*1000)
 685         self.failUnlessEqual(p("10MiB"), 10*1024*1024)
 686         self.failUnlessEqual(p("5G"), 5*1000*1000*1000)
 687         self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024)
 688         e = self.failUnlessRaises(ValueError, p, "12 cubits")
 689         self.failUnless("12 cubits" in str(e))
 690
 691 class Limiter(unittest.TestCase):
 692     timeout = 480 # This takes longer than 240 seconds on Francois's arm box.
 693
 694     def job(self, i, foo):
 695         self.calls.append( (i, foo) )
 696         self.simultaneous += 1
 697         self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous)
 698         d = defer.Deferred()
 699         def _done():
 700             self.simultaneous -= 1
 701             d.callback("done %d" % i)
 702         reactor.callLater(1.0, _done)
 703         return d
 704
 705     def bad_job(self, i, foo):
 706         raise ValueError("bad_job %d" % i)
 707
 708     def test_limiter(self):
 709         self.calls = []
 710         self.simultaneous = 0
 711         self.peak_simultaneous = 0
 712         l = limiter.ConcurrencyLimiter()
 713         dl = []
 714         for i in range(20):
 715             dl.append(l.add(self.job, i, foo=str(i)))
 716         d = defer.DeferredList(dl, fireOnOneErrback=True)
 717         def _done(res):
 718             self.failUnlessEqual(self.simultaneous, 0)
 719             self.failUnless(self.peak_simultaneous <= 10)
 720             self.failUnlessEqual(len(self.calls), 20)
 721             for i in range(20):
 722                 self.failUnless( (i, str(i)) in self.calls)
 723         d.addCallback(_done)
 724         return d
 725
 726     def test_errors(self):
 727         self.calls = []
 728         self.simultaneous = 0
 729         self.peak_simultaneous = 0
 730         l = limiter.ConcurrencyLimiter()
 731         dl = []
 732         for i in range(20):
 733             dl.append(l.add(self.job, i, foo=str(i)))
 734         d2 = l.add(self.bad_job, 21, "21")
 735         d = defer.DeferredList(dl, fireOnOneErrback=True)
 736         def _most_done(res):
 737             results = []
 738             for (success, result) in res:
 739                 self.failUnlessEqual(success, True)
 740                 results.append(result)
 741             results.sort()
 742             expected_results = ["done %d" % i for i in range(20)]
 743             expected_results.sort()
 744             self.failUnlessEqual(results, expected_results)
 745             self.failUnless(self.peak_simultaneous <= 10)
 746             self.failUnlessEqual(len(self.calls), 20)
 747             for i in range(20):
 748                 self.failUnless( (i, str(i)) in self.calls)
 749             def _good(res):
 750                 self.fail("should have failed, not got %s" % (res,))
 751             def _err(f):
 752                 f.trap(ValueError)
 753                 self.failUnless("bad_job 21" in str(f))
 754             d2.addCallbacks(_good, _err)
 755             return d2
 756         d.addCallback(_most_done)
 757         def _all_done(res):
 758             self.failUnlessEqual(self.simultaneous, 0)
 759             self.failUnless(self.peak_simultaneous <= 10)
 760             self.failUnlessEqual(len(self.calls), 20)
 761             for i in range(20):
 762                 self.failUnless( (i, str(i)) in self.calls)
 763         d.addCallback(_all_done)
 764         return d
 765
 766 class TimeFormat(unittest.TestCase):
 767     def test_epoch(self):
 768         return self._help_test_epoch()
 769
 770     def test_epoch_in_London(self):
 771         # Europe/London is a particularly troublesome timezone.  Nowadays, its
 772         # offset from GMT is 0.  But in 1970, its offset from GMT was 1.
 773         # (Apparently in 1970 Britain had redefined standard time to be GMT+1
 774         # and stayed in standard time all year round, whereas today
 775         # Europe/London standard time is GMT and Europe/London Daylight
 776         # Savings Time is GMT+1.)  The current implementation of
 777         # time_format.iso_utc_time_to_localseconds() breaks if the timezone is
 778         # Europe/London.  (As soon as this unit test is done then I'll change
 779         # that implementation to something that works even in this case...)
 780         origtz = os.environ.get('TZ')
 781         os.environ['TZ'] = "Europe/London"
 782         if hasattr(time, 'tzset'):
 783             time.tzset()
 784         try:
 785             return self._help_test_epoch()
 786         finally:
 787             if origtz is None:
 788                 del os.environ['TZ']
 789             else:
 790                 os.environ['TZ'] = origtz
 791             if hasattr(time, 'tzset'):
 792                 time.tzset()
 793
 794     def _help_test_epoch(self):
 795         origtzname = time.tzname
 796         s = time_format.iso_utc_time_to_seconds("1970-01-01T00:00:01")
 797         self.failUnlessEqual(s, 1.0)
 798         s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01")
 799         self.failUnlessEqual(s, 1.0)
 800         s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01")
 801         self.failUnlessEqual(s, 1.0)
 802
 803         self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01")
 804         self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "),
 805                              "1970-01-01 00:00:01")
 806
 807         now = time.time()
 808         isostr = time_format.iso_utc(now)
 809         timestamp = time_format.iso_utc_time_to_seconds(isostr)
 810         self.failUnlessEqual(int(timestamp), int(now))
 811
 812         def my_time():
 813             return 1.0
 814         self.failUnlessEqual(time_format.iso_utc(t=my_time),
 815                              "1970-01-01_00:00:01")
 816         e = self.failUnlessRaises(ValueError,
 817                                   time_format.iso_utc_time_to_seconds,
 818                                   "invalid timestring")
 819         self.failUnless("not a complete ISO8601 timestamp" in str(e))
 820         s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01.500")
 821         self.failUnlessEqual(s, 1.5)
 822
 823         # Look for daylight-savings-related errors.
 824         thatmomentinmarch = time_format.iso_utc_time_to_seconds("2009-03-20 21:49:02.226536")
 825         self.failUnlessEqual(thatmomentinmarch, 1237585742.226536)
 826         self.failUnlessEqual(origtzname, time.tzname)
 827
 828 class CacheDir(unittest.TestCase):
 829     def test_basic(self):
 830         basedir = "test_util/CacheDir/test_basic"
 831
 832         def _failIfExists(name):
 833             absfn = os.path.join(basedir, name)
 834             self.failIf(os.path.exists(absfn),
 835                         "%s exists but it shouldn't" % absfn)
 836
 837         def _failUnlessExists(name):
 838             absfn = os.path.join(basedir, name)
 839             self.failUnless(os.path.exists(absfn),
 840                             "%s doesn't exist but it should" % absfn)
 841
 842         cdm = cachedir.CacheDirectoryManager(basedir)
 843         a = cdm.get_file("a")
 844         b = cdm.get_file("b")
 845         c = cdm.get_file("c")
 846         f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f
 847         f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f
 848         f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f
 849
 850         _failUnlessExists("a")
 851         _failUnlessExists("b")
 852         _failUnlessExists("c")
 853
 854         cdm.check()
 855
 856         _failUnlessExists("a")
 857         _failUnlessExists("b")
 858         _failUnlessExists("c")
 859
 860         del a
 861         # this file won't be deleted yet, because it isn't old enough
 862         cdm.check()
 863         _failUnlessExists("a")
 864         _failUnlessExists("b")
 865         _failUnlessExists("c")
 866
 867         # we change the definition of "old" to make everything old
 868         cdm.old = -10
 869
 870         cdm.check()
 871         _failIfExists("a")
 872         _failUnlessExists("b")
 873         _failUnlessExists("c")
 874
 875         cdm.old = 60*60
 876
 877         del b
 878
 879         cdm.check()
 880         _failIfExists("a")
 881         _failUnlessExists("b")
 882         _failUnlessExists("c")
 883
 884         b2 = cdm.get_file("b")
 885
 886         cdm.check()
 887         _failIfExists("a")
 888         _failUnlessExists("b")
 889         _failUnlessExists("c")
 890
 891 ctr = [0]
 892 class EqButNotIs:
 893     def __init__(self, x):
 894         self.x = x
 895         self.hash = ctr[0]
 896         ctr[0] += 1
 897     def __repr__(self):
 898         return "<%s %s>" % (self.__class__.__name__, self.x,)
 899     def __hash__(self):
 900         return self.hash
 901     def __le__(self, other):
 902         return self.x <= other
 903     def __lt__(self, other):
 904         return self.x < other
 905     def __ge__(self, other):
 906         return self.x >= other
 907     def __gt__(self, other):
 908         return self.x > other
 909     def __ne__(self, other):
 910         return self.x != other
 911     def __eq__(self, other):
 912         return self.x == other
 913
 914 class DictUtil(unittest.TestCase):
 915     def _help_test_empty_dict(self, klass):
 916         d1 = klass()
 917         d2 = klass({})
 918
 919         self.failUnless(d1 == d2, "d1: %r, d2: %r" % (d1, d2,))
 920         self.failUnless(len(d1) == 0)
 921         self.failUnless(len(d2) == 0)
 922
 923     def _help_test_nonempty_dict(self, klass):
 924         d1 = klass({'a': 1, 'b': "eggs", 3: "spam",})
 925         d2 = klass({'a': 1, 'b': "eggs", 3: "spam",})
 926
 927         self.failUnless(d1 == d2)
 928         self.failUnless(len(d1) == 3, "%s, %s" % (len(d1), d1,))
 929         self.failUnless(len(d2) == 3)
 930
 931     def _help_test_eq_but_notis(self, klass):
 932         d = klass({'a': 3, 'b': EqButNotIs(3), 'c': 3})
 933         d.pop('b')
 934
 935         d.clear()
 936         d['a'] = 3
 937         d['b'] = EqButNotIs(3)
 938         d['c'] = 3
 939         d.pop('b')
 940
 941         d.clear()
 942         d['b'] = EqButNotIs(3)
 943         d['a'] = 3
 944         d['c'] = 3
 945         d.pop('b')
 946
 947         d.clear()
 948         d['a'] = EqButNotIs(3)
 949         d['c'] = 3
 950         d['a'] = 3
 951
 952         d.clear()
 953         fake3 = EqButNotIs(3)
 954         fake7 = EqButNotIs(7)
 955         d[fake3] = fake7
 956         d[3] = 7
 957         d[3] = 8
 958         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
 959         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
 960         # The real 7 should have been ejected by the d[3] = 8.
 961         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
 962         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
 963         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
 964         d[fake3] = 8
 965
 966         d.clear()
 967         d[3] = 7
 968         fake3 = EqButNotIs(3)
 969         fake7 = EqButNotIs(7)
 970         d[fake3] = fake7
 971         d[3] = 8
 972         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
 973         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
 974         # The real 7 should have been ejected by the d[3] = 8.
 975         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
 976         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
 977         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
 978         d[fake3] = 8
 979
 980     def test_all(self):
 981         self._help_test_eq_but_notis(dictutil.UtilDict)
 982         self._help_test_eq_but_notis(dictutil.NumDict)
 983         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
 984         self._help_test_nonempty_dict(dictutil.UtilDict)
 985         self._help_test_nonempty_dict(dictutil.NumDict)
 986         self._help_test_nonempty_dict(dictutil.ValueOrderedDict)
 987         self._help_test_eq_but_notis(dictutil.UtilDict)
 988         self._help_test_eq_but_notis(dictutil.NumDict)
 989         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
 990
 991     def test_dict_of_sets(self):
 992         ds = dictutil.DictOfSets()
 993         ds.add(1, "a")
 994         ds.add(2, "b")
 995         ds.add(2, "b")
 996         ds.add(2, "c")
 997         self.failUnlessEqual(ds[1], set(["a"]))
 998         self.failUnlessEqual(ds[2], set(["b", "c"]))
 999         ds.discard(3, "d") # should not raise an exception
1000         ds.discard(2, "b")
1001         self.failUnlessEqual(ds[2], set(["c"]))
1002         ds.discard(2, "c")
1003         self.failIf(2 in ds)
1004
1005         ds.union(1, ["a", "e"])
1006         ds.union(3, ["f"])
1007         self.failUnlessEqual(ds[1], set(["a","e"]))
1008         self.failUnlessEqual(ds[3], set(["f"]))
1009         ds2 = dictutil.DictOfSets()
1010         ds2.add(3, "f")
1011         ds2.add(3, "g")
1012         ds2.add(4, "h")
1013         ds.update(ds2)
1014         self.failUnlessEqual(ds[1], set(["a","e"]))
1015         self.failUnlessEqual(ds[3], set(["f", "g"]))
1016         self.failUnlessEqual(ds[4], set(["h"]))
1017
1018     def test_move(self):
1019         d1 = {1: "a", 2: "b"}
1020         d2 = {2: "c", 3: "d"}
1021         dictutil.move(1, d1, d2)
1022         self.failUnlessEqual(d1, {2: "b"})
1023         self.failUnlessEqual(d2, {1: "a", 2: "c", 3: "d"})
1024
1025         d1 = {1: "a", 2: "b"}
1026         d2 = {2: "c", 3: "d"}
1027         dictutil.move(2, d1, d2)
1028         self.failUnlessEqual(d1, {1: "a"})
1029         self.failUnlessEqual(d2, {2: "b", 3: "d"})
1030
1031         d1 = {1: "a", 2: "b"}
1032         d2 = {2: "c", 3: "d"}
1033         self.failUnlessRaises(KeyError, dictutil.move, 5, d1, d2, strict=True)
1034
1035     def test_subtract(self):
1036         d1 = {1: "a", 2: "b"}
1037         d2 = {2: "c", 3: "d"}
1038         d3 = dictutil.subtract(d1, d2)
1039         self.failUnlessEqual(d3, {1: "a"})
1040
1041         d1 = {1: "a", 2: "b"}
1042         d2 = {2: "c"}
1043         d3 = dictutil.subtract(d1, d2)
1044         self.failUnlessEqual(d3, {1: "a"})
1045
1046     def test_utildict(self):
1047         d = dictutil.UtilDict({1: "a", 2: "b"})
1048         d.del_if_present(1)
1049         d.del_if_present(3)
1050         self.failUnlessEqual(d, {2: "b"})
1051         def eq(a, b):
1052             return a == b
1053         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1054
1055         d = dictutil.UtilDict({1: "b", 2: "a"})
1056         self.failUnlessEqual(d.items_sorted_by_value(),
1057                              [(2, "a"), (1, "b")])
1058         self.failUnlessEqual(d.items_sorted_by_key(),
1059                              [(1, "b"), (2, "a")])
1060         self.failUnlessEqual(repr(d), "{1: 'b', 2: 'a'}")
1061         self.failUnless(1 in d)
1062
1063         d2 = dictutil.UtilDict({3: "c", 4: "d"})
1064         self.failUnless(d != d2)
1065         self.failUnless(d2 > d)
1066         self.failUnless(d2 >= d)
1067         self.failUnless(d <= d2)
1068         self.failUnless(d < d2)
1069         self.failUnlessEqual(d[1], "b")
1070         self.failUnlessEqual(sorted(list([k for k in d])), [1,2])
1071
1072         d3 = d.copy()
1073         self.failUnlessEqual(d, d3)
1074         self.failUnless(isinstance(d3, dictutil.UtilDict))
1075
1076         d4 = d.fromkeys([3,4], "e")
1077         self.failUnlessEqual(d4, {3: "e", 4: "e"})
1078
1079         self.failUnlessEqual(d.get(1), "b")
1080         self.failUnlessEqual(d.get(3), None)
1081         self.failUnlessEqual(d.get(3, "default"), "default")
1082         self.failUnlessEqual(sorted(list(d.items())),
1083                              [(1, "b"), (2, "a")])
1084         self.failUnlessEqual(sorted(list(d.iteritems())),
1085                              [(1, "b"), (2, "a")])
1086         self.failUnlessEqual(sorted(d.keys()), [1, 2])
1087         self.failUnlessEqual(sorted(d.values()), ["a", "b"])
1088         x = d.setdefault(1, "new")
1089         self.failUnlessEqual(x, "b")
1090         self.failUnlessEqual(d[1], "b")
1091         x = d.setdefault(3, "new")
1092         self.failUnlessEqual(x, "new")
1093         self.failUnlessEqual(d[3], "new")
1094         del d[3]
1095
1096         x = d.popitem()
1097         self.failUnless(x in [(1, "b"), (2, "a")])
1098         x = d.popitem()
1099         self.failUnless(x in [(1, "b"), (2, "a")])
1100         self.failUnlessRaises(KeyError, d.popitem)
1101
1102     def test_numdict(self):
1103         d = dictutil.NumDict({"a": 1, "b": 2})
1104
1105         d.add_num("a", 10, 5)
1106         d.add_num("c", 20, 5)
1107         d.add_num("d", 30)
1108         self.failUnlessEqual(d, {"a": 11, "b": 2, "c": 25, "d": 30})
1109
1110         d.subtract_num("a", 10)
1111         d.subtract_num("e", 10)
1112         d.subtract_num("f", 10, 15)
1113         self.failUnlessEqual(d, {"a": 1, "b": 2, "c": 25, "d": 30,
1114                                  "e": -10, "f": 5})
1115
1116         self.failUnlessEqual(d.sum(), sum([1, 2, 25, 30, -10, 5]))
1117
1118         d = dictutil.NumDict()
1119         d.inc("a")
1120         d.inc("a")
1121         d.inc("b", 5)
1122         self.failUnlessEqual(d, {"a": 2, "b": 6})
1123         d.dec("a")
1124         d.dec("c")
1125         d.dec("d", 5)
1126         self.failUnlessEqual(d, {"a": 1, "b": 6, "c": -1, "d": 4})
1127         self.failUnlessEqual(d.items_sorted_by_key(),
1128                              [("a", 1), ("b", 6), ("c", -1), ("d", 4)])
1129         self.failUnlessEqual(d.items_sorted_by_value(),
1130                              [("c", -1), ("a", 1), ("d", 4), ("b", 6)])
1131         self.failUnlessEqual(d.item_with_largest_value(), ("b", 6))
1132
1133         d = dictutil.NumDict({"a": 1, "b": 2})
1134         self.failUnlessEqual(repr(d), "{'a': 1, 'b': 2}")
1135         self.failUnless("a" in d)
1136
1137         d2 = dictutil.NumDict({"c": 3, "d": 4})
1138         self.failUnless(d != d2)
1139         self.failUnless(d2 > d)
1140         self.failUnless(d2 >= d)
1141         self.failUnless(d <= d2)
1142         self.failUnless(d < d2)
1143         self.failUnlessEqual(d["a"], 1)
1144         self.failUnlessEqual(sorted(list([k for k in d])), ["a","b"])
1145         def eq(a, b):
1146             return a == b
1147         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1148
1149         d3 = d.copy()
1150         self.failUnlessEqual(d, d3)
1151         self.failUnless(isinstance(d3, dictutil.NumDict))
1152
1153         d4 = d.fromkeys(["a","b"], 5)
1154         self.failUnlessEqual(d4, {"a": 5, "b": 5})
1155
1156         self.failUnlessEqual(d.get("a"), 1)
1157         self.failUnlessEqual(d.get("c"), 0)
1158         self.failUnlessEqual(d.get("c", 5), 5)
1159         self.failUnlessEqual(sorted(list(d.items())),
1160                              [("a", 1), ("b", 2)])
1161         self.failUnlessEqual(sorted(list(d.iteritems())),
1162                              [("a", 1), ("b", 2)])
1163         self.failUnlessEqual(sorted(d.keys()), ["a", "b"])
1164         self.failUnlessEqual(sorted(d.values()), [1, 2])
1165         self.failUnless(d.has_key("a"))
1166         self.failIf(d.has_key("c"))
1167
1168         x = d.setdefault("c", 3)
1169         self.failUnlessEqual(x, 3)
1170         self.failUnlessEqual(d["c"], 3)
1171         x = d.setdefault("c", 5)
1172         self.failUnlessEqual(x, 3)
1173         self.failUnlessEqual(d["c"], 3)
1174         del d["c"]
1175
1176         x = d.popitem()
1177         self.failUnless(x in [("a", 1), ("b", 2)])
1178         x = d.popitem()
1179         self.failUnless(x in [("a", 1), ("b", 2)])
1180         self.failUnlessRaises(KeyError, d.popitem)
1181
1182         d.update({"c": 3})
1183         d.update({"c": 4, "d": 5})
1184         self.failUnlessEqual(d, {"c": 4, "d": 5})
1185
1186     def test_del_if_present(self):
1187         d = {1: "a", 2: "b"}
1188         dictutil.del_if_present(d, 1)
1189         dictutil.del_if_present(d, 3)
1190         self.failUnlessEqual(d, {2: "b"})
1191
1192     def test_valueordereddict(self):
1193         d = dictutil.ValueOrderedDict()
1194         d["a"] = 3
1195         d["b"] = 2
1196         d["c"] = 1
1197
1198         self.failUnlessEqual(d, {"a": 3, "b": 2, "c": 1})
1199         self.failUnlessEqual(d.items(), [("c", 1), ("b", 2), ("a", 3)])
1200         self.failUnlessEqual(d.values(), [1, 2, 3])
1201         self.failUnlessEqual(d.keys(), ["c", "b", "a"])
1202         self.failUnlessEqual(repr(d), "<ValueOrderedDict {c: 1, b: 2, a: 3}>")
1203         def eq(a, b):
1204             return a == b
1205         self.failIf(d == {"a": 4})
1206         self.failUnless(d != {"a": 4})
1207
1208         x = d.setdefault("d", 0)
1209         self.failUnlessEqual(x, 0)
1210         self.failUnlessEqual(d["d"], 0)
1211         x = d.setdefault("d", -1)
1212         self.failUnlessEqual(x, 0)
1213         self.failUnlessEqual(d["d"], 0)
1214
1215         x = d.remove("e", "default", False)
1216         self.failUnlessEqual(x, "default")
1217         self.failUnlessRaises(KeyError, d.remove, "e", "default", True)
1218         x = d.remove("d", 5)
1219         self.failUnlessEqual(x, 0)
1220
1221         x = d.__getitem__("c")
1222         self.failUnlessEqual(x, 1)
1223         x = d.__getitem__("e", "default", False)
1224         self.failUnlessEqual(x, "default")
1225         self.failUnlessRaises(KeyError, d.__getitem__, "e", "default", True)
1226
1227         self.failUnlessEqual(d.popitem(), ("c", 1))
1228         self.failUnlessEqual(d.popitem(), ("b", 2))
1229         self.failUnlessEqual(d.popitem(), ("a", 3))
1230         self.failUnlessRaises(KeyError, d.popitem)
1231
1232         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1233         x = d.pop("d", "default", False)
1234         self.failUnlessEqual(x, "default")
1235         self.failUnlessRaises(KeyError, d.pop, "d", "default", True)
1236         x = d.pop("b")
1237         self.failUnlessEqual(x, 2)
1238         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1239
1240         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1241         x = d.pop_from_list(1) # pop the second item, b/2
1242         self.failUnlessEqual(x, "b")
1243         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1244
1245 class Pipeline(unittest.TestCase):
1246     def pause(self, *args, **kwargs):
1247         d = defer.Deferred()
1248         self.calls.append( (d, args, kwargs) )
1249         return d
1250
1251     def failUnlessCallsAre(self, expected):
1252         #print self.calls
1253         #print expected
1254         self.failUnlessEqual(len(self.calls), len(expected), self.calls)
1255         for i,c in enumerate(self.calls):
1256             self.failUnlessEqual(c[1:], expected[i], str(i))
1257
1258     def test_basic(self):
1259         self.calls = []
1260         finished = []
1261         p = pipeline.Pipeline(100)
1262
1263         d = p.flush() # fires immediately
1264         d.addCallbacks(finished.append, log.err)
1265         self.failUnlessEqual(len(finished), 1)
1266         finished = []
1267
1268         d = p.add(10, self.pause, "one")
1269         # the call should start right away, and our return Deferred should
1270         # fire right away
1271         d.addCallbacks(finished.append, log.err)
1272         self.failUnlessEqual(len(finished), 1)
1273         self.failUnlessEqual(finished[0], None)
1274         self.failUnlessCallsAre([ ( ("one",) , {} ) ])
1275         self.failUnlessEqual(p.gauge, 10)
1276
1277         # pipeline: [one]
1278
1279         finished = []
1280         d = p.add(20, self.pause, "two", kw=2)
1281         # pipeline: [one, two]
1282
1283         # the call and the Deferred should fire right away
1284         d.addCallbacks(finished.append, log.err)
1285         self.failUnlessEqual(len(finished), 1)
1286         self.failUnlessEqual(finished[0], None)
1287         self.failUnlessCallsAre([ ( ("one",) , {} ),
1288                                   ( ("two",) , {"kw": 2} ),
1289                                   ])
1290         self.failUnlessEqual(p.gauge, 30)
1291
1292         self.calls[0][0].callback("one-result")
1293         # pipeline: [two]
1294         self.failUnlessEqual(p.gauge, 20)
1295
1296         finished = []
1297         d = p.add(90, self.pause, "three", "posarg1")
1298         # pipeline: [two, three]
1299         flushed = []
1300         fd = p.flush()
1301         fd.addCallbacks(flushed.append, log.err)
1302         self.failUnlessEqual(flushed, [])
1303
1304         # the call will be made right away, but the return Deferred will not,
1305         # because the pipeline is now full.
1306         d.addCallbacks(finished.append, log.err)
1307         self.failUnlessEqual(len(finished), 0)
1308         self.failUnlessCallsAre([ ( ("one",) , {} ),
1309                                   ( ("two",) , {"kw": 2} ),
1310                                   ( ("three", "posarg1"), {} ),
1311                                   ])
1312         self.failUnlessEqual(p.gauge, 110)
1313
1314         self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause)
1315
1316         # retiring either call will unblock the pipeline, causing the #3
1317         # Deferred to fire
1318         self.calls[2][0].callback("three-result")
1319         # pipeline: [two]
1320
1321         self.failUnlessEqual(len(finished), 1)
1322         self.failUnlessEqual(finished[0], None)
1323         self.failUnlessEqual(flushed, [])
1324
1325         # retiring call#2 will finally allow the flush() Deferred to fire
1326         self.calls[1][0].callback("two-result")
1327         self.failUnlessEqual(len(flushed), 1)
1328
1329     def test_errors(self):
1330         self.calls = []
1331         p = pipeline.Pipeline(100)
1332
1333         d1 = p.add(200, self.pause, "one")
1334         d2 = p.flush()
1335
1336         finished = []
1337         d1.addBoth(finished.append)
1338         self.failUnlessEqual(finished, [])
1339
1340         flushed = []
1341         d2.addBoth(flushed.append)
1342         self.failUnlessEqual(flushed, [])
1343
1344         self.calls[0][0].errback(ValueError("oops"))
1345
1346         self.failUnlessEqual(len(finished), 1)
1347         f = finished[0]
1348         self.failUnless(isinstance(f, Failure))
1349         self.failUnless(f.check(pipeline.PipelineError))
1350         r = repr(f.value)
1351         self.failUnless("ValueError" in r, r)
1352         f2 = f.value.error
1353         self.failUnless(f2.check(ValueError))
1354
1355         self.failUnlessEqual(len(flushed), 1)
1356         f = flushed[0]
1357         self.failUnless(isinstance(f, Failure))
1358         self.failUnless(f.check(pipeline.PipelineError))
1359         f2 = f.value.error
1360         self.failUnless(f2.check(ValueError))
1361
1362         # now that the pipeline is in the failed state, any new calls will
1363         # fail immediately
1364
1365         d3 = p.add(20, self.pause, "two")
1366
1367         finished = []
1368         d3.addBoth(finished.append)
1369         self.failUnlessEqual(len(finished), 1)
1370         f = finished[0]
1371         self.failUnless(isinstance(f, Failure))
1372         self.failUnless(f.check(pipeline.PipelineError))
1373         r = repr(f.value)
1374         self.failUnless("ValueError" in r, r)
1375         f2 = f.value.error
1376         self.failUnless(f2.check(ValueError))
1377
1378         d4 = p.flush()
1379         flushed = []
1380         d4.addBoth(flushed.append)
1381         self.failUnlessEqual(len(flushed), 1)
1382         f = flushed[0]
1383         self.failUnless(isinstance(f, Failure))
1384         self.failUnless(f.check(pipeline.PipelineError))
1385         f2 = f.value.error
1386         self.failUnless(f2.check(ValueError))
1387
1388     def test_errors2(self):
1389         self.calls = []
1390         p = pipeline.Pipeline(100)
1391
1392         d1 = p.add(10, self.pause, "one")
1393         d2 = p.add(20, self.pause, "two")
1394         d3 = p.add(30, self.pause, "three")
1395         d4 = p.flush()
1396
1397         # one call fails, then the second one succeeds: make sure
1398         # ExpandableDeferredList tolerates the second one
1399
1400         flushed = []
1401         d4.addBoth(flushed.append)
1402         self.failUnlessEqual(flushed, [])
1403
1404         self.calls[0][0].errback(ValueError("oops"))
1405         self.failUnlessEqual(len(flushed), 1)
1406         f = flushed[0]
1407         self.failUnless(isinstance(f, Failure))
1408         self.failUnless(f.check(pipeline.PipelineError))
1409         f2 = f.value.error
1410         self.failUnless(f2.check(ValueError))
1411
1412         self.calls[1][0].callback("two-result")
1413         self.calls[2][0].errback(ValueError("three-error"))