src/allmydata/test/test_util.py

   1
   2 def foo(): pass # keep the line number constant
   3
   4 import os, time
   5 from StringIO import StringIO
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.python.failure import Failure
   9
  10 from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
  11 from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
  12 from allmydata.util import limiter, time_format, pollmixin, cachedir
  13 from allmydata.util import statistics, dictutil, rrefutil
  14 from allmydata.util.rrefutil import ServerFailure
  15
  16 class Base32(unittest.TestCase):
  17     def test_b2a_matches_Pythons(self):
  18         import base64
  19         y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0"
  20         x = base64.b32encode(y)
  21         while x and x[-1] == '=':
  22             x = x[:-1]
  23         x = x.lower()
  24         self.failUnlessEqual(base32.b2a(y), x)
  25     def test_b2a(self):
  26         self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a")
  27     def test_b2a_or_none(self):
  28         self.failUnlessEqual(base32.b2a_or_none(None), None)
  29         self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a")
  30     def test_a2b(self):
  31         self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34")
  32         self.failUnlessRaises(AssertionError, base32.a2b, "b0gus")
  33
  34 class IDLib(unittest.TestCase):
  35     def test_nodeid_b2a(self):
  36         self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
  37
  38 class NoArgumentException(Exception):
  39     def __init__(self):
  40         pass
  41
  42 class HumanReadable(unittest.TestCase):
  43     def test_repr(self):
  44         hr = humanreadable.hr
  45         self.failUnlessEqual(hr(foo), "<foo() at test_util.py:2>")
  46         self.failUnlessEqual(hr(self.test_repr),
  47                              "<bound method HumanReadable.test_repr of <allmydata.test.test_util.HumanReadable testMethod=test_repr>>")
  48         self.failUnlessEqual(hr(1L), "1")
  49         self.failUnlessEqual(hr(10**40),
  50                              "100000000000000000...000000000000000000")
  51         self.failUnlessEqual(hr(self), "<allmydata.test.test_util.HumanReadable testMethod=test_repr>")
  52         self.failUnlessEqual(hr([1,2]), "[1, 2]")
  53         self.failUnlessEqual(hr({1:2}), "{1:2}")
  54         try:
  55             raise ValueError
  56         except Exception, e:
  57             self.failUnless(
  58                 hr(e) == "<ValueError: ()>" # python-2.4
  59                 or hr(e) == "ValueError()") # python-2.5
  60         try:
  61             raise ValueError("oops")
  62         except Exception, e:
  63             self.failUnless(
  64                 hr(e) == "<ValueError: 'oops'>" # python-2.4
  65                 or hr(e) == "ValueError('oops',)") # python-2.5
  66         try:
  67             raise NoArgumentException
  68         except Exception, e:
  69             self.failUnless(
  70                 hr(e) == "<NoArgumentException>" # python-2.4
  71                 or hr(e) == "NoArgumentException()") # python-2.5
  72
  73
  74 class MyList(list):
  75     pass
  76
  77 class Math(unittest.TestCase):
  78     def test_div_ceil(self):
  79         f = mathutil.div_ceil
  80         self.failUnlessEqual(f(0, 1), 0)
  81         self.failUnlessEqual(f(0, 2), 0)
  82         self.failUnlessEqual(f(0, 3), 0)
  83         self.failUnlessEqual(f(1, 3), 1)
  84         self.failUnlessEqual(f(2, 3), 1)
  85         self.failUnlessEqual(f(3, 3), 1)
  86         self.failUnlessEqual(f(4, 3), 2)
  87         self.failUnlessEqual(f(5, 3), 2)
  88         self.failUnlessEqual(f(6, 3), 2)
  89         self.failUnlessEqual(f(7, 3), 3)
  90
  91     def test_next_multiple(self):
  92         f = mathutil.next_multiple
  93         self.failUnlessEqual(f(5, 1), 5)
  94         self.failUnlessEqual(f(5, 2), 6)
  95         self.failUnlessEqual(f(5, 3), 6)
  96         self.failUnlessEqual(f(5, 4), 8)
  97         self.failUnlessEqual(f(5, 5), 5)
  98         self.failUnlessEqual(f(5, 6), 6)
  99         self.failUnlessEqual(f(32, 1), 32)
 100         self.failUnlessEqual(f(32, 2), 32)
 101         self.failUnlessEqual(f(32, 3), 33)
 102         self.failUnlessEqual(f(32, 4), 32)
 103         self.failUnlessEqual(f(32, 5), 35)
 104         self.failUnlessEqual(f(32, 6), 36)
 105         self.failUnlessEqual(f(32, 7), 35)
 106         self.failUnlessEqual(f(32, 8), 32)
 107         self.failUnlessEqual(f(32, 9), 36)
 108         self.failUnlessEqual(f(32, 10), 40)
 109         self.failUnlessEqual(f(32, 11), 33)
 110         self.failUnlessEqual(f(32, 12), 36)
 111         self.failUnlessEqual(f(32, 13), 39)
 112         self.failUnlessEqual(f(32, 14), 42)
 113         self.failUnlessEqual(f(32, 15), 45)
 114         self.failUnlessEqual(f(32, 16), 32)
 115         self.failUnlessEqual(f(32, 17), 34)
 116         self.failUnlessEqual(f(32, 18), 36)
 117         self.failUnlessEqual(f(32, 589), 589)
 118
 119     def test_pad_size(self):
 120         f = mathutil.pad_size
 121         self.failUnlessEqual(f(0, 4), 0)
 122         self.failUnlessEqual(f(1, 4), 3)
 123         self.failUnlessEqual(f(2, 4), 2)
 124         self.failUnlessEqual(f(3, 4), 1)
 125         self.failUnlessEqual(f(4, 4), 0)
 126         self.failUnlessEqual(f(5, 4), 3)
 127
 128     def test_is_power_of_k(self):
 129         f = mathutil.is_power_of_k
 130         for i in range(1, 100):
 131             if i in (1, 2, 4, 8, 16, 32, 64):
 132                 self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
 133             else:
 134                 self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
 135         for i in range(1, 100):
 136             if i in (1, 3, 9, 27, 81):
 137                 self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
 138             else:
 139                 self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
 140
 141     def test_next_power_of_k(self):
 142         f = mathutil.next_power_of_k
 143         self.failUnlessEqual(f(0,2), 1)
 144         self.failUnlessEqual(f(1,2), 1)
 145         self.failUnlessEqual(f(2,2), 2)
 146         self.failUnlessEqual(f(3,2), 4)
 147         self.failUnlessEqual(f(4,2), 4)
 148         for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
 149         for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
 150         for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
 151         for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
 152         for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
 153
 154         self.failUnlessEqual(f(0,3), 1)
 155         self.failUnlessEqual(f(1,3), 1)
 156         self.failUnlessEqual(f(2,3), 3)
 157         self.failUnlessEqual(f(3,3), 3)
 158         for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
 159         for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
 160         for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
 161         for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
 162
 163     def test_ave(self):
 164         f = mathutil.ave
 165         self.failUnlessEqual(f([1,2,3]), 2)
 166         self.failUnlessEqual(f([0,0,0,4]), 1)
 167         self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
 168
 169     def test_round_sigfigs(self):
 170         f = mathutil.round_sigfigs
 171         self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002)
 172
 173 class Statistics(unittest.TestCase):
 174     def should_assert(self, msg, func, *args, **kwargs):
 175         try:
 176             func(*args, **kwargs)
 177             self.fail(msg)
 178         except AssertionError, e:
 179             pass
 180
 181     def failUnlessListEqual(self, a, b, msg = None):
 182         self.failUnlessEqual(len(a), len(b))
 183         for i in range(len(a)):
 184             self.failUnlessEqual(a[i], b[i], msg)
 185
 186     def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None):
 187         self.failUnlessEqual(len(a), len(b))
 188         for i in range(len(a)):
 189             self.failUnlessAlmostEqual(a[i], b[i], places, msg)
 190
 191     def test_binomial_coeff(self):
 192         f = statistics.binomial_coeff
 193         self.failUnlessEqual(f(20, 0), 1)
 194         self.failUnlessEqual(f(20, 1), 20)
 195         self.failUnlessEqual(f(20, 2), 190)
 196         self.failUnlessEqual(f(20, 8), f(20, 12))
 197         self.should_assert("Should assert if n < k", f, 2, 3)
 198
 199     def test_binomial_distribution_pmf(self):
 200         f = statistics.binomial_distribution_pmf
 201
 202         pmf_comp = f(2, .1)
 203         pmf_stat = [0.81, 0.18, 0.01]
 204         self.failUnlessListAlmostEqual(pmf_comp, pmf_stat)
 205
 206         # Summing across a PMF should give the total probability 1
 207         self.failUnlessAlmostEqual(sum(pmf_comp), 1)
 208         self.should_assert("Should assert if not 0<=p<=1", f, 1, -1)
 209         self.should_assert("Should assert if n < 1", f, 0, .1)
 210
 211         out = StringIO()
 212         statistics.print_pmf(pmf_comp, out=out)
 213         lines = out.getvalue().splitlines()
 214         self.failUnlessEqual(lines[0], "i=0: 0.81")
 215         self.failUnlessEqual(lines[1], "i=1: 0.18")
 216         self.failUnlessEqual(lines[2], "i=2: 0.01")
 217
 218     def test_survival_pmf(self):
 219         f = statistics.survival_pmf
 220         # Cross-check binomial-distribution method against convolution
 221         # method.
 222         p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20
 223         pmf1 = statistics.survival_pmf_via_conv(p_list)
 224         pmf2 = statistics.survival_pmf_via_bd(p_list)
 225         self.failUnlessListAlmostEqual(pmf1, pmf2)
 226         self.failUnlessTrue(statistics.valid_pmf(pmf1))
 227         self.should_assert("Should assert if p_i > 1", f, [1.1]);
 228         self.should_assert("Should assert if p_i < 0", f, [-.1]);
 229
 230     def test_repair_count_pmf(self):
 231         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 232         repair_pmf = statistics.repair_count_pmf(survival_pmf, 3)
 233         # repair_pmf[0] == sum(survival_pmf[0,1,2,5])
 234         # repair_pmf[1] == survival_pmf[4]
 235         # repair_pmf[2] = survival_pmf[3]
 236         self.failUnlessListAlmostEqual(repair_pmf,
 237                                        [0.00001 + 0.00045 + 0.0081 + 0.59049,
 238                                         .32805,
 239                                         .0729,
 240                                         0, 0, 0])
 241
 242     def test_repair_cost(self):
 243         survival_pmf = statistics.binomial_distribution_pmf(5, .9)
 244         bwcost = statistics.bandwidth_cost_function
 245         cost = statistics.mean_repair_cost(bwcost, 1000,
 246                                            survival_pmf, 3, ul_dl_ratio=1.0)
 247         self.failUnlessAlmostEqual(cost, 558.90)
 248         cost = statistics.mean_repair_cost(bwcost, 1000,
 249                                            survival_pmf, 3, ul_dl_ratio=8.0)
 250         self.failUnlessAlmostEqual(cost, 1664.55)
 251
 252         # I haven't manually checked the math beyond here -warner
 253         cost = statistics.eternal_repair_cost(bwcost, 1000,
 254                                               survival_pmf, 3,
 255                                               discount_rate=0, ul_dl_ratio=1.0)
 256         self.failUnlessAlmostEqual(cost, 65292.056074766246)
 257         cost = statistics.eternal_repair_cost(bwcost, 1000,
 258                                               survival_pmf, 3,
 259                                               discount_rate=0.05,
 260                                               ul_dl_ratio=1.0)
 261         self.failUnlessAlmostEqual(cost, 9133.6097158191551)
 262
 263     def test_convolve(self):
 264         f = statistics.convolve
 265         v1 = [ 1, 2, 3 ]
 266         v2 = [ 4, 5, 6 ]
 267         v3 = [ 7, 8 ]
 268         v1v2result = [ 4, 13, 28, 27, 18 ]
 269         # Convolution is commutative
 270         r1 = f(v1, v2)
 271         r2 = f(v2, v1)
 272         self.failUnlessListEqual(r1, r2, "Convolution should be commutative")
 273         self.failUnlessListEqual(r1, v1v2result, "Didn't match known result")
 274         # Convolution is associative
 275         r1 = f(f(v1, v2), v3)
 276         r2 = f(v1, f(v2, v3))
 277         self.failUnlessListEqual(r1, r2, "Convolution should be associative")
 278         # Convolution is distributive
 279         r1 = f(v3, [ a + b for a, b in zip(v1, v2) ])
 280         tmp1 = f(v3, v1)
 281         tmp2 = f(v3, v2)
 282         r2 = [ a + b for a, b in zip(tmp1, tmp2) ]
 283         self.failUnlessListEqual(r1, r2, "Convolution should be distributive")
 284         # Convolution is scalar multiplication associative
 285         tmp1 = f(v1, v2)
 286         r1 = [ a * 4 for a in tmp1 ]
 287         tmp2 = [ a * 4 for a in v1 ]
 288         r2 = f(tmp2, v2)
 289         self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative")
 290
 291     def test_find_k(self):
 292         f = statistics.find_k
 293         g = statistics.pr_file_loss
 294         plist = [.9] * 10 + [.8] * 10 # N=20
 295         t = .0001
 296         k = f(plist, t)
 297         self.failUnlessEqual(k, 10)
 298         self.failUnless(g(plist, k) < t)
 299
 300     def test_pr_file_loss(self):
 301         f = statistics.pr_file_loss
 302         plist = [.5] * 10
 303         self.failUnlessEqual(f(plist, 3), .0546875)
 304
 305     def test_pr_backup_file_loss(self):
 306         f = statistics.pr_backup_file_loss
 307         plist = [.5] * 10
 308         self.failUnlessEqual(f(plist, .5, 3), .02734375)
 309
 310
 311 class Asserts(unittest.TestCase):
 312     def should_assert(self, func, *args, **kwargs):
 313         try:
 314             func(*args, **kwargs)
 315         except AssertionError, e:
 316             return str(e)
 317         except Exception, e:
 318             self.fail("assert failed with non-AssertionError: %s" % e)
 319         self.fail("assert was not caught")
 320
 321     def should_not_assert(self, func, *args, **kwargs):
 322         if "re" in kwargs:
 323             regexp = kwargs["re"]
 324             del kwargs["re"]
 325         try:
 326             func(*args, **kwargs)
 327         except AssertionError, e:
 328             self.fail("assertion fired when it should not have: %s" % e)
 329         except Exception, e:
 330             self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e)
 331         return # we're happy
 332
 333
 334     def test_assert(self):
 335         f = assertutil._assert
 336         self.should_assert(f)
 337         self.should_assert(f, False)
 338         self.should_not_assert(f, True)
 339
 340         m = self.should_assert(f, False, "message")
 341         self.failUnlessEqual(m, "'message' <type 'str'>", m)
 342         m = self.should_assert(f, False, "message1", othermsg=12)
 343         self.failUnlessEqual("'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 344         m = self.should_assert(f, False, othermsg="message2")
 345         self.failUnlessEqual("othermsg: 'message2' <type 'str'>", m)
 346
 347     def test_precondition(self):
 348         f = assertutil.precondition
 349         self.should_assert(f)
 350         self.should_assert(f, False)
 351         self.should_not_assert(f, True)
 352
 353         m = self.should_assert(f, False, "message")
 354         self.failUnlessEqual("precondition: 'message' <type 'str'>", m)
 355         m = self.should_assert(f, False, "message1", othermsg=12)
 356         self.failUnlessEqual("precondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 357         m = self.should_assert(f, False, othermsg="message2")
 358         self.failUnlessEqual("precondition: othermsg: 'message2' <type 'str'>", m)
 359
 360     def test_postcondition(self):
 361         f = assertutil.postcondition
 362         self.should_assert(f)
 363         self.should_assert(f, False)
 364         self.should_not_assert(f, True)
 365
 366         m = self.should_assert(f, False, "message")
 367         self.failUnlessEqual("postcondition: 'message' <type 'str'>", m)
 368         m = self.should_assert(f, False, "message1", othermsg=12)
 369         self.failUnlessEqual("postcondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 370         m = self.should_assert(f, False, othermsg="message2")
 371         self.failUnlessEqual("postcondition: othermsg: 'message2' <type 'str'>", m)
 372
 373 class FileUtil(unittest.TestCase):
 374     def mkdir(self, basedir, path, mode=0777):
 375         fn = os.path.join(basedir, path)
 376         fileutil.make_dirs(fn, mode)
 377
 378     def touch(self, basedir, path, mode=None, data="touch\n"):
 379         fn = os.path.join(basedir, path)
 380         f = open(fn, "w")
 381         f.write(data)
 382         f.close()
 383         if mode is not None:
 384             os.chmod(fn, mode)
 385
 386     def test_rm_dir(self):
 387         basedir = "util/FileUtil/test_rm_dir"
 388         fileutil.make_dirs(basedir)
 389         # create it again to test idempotency
 390         fileutil.make_dirs(basedir)
 391         d = os.path.join(basedir, "doomed")
 392         self.mkdir(d, "a/b")
 393         self.touch(d, "a/b/1.txt")
 394         self.touch(d, "a/b/2.txt", 0444)
 395         self.touch(d, "a/b/3.txt", 0)
 396         self.mkdir(d, "a/c")
 397         self.touch(d, "a/c/1.txt")
 398         self.touch(d, "a/c/2.txt", 0444)
 399         self.touch(d, "a/c/3.txt", 0)
 400         os.chmod(os.path.join(d, "a/c"), 0444)
 401         self.mkdir(d, "a/d")
 402         self.touch(d, "a/d/1.txt")
 403         self.touch(d, "a/d/2.txt", 0444)
 404         self.touch(d, "a/d/3.txt", 0)
 405         os.chmod(os.path.join(d, "a/d"), 0)
 406
 407         fileutil.rm_dir(d)
 408         self.failIf(os.path.exists(d))
 409         # remove it again to test idempotency
 410         fileutil.rm_dir(d)
 411
 412     def test_remove_if_possible(self):
 413         basedir = "util/FileUtil/test_remove_if_possible"
 414         fileutil.make_dirs(basedir)
 415         self.touch(basedir, "here")
 416         fn = os.path.join(basedir, "here")
 417         fileutil.remove_if_possible(fn)
 418         self.failIf(os.path.exists(fn))
 419         fileutil.remove_if_possible(fn) # should be idempotent
 420         fileutil.rm_dir(basedir)
 421         fileutil.remove_if_possible(fn) # should survive errors
 422
 423     def test_open_or_create(self):
 424         basedir = "util/FileUtil/test_open_or_create"
 425         fileutil.make_dirs(basedir)
 426         fn = os.path.join(basedir, "here")
 427         f = fileutil.open_or_create(fn)
 428         f.write("stuff.")
 429         f.close()
 430         f = fileutil.open_or_create(fn)
 431         f.seek(0, 2)
 432         f.write("more.")
 433         f.close()
 434         f = open(fn, "r")
 435         data = f.read()
 436         f.close()
 437         self.failUnlessEqual(data, "stuff.more.")
 438
 439     def test_NamedTemporaryDirectory(self):
 440         basedir = "util/FileUtil/test_NamedTemporaryDirectory"
 441         fileutil.make_dirs(basedir)
 442         td = fileutil.NamedTemporaryDirectory(dir=basedir)
 443         name = td.name
 444         self.failUnless(basedir in name)
 445         self.failUnless(basedir in repr(td))
 446         self.failUnless(os.path.isdir(name))
 447         del td
 448         # it is conceivable that we need to force gc here, but I'm not sure
 449         self.failIf(os.path.isdir(name))
 450
 451     def test_rename(self):
 452         basedir = "util/FileUtil/test_rename"
 453         fileutil.make_dirs(basedir)
 454         self.touch(basedir, "here")
 455         fn = os.path.join(basedir, "here")
 456         fn2 = os.path.join(basedir, "there")
 457         fileutil.rename(fn, fn2)
 458         self.failIf(os.path.exists(fn))
 459         self.failUnless(os.path.exists(fn2))
 460
 461     def test_du(self):
 462         basedir = "util/FileUtil/test_du"
 463         fileutil.make_dirs(basedir)
 464         d = os.path.join(basedir, "space-consuming")
 465         self.mkdir(d, "a/b")
 466         self.touch(d, "a/b/1.txt", data="a"*10)
 467         self.touch(d, "a/b/2.txt", data="b"*11)
 468         self.mkdir(d, "a/c")
 469         self.touch(d, "a/c/1.txt", data="c"*12)
 470         self.touch(d, "a/c/2.txt", data="d"*13)
 471
 472         used = fileutil.du(basedir)
 473         self.failUnlessEqual(10+11+12+13, used)
 474
 475 class PollMixinTests(unittest.TestCase):
 476     def setUp(self):
 477         self.pm = pollmixin.PollMixin()
 478
 479     def test_PollMixin_True(self):
 480         d = self.pm.poll(check_f=lambda : True,
 481                          pollinterval=0.1)
 482         return d
 483
 484     def test_PollMixin_False_then_True(self):
 485         i = iter([False, True])
 486         d = self.pm.poll(check_f=i.next,
 487                          pollinterval=0.1)
 488         return d
 489
 490     def test_timeout(self):
 491         d = self.pm.poll(check_f=lambda: False,
 492                          pollinterval=0.01,
 493                          timeout=1)
 494         def _suc(res):
 495             self.fail("poll should have failed, not returned %s" % (res,))
 496         def _err(f):
 497             f.trap(pollmixin.TimeoutError)
 498             return None # success
 499         d.addCallbacks(_suc, _err)
 500         return d
 501
 502 class DeferredUtilTests(unittest.TestCase):
 503     def test_gather_results(self):
 504         d1 = defer.Deferred()
 505         d2 = defer.Deferred()
 506         res = deferredutil.gatherResults([d1, d2])
 507         d1.errback(ValueError("BAD"))
 508         def _callb(res):
 509             self.fail("Should have errbacked, not resulted in %s" % (res,))
 510         def _errb(thef):
 511             thef.trap(ValueError)
 512         res.addCallbacks(_callb, _errb)
 513         return res
 514
 515     def test_success(self):
 516         d1, d2 = defer.Deferred(), defer.Deferred()
 517         good = []
 518         bad = []
 519         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 520         dlss.addCallbacks(good.append, bad.append)
 521         d1.callback(1)
 522         d2.callback(2)
 523         self.failUnlessEqual(good, [[1,2]])
 524         self.failUnlessEqual(bad, [])
 525
 526     def test_failure(self):
 527         d1, d2 = defer.Deferred(), defer.Deferred()
 528         good = []
 529         bad = []
 530         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 531         dlss.addCallbacks(good.append, bad.append)
 532         d1.addErrback(lambda _ignore: None)
 533         d2.addErrback(lambda _ignore: None)
 534         d1.callback(1)
 535         d2.errback(ValueError())
 536         self.failUnlessEqual(good, [])
 537         self.failUnlessEqual(len(bad), 1)
 538         f = bad[0]
 539         self.failUnless(isinstance(f, Failure))
 540         self.failUnless(f.check(ValueError))
 541
 542 class HashUtilTests(unittest.TestCase):
 543
 544     def test_random_key(self):
 545         k = hashutil.random_key()
 546         self.failUnlessEqual(len(k), hashutil.KEYLEN)
 547
 548     def test_sha256d(self):
 549         h1 = hashutil.tagged_hash("tag1", "value")
 550         h2 = hashutil.tagged_hasher("tag1")
 551         h2.update("value")
 552         h2a = h2.digest()
 553         h2b = h2.digest()
 554         self.failUnlessEqual(h1, h2a)
 555         self.failUnlessEqual(h2a, h2b)
 556
 557     def test_sha256d_truncated(self):
 558         h1 = hashutil.tagged_hash("tag1", "value", 16)
 559         h2 = hashutil.tagged_hasher("tag1", 16)
 560         h2.update("value")
 561         h2 = h2.digest()
 562         self.failUnlessEqual(len(h1), 16)
 563         self.failUnlessEqual(len(h2), 16)
 564         self.failUnlessEqual(h1, h2)
 565
 566     def test_chk(self):
 567         h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret")
 568         h2 = hashutil.convergence_hasher(3, 10, 1000, "secret")
 569         h2.update("data")
 570         h2 = h2.digest()
 571         self.failUnlessEqual(h1, h2)
 572
 573     def test_hashers(self):
 574         h1 = hashutil.block_hash("foo")
 575         h2 = hashutil.block_hasher()
 576         h2.update("foo")
 577         self.failUnlessEqual(h1, h2.digest())
 578
 579         h1 = hashutil.uri_extension_hash("foo")
 580         h2 = hashutil.uri_extension_hasher()
 581         h2.update("foo")
 582         self.failUnlessEqual(h1, h2.digest())
 583
 584         h1 = hashutil.plaintext_hash("foo")
 585         h2 = hashutil.plaintext_hasher()
 586         h2.update("foo")
 587         self.failUnlessEqual(h1, h2.digest())
 588
 589         h1 = hashutil.crypttext_hash("foo")
 590         h2 = hashutil.crypttext_hasher()
 591         h2.update("foo")
 592         self.failUnlessEqual(h1, h2.digest())
 593
 594         h1 = hashutil.crypttext_segment_hash("foo")
 595         h2 = hashutil.crypttext_segment_hasher()
 596         h2.update("foo")
 597         self.failUnlessEqual(h1, h2.digest())
 598
 599         h1 = hashutil.plaintext_segment_hash("foo")
 600         h2 = hashutil.plaintext_segment_hasher()
 601         h2.update("foo")
 602         self.failUnlessEqual(h1, h2.digest())
 603
 604 class Abbreviate(unittest.TestCase):
 605     def test_time(self):
 606         a = abbreviate.abbreviate_time
 607         self.failUnlessEqual(a(None), "unknown")
 608         self.failUnlessEqual(a(0), "0 seconds")
 609         self.failUnlessEqual(a(1), "1 second")
 610         self.failUnlessEqual(a(2), "2 seconds")
 611         self.failUnlessEqual(a(119), "119 seconds")
 612         MIN = 60
 613         self.failUnlessEqual(a(2*MIN), "2 minutes")
 614         self.failUnlessEqual(a(60*MIN), "60 minutes")
 615         self.failUnlessEqual(a(179*MIN), "179 minutes")
 616         HOUR = 60*MIN
 617         self.failUnlessEqual(a(180*MIN), "3 hours")
 618         self.failUnlessEqual(a(4*HOUR), "4 hours")
 619         DAY = 24*HOUR
 620         MONTH = 30*DAY
 621         self.failUnlessEqual(a(2*DAY), "2 days")
 622         self.failUnlessEqual(a(2*MONTH), "2 months")
 623         YEAR = 365*DAY
 624         self.failUnlessEqual(a(5*YEAR), "5 years")
 625
 626     def test_space(self):
 627         tests_si = [(None, "unknown"),
 628                     (0, "0 B"),
 629                     (1, "1 B"),
 630                     (999, "999 B"),
 631                     (1000, "1000 B"),
 632                     (1023, "1023 B"),
 633                     (1024, "1.02 kB"),
 634                     (20*1000, "20.00 kB"),
 635                     (1024*1024, "1.05 MB"),
 636                     (1000*1000, "1.00 MB"),
 637                     (1000*1000*1000, "1.00 GB"),
 638                     (1000*1000*1000*1000, "1.00 TB"),
 639                     (1000*1000*1000*1000*1000, "1.00 PB"),
 640                     (1234567890123456, "1.23 PB"),
 641                     ]
 642         for (x, expected) in tests_si:
 643             got = abbreviate.abbreviate_space(x, SI=True)
 644             self.failUnlessEqual(got, expected)
 645
 646         tests_base1024 = [(None, "unknown"),
 647                           (0, "0 B"),
 648                           (1, "1 B"),
 649                           (999, "999 B"),
 650                           (1000, "1000 B"),
 651                           (1023, "1023 B"),
 652                           (1024, "1.00 kiB"),
 653                           (20*1024, "20.00 kiB"),
 654                           (1000*1000, "976.56 kiB"),
 655                           (1024*1024, "1.00 MiB"),
 656                           (1024*1024*1024, "1.00 GiB"),
 657                           (1024*1024*1024*1024, "1.00 TiB"),
 658                           (1000*1000*1000*1000*1000, "909.49 TiB"),
 659                           (1024*1024*1024*1024*1024, "1.00 PiB"),
 660                           (1234567890123456, "1.10 PiB"),
 661                     ]
 662         for (x, expected) in tests_base1024:
 663             got = abbreviate.abbreviate_space(x, SI=False)
 664             self.failUnlessEqual(got, expected)
 665
 666         self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567),
 667                              "(1.23 MB, 1.18 MiB)")
 668
 669     def test_parse_space(self):
 670         p = abbreviate.parse_abbreviated_size
 671         self.failUnlessEqual(p(""), None)
 672         self.failUnlessEqual(p(None), None)
 673         self.failUnlessEqual(p("123"), 123)
 674         self.failUnlessEqual(p("123B"), 123)
 675         self.failUnlessEqual(p("2K"), 2000)
 676         self.failUnlessEqual(p("2kb"), 2000)
 677         self.failUnlessEqual(p("2KiB"), 2048)
 678         self.failUnlessEqual(p("10MB"), 10*1000*1000)
 679         self.failUnlessEqual(p("10MiB"), 10*1024*1024)
 680         self.failUnlessEqual(p("5G"), 5*1000*1000*1000)
 681         self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024)
 682         e = self.failUnlessRaises(ValueError, p, "12 cubits")
 683         self.failUnless("12 cubits" in str(e))
 684
 685 class Limiter(unittest.TestCase):
 686     def job(self, i, foo):
 687         self.calls.append( (i, foo) )
 688         self.simultaneous += 1
 689         self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous)
 690         d = defer.Deferred()
 691         def _done():
 692             self.simultaneous -= 1
 693             d.callback("done %d" % i)
 694         reactor.callLater(1.0, _done)
 695         return d
 696
 697     def bad_job(self, i, foo):
 698         raise ValueError("bad_job %d" % i)
 699
 700     def test_limiter(self):
 701         self.calls = []
 702         self.simultaneous = 0
 703         self.peak_simultaneous = 0
 704         l = limiter.ConcurrencyLimiter()
 705         dl = []
 706         for i in range(20):
 707             dl.append(l.add(self.job, i, foo=str(i)))
 708         d = defer.DeferredList(dl, fireOnOneErrback=True)
 709         def _done(res):
 710             self.failUnlessEqual(self.simultaneous, 0)
 711             self.failUnless(self.peak_simultaneous <= 10)
 712             self.failUnlessEqual(len(self.calls), 20)
 713             for i in range(20):
 714                 self.failUnless( (i, str(i)) in self.calls)
 715         d.addCallback(_done)
 716         return d
 717
 718     def test_errors(self):
 719         self.calls = []
 720         self.simultaneous = 0
 721         self.peak_simultaneous = 0
 722         l = limiter.ConcurrencyLimiter()
 723         dl = []
 724         for i in range(20):
 725             dl.append(l.add(self.job, i, foo=str(i)))
 726         d2 = l.add(self.bad_job, 21, "21")
 727         d = defer.DeferredList(dl, fireOnOneErrback=True)
 728         def _most_done(res):
 729             results = []
 730             for (success, result) in res:
 731                 self.failUnlessEqual(success, True)
 732                 results.append(result)
 733             results.sort()
 734             expected_results = ["done %d" % i for i in range(20)]
 735             expected_results.sort()
 736             self.failUnlessEqual(results, expected_results)
 737             self.failUnless(self.peak_simultaneous <= 10)
 738             self.failUnlessEqual(len(self.calls), 20)
 739             for i in range(20):
 740                 self.failUnless( (i, str(i)) in self.calls)
 741             def _good(res):
 742                 self.fail("should have failed, not got %s" % (res,))
 743             def _err(f):
 744                 f.trap(ValueError)
 745                 self.failUnless("bad_job 21" in str(f))
 746             d2.addCallbacks(_good, _err)
 747             return d2
 748         d.addCallback(_most_done)
 749         def _all_done(res):
 750             self.failUnlessEqual(self.simultaneous, 0)
 751             self.failUnless(self.peak_simultaneous <= 10)
 752             self.failUnlessEqual(len(self.calls), 20)
 753             for i in range(20):
 754                 self.failUnless( (i, str(i)) in self.calls)
 755         d.addCallback(_all_done)
 756         return d
 757
 758 class TimeFormat(unittest.TestCase):
 759     def test_epoch(self):
 760         s = time_format.iso_utc_time_to_localseconds("1970-01-01T00:00:01")
 761         self.failUnlessEqual(s, 1.0)
 762         s = time_format.iso_utc_time_to_localseconds("1970-01-01_00:00:01")
 763         self.failUnlessEqual(s, 1.0)
 764         s = time_format.iso_utc_time_to_localseconds("1970-01-01 00:00:01")
 765         self.failUnlessEqual(s, 1.0)
 766
 767         self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01")
 768         self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "),
 769                              "1970-01-01 00:00:01")
 770         now = time.time()
 771         def my_time():
 772             return 1.0
 773         self.failUnlessEqual(time_format.iso_utc(t=my_time),
 774                              "1970-01-01_00:00:01")
 775         e = self.failUnlessRaises(ValueError,
 776                                   time_format.iso_utc_time_to_localseconds,
 777                                   "invalid timestring")
 778         self.failUnless("not a complete ISO8601 timestamp" in str(e))
 779         s = time_format.iso_utc_time_to_localseconds("1970-01-01_00:00:01.500")
 780         self.failUnlessEqual(s, 1.5)
 781
 782 class CacheDir(unittest.TestCase):
 783     def test_basic(self):
 784         basedir = "test_util/CacheDir/test_basic"
 785
 786         def _failIfExists(name):
 787             absfn = os.path.join(basedir, name)
 788             self.failIf(os.path.exists(absfn),
 789                         "%s exists but it shouldn't" % absfn)
 790
 791         def _failUnlessExists(name):
 792             absfn = os.path.join(basedir, name)
 793             self.failUnless(os.path.exists(absfn),
 794                             "%s doesn't exist but it should" % absfn)
 795
 796         cdm = cachedir.CacheDirectoryManager(basedir)
 797         a = cdm.get_file("a")
 798         b = cdm.get_file("b")
 799         c = cdm.get_file("c")
 800         f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f
 801         f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f
 802         f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f
 803
 804         _failUnlessExists("a")
 805         _failUnlessExists("b")
 806         _failUnlessExists("c")
 807
 808         cdm.check()
 809
 810         _failUnlessExists("a")
 811         _failUnlessExists("b")
 812         _failUnlessExists("c")
 813
 814         del a
 815         # this file won't be deleted yet, because it isn't old enough
 816         cdm.check()
 817         _failUnlessExists("a")
 818         _failUnlessExists("b")
 819         _failUnlessExists("c")
 820
 821         # we change the definition of "old" to make everything old
 822         cdm.old = -10
 823
 824         cdm.check()
 825         _failIfExists("a")
 826         _failUnlessExists("b")
 827         _failUnlessExists("c")
 828
 829         cdm.old = 60*60
 830
 831         del b
 832
 833         cdm.check()
 834         _failIfExists("a")
 835         _failUnlessExists("b")
 836         _failUnlessExists("c")
 837
 838         b2 = cdm.get_file("b")
 839
 840         cdm.check()
 841         _failIfExists("a")
 842         _failUnlessExists("b")
 843         _failUnlessExists("c")
 844
 845 ctr = [0]
 846 class EqButNotIs:
 847     def __init__(self, x):
 848         self.x = x
 849         self.hash = ctr[0]
 850         ctr[0] += 1
 851     def __repr__(self):
 852         return "<%s %s>" % (self.__class__.__name__, self.x,)
 853     def __hash__(self):
 854         return self.hash
 855     def __le__(self, other):
 856         return self.x <= other
 857     def __lt__(self, other):
 858         return self.x < other
 859     def __ge__(self, other):
 860         return self.x >= other
 861     def __gt__(self, other):
 862         return self.x > other
 863     def __ne__(self, other):
 864         return self.x != other
 865     def __eq__(self, other):
 866         return self.x == other
 867
 868 class DictUtil(unittest.TestCase):
 869     def _help_test_empty_dict(self, klass):
 870         d1 = klass()
 871         d2 = klass({})
 872
 873         self.failUnless(d1 == d2, "d1: %r, d2: %r" % (d1, d2,))
 874         self.failUnless(len(d1) == 0)
 875         self.failUnless(len(d2) == 0)
 876
 877     def _help_test_nonempty_dict(self, klass):
 878         d1 = klass({'a': 1, 'b': "eggs", 3: "spam",})
 879         d2 = klass({'a': 1, 'b': "eggs", 3: "spam",})
 880
 881         self.failUnless(d1 == d2)
 882         self.failUnless(len(d1) == 3, "%s, %s" % (len(d1), d1,))
 883         self.failUnless(len(d2) == 3)
 884
 885     def _help_test_eq_but_notis(self, klass):
 886         d = klass({'a': 3, 'b': EqButNotIs(3), 'c': 3})
 887         d.pop('b')
 888
 889         d.clear()
 890         d['a'] = 3
 891         d['b'] = EqButNotIs(3)
 892         d['c'] = 3
 893         d.pop('b')
 894
 895         d.clear()
 896         d['b'] = EqButNotIs(3)
 897         d['a'] = 3
 898         d['c'] = 3
 899         d.pop('b')
 900
 901         d.clear()
 902         d['a'] = EqButNotIs(3)
 903         d['c'] = 3
 904         d['a'] = 3
 905
 906         d.clear()
 907         fake3 = EqButNotIs(3)
 908         fake7 = EqButNotIs(7)
 909         d[fake3] = fake7
 910         d[3] = 7
 911         d[3] = 8
 912         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
 913         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
 914         # The real 7 should have been ejected by the d[3] = 8.
 915         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
 916         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
 917         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
 918         d[fake3] = 8
 919
 920         d.clear()
 921         d[3] = 7
 922         fake3 = EqButNotIs(3)
 923         fake7 = EqButNotIs(7)
 924         d[fake3] = fake7
 925         d[3] = 8
 926         self.failUnless(filter(lambda x: x is 8,  d.itervalues()))
 927         self.failUnless(filter(lambda x: x is fake7,  d.itervalues()))
 928         # The real 7 should have been ejected by the d[3] = 8.
 929         self.failUnless(not filter(lambda x: x is 7,  d.itervalues()))
 930         self.failUnless(filter(lambda x: x is fake3,  d.iterkeys()))
 931         self.failUnless(filter(lambda x: x is 3,  d.iterkeys()))
 932         d[fake3] = 8
 933
 934     def test_all(self):
 935         self._help_test_eq_but_notis(dictutil.UtilDict)
 936         self._help_test_eq_but_notis(dictutil.NumDict)
 937         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
 938         self._help_test_nonempty_dict(dictutil.UtilDict)
 939         self._help_test_nonempty_dict(dictutil.NumDict)
 940         self._help_test_nonempty_dict(dictutil.ValueOrderedDict)
 941         self._help_test_eq_but_notis(dictutil.UtilDict)
 942         self._help_test_eq_but_notis(dictutil.NumDict)
 943         self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
 944
 945     def test_dict_of_sets(self):
 946         ds = dictutil.DictOfSets()
 947         ds.add(1, "a")
 948         ds.add(2, "b")
 949         ds.add(2, "b")
 950         ds.add(2, "c")
 951         self.failUnlessEqual(ds[1], set(["a"]))
 952         self.failUnlessEqual(ds[2], set(["b", "c"]))
 953         ds.discard(3, "d") # should not raise an exception
 954         ds.discard(2, "b")
 955         self.failUnlessEqual(ds[2], set(["c"]))
 956         ds.discard(2, "c")
 957         self.failIf(2 in ds)
 958
 959         ds.union(1, ["a", "e"])
 960         ds.union(3, ["f"])
 961         self.failUnlessEqual(ds[1], set(["a","e"]))
 962         self.failUnlessEqual(ds[3], set(["f"]))
 963         ds2 = dictutil.DictOfSets()
 964         ds2.add(3, "f")
 965         ds2.add(3, "g")
 966         ds2.add(4, "h")
 967         ds.update(ds2)
 968         self.failUnlessEqual(ds[1], set(["a","e"]))
 969         self.failUnlessEqual(ds[3], set(["f", "g"]))
 970         self.failUnlessEqual(ds[4], set(["h"]))
 971
 972     def test_move(self):
 973         d1 = {1: "a", 2: "b"}
 974         d2 = {2: "c", 3: "d"}
 975         dictutil.move(1, d1, d2)
 976         self.failUnlessEqual(d1, {2: "b"})
 977         self.failUnlessEqual(d2, {1: "a", 2: "c", 3: "d"})
 978
 979         d1 = {1: "a", 2: "b"}
 980         d2 = {2: "c", 3: "d"}
 981         dictutil.move(2, d1, d2)
 982         self.failUnlessEqual(d1, {1: "a"})
 983         self.failUnlessEqual(d2, {2: "b", 3: "d"})
 984
 985         d1 = {1: "a", 2: "b"}
 986         d2 = {2: "c", 3: "d"}
 987         self.failUnlessRaises(KeyError, dictutil.move, 5, d1, d2, strict=True)
 988
 989     def test_subtract(self):
 990         d1 = {1: "a", 2: "b"}
 991         d2 = {2: "c", 3: "d"}
 992         d3 = dictutil.subtract(d1, d2)
 993         self.failUnlessEqual(d3, {1: "a"})
 994
 995         d1 = {1: "a", 2: "b"}
 996         d2 = {2: "c"}
 997         d3 = dictutil.subtract(d1, d2)
 998         self.failUnlessEqual(d3, {1: "a"})
 999
1000     def test_utildict(self):
1001         d = dictutil.UtilDict({1: "a", 2: "b"})
1002         d.del_if_present(1)
1003         d.del_if_present(3)
1004         self.failUnlessEqual(d, {2: "b"})
1005         def eq(a, b):
1006             return a == b
1007         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1008
1009         d = dictutil.UtilDict({1: "b", 2: "a"})
1010         self.failUnlessEqual(d.items_sorted_by_value(),
1011                              [(2, "a"), (1, "b")])
1012         self.failUnlessEqual(d.items_sorted_by_key(),
1013                              [(1, "b"), (2, "a")])
1014         self.failUnlessEqual(repr(d), "{1: 'b', 2: 'a'}")
1015         self.failUnless(1 in d)
1016
1017         d2 = dictutil.UtilDict({3: "c", 4: "d"})
1018         self.failUnless(d != d2)
1019         self.failUnless(d2 > d)
1020         self.failUnless(d2 >= d)
1021         self.failUnless(d <= d2)
1022         self.failUnless(d < d2)
1023         self.failUnlessEqual(d[1], "b")
1024         self.failUnlessEqual(sorted(list([k for k in d])), [1,2])
1025
1026         d3 = d.copy()
1027         self.failUnlessEqual(d, d3)
1028         self.failUnless(isinstance(d3, dictutil.UtilDict))
1029
1030         d4 = d.fromkeys([3,4], "e")
1031         self.failUnlessEqual(d4, {3: "e", 4: "e"})
1032
1033         self.failUnlessEqual(d.get(1), "b")
1034         self.failUnlessEqual(d.get(3), None)
1035         self.failUnlessEqual(d.get(3, "default"), "default")
1036         self.failUnlessEqual(sorted(list(d.items())),
1037                              [(1, "b"), (2, "a")])
1038         self.failUnlessEqual(sorted(list(d.iteritems())),
1039                              [(1, "b"), (2, "a")])
1040         self.failUnlessEqual(sorted(d.keys()), [1, 2])
1041         self.failUnlessEqual(sorted(d.values()), ["a", "b"])
1042         x = d.setdefault(1, "new")
1043         self.failUnlessEqual(x, "b")
1044         self.failUnlessEqual(d[1], "b")
1045         x = d.setdefault(3, "new")
1046         self.failUnlessEqual(x, "new")
1047         self.failUnlessEqual(d[3], "new")
1048         del d[3]
1049
1050         x = d.popitem()
1051         self.failUnless(x in [(1, "b"), (2, "a")])
1052         x = d.popitem()
1053         self.failUnless(x in [(1, "b"), (2, "a")])
1054         self.failUnlessRaises(KeyError, d.popitem)
1055
1056     def test_numdict(self):
1057         d = dictutil.NumDict({"a": 1, "b": 2})
1058
1059         d.add_num("a", 10, 5)
1060         d.add_num("c", 20, 5)
1061         d.add_num("d", 30)
1062         self.failUnlessEqual(d, {"a": 11, "b": 2, "c": 25, "d": 30})
1063
1064         d.subtract_num("a", 10)
1065         d.subtract_num("e", 10)
1066         d.subtract_num("f", 10, 15)
1067         self.failUnlessEqual(d, {"a": 1, "b": 2, "c": 25, "d": 30,
1068                                  "e": -10, "f": 5})
1069
1070         self.failUnlessEqual(d.sum(), sum([1, 2, 25, 30, -10, 5]))
1071
1072         d = dictutil.NumDict()
1073         d.inc("a")
1074         d.inc("a")
1075         d.inc("b", 5)
1076         self.failUnlessEqual(d, {"a": 2, "b": 6})
1077         d.dec("a")
1078         d.dec("c")
1079         d.dec("d", 5)
1080         self.failUnlessEqual(d, {"a": 1, "b": 6, "c": -1, "d": 4})
1081         self.failUnlessEqual(d.items_sorted_by_key(),
1082                              [("a", 1), ("b", 6), ("c", -1), ("d", 4)])
1083         self.failUnlessEqual(d.items_sorted_by_value(),
1084                              [("c", -1), ("a", 1), ("d", 4), ("b", 6)])
1085         self.failUnlessEqual(d.item_with_largest_value(), ("b", 6))
1086
1087         d = dictutil.NumDict({"a": 1, "b": 2})
1088         self.failUnlessEqual(repr(d), "{'a': 1, 'b': 2}")
1089         self.failUnless("a" in d)
1090
1091         d2 = dictutil.NumDict({"c": 3, "d": 4})
1092         self.failUnless(d != d2)
1093         self.failUnless(d2 > d)
1094         self.failUnless(d2 >= d)
1095         self.failUnless(d <= d2)
1096         self.failUnless(d < d2)
1097         self.failUnlessEqual(d["a"], 1)
1098         self.failUnlessEqual(sorted(list([k for k in d])), ["a","b"])
1099         def eq(a, b):
1100             return a == b
1101         self.failUnlessRaises(TypeError, eq, d, "not a dict")
1102
1103         d3 = d.copy()
1104         self.failUnlessEqual(d, d3)
1105         self.failUnless(isinstance(d3, dictutil.NumDict))
1106
1107         d4 = d.fromkeys(["a","b"], 5)
1108         self.failUnlessEqual(d4, {"a": 5, "b": 5})
1109
1110         self.failUnlessEqual(d.get("a"), 1)
1111         self.failUnlessEqual(d.get("c"), 0)
1112         self.failUnlessEqual(d.get("c", 5), 5)
1113         self.failUnlessEqual(sorted(list(d.items())),
1114                              [("a", 1), ("b", 2)])
1115         self.failUnlessEqual(sorted(list(d.iteritems())),
1116                              [("a", 1), ("b", 2)])
1117         self.failUnlessEqual(sorted(d.keys()), ["a", "b"])
1118         self.failUnlessEqual(sorted(d.values()), [1, 2])
1119         self.failUnless(d.has_key("a"))
1120         self.failIf(d.has_key("c"))
1121
1122         x = d.setdefault("c", 3)
1123         self.failUnlessEqual(x, 3)
1124         self.failUnlessEqual(d["c"], 3)
1125         x = d.setdefault("c", 5)
1126         self.failUnlessEqual(x, 3)
1127         self.failUnlessEqual(d["c"], 3)
1128         del d["c"]
1129
1130         x = d.popitem()
1131         self.failUnless(x in [("a", 1), ("b", 2)])
1132         x = d.popitem()
1133         self.failUnless(x in [("a", 1), ("b", 2)])
1134         self.failUnlessRaises(KeyError, d.popitem)
1135
1136         d.update({"c": 3})
1137         d.update({"c": 4, "d": 5})
1138         self.failUnlessEqual(d, {"c": 4, "d": 5})
1139
1140     def test_del_if_present(self):
1141         d = {1: "a", 2: "b"}
1142         dictutil.del_if_present(d, 1)
1143         dictutil.del_if_present(d, 3)
1144         self.failUnlessEqual(d, {2: "b"})
1145
1146     def test_valueordereddict(self):
1147         d = dictutil.ValueOrderedDict()
1148         d["a"] = 3
1149         d["b"] = 2
1150         d["c"] = 1
1151
1152         self.failUnlessEqual(d, {"a": 3, "b": 2, "c": 1})
1153         self.failUnlessEqual(d.items(), [("c", 1), ("b", 2), ("a", 3)])
1154         self.failUnlessEqual(d.values(), [1, 2, 3])
1155         self.failUnlessEqual(d.keys(), ["c", "b", "a"])
1156         self.failUnlessEqual(repr(d), "<ValueOrderedDict {c: 1, b: 2, a: 3}>")
1157         def eq(a, b):
1158             return a == b
1159         self.failIf(d == {"a": 4})
1160         self.failUnless(d != {"a": 4})
1161
1162         x = d.setdefault("d", 0)
1163         self.failUnlessEqual(x, 0)
1164         self.failUnlessEqual(d["d"], 0)
1165         x = d.setdefault("d", -1)
1166         self.failUnlessEqual(x, 0)
1167         self.failUnlessEqual(d["d"], 0)
1168
1169         x = d.remove("e", "default", False)
1170         self.failUnlessEqual(x, "default")
1171         self.failUnlessRaises(KeyError, d.remove, "e", "default", True)
1172         x = d.remove("d", 5)
1173         self.failUnlessEqual(x, 0)
1174
1175         x = d.__getitem__("c")
1176         self.failUnlessEqual(x, 1)
1177         x = d.__getitem__("e", "default", False)
1178         self.failUnlessEqual(x, "default")
1179         self.failUnlessRaises(KeyError, d.__getitem__, "e", "default", True)
1180
1181         self.failUnlessEqual(d.popitem(), ("c", 1))
1182         self.failUnlessEqual(d.popitem(), ("b", 2))
1183         self.failUnlessEqual(d.popitem(), ("a", 3))
1184         self.failUnlessRaises(KeyError, d.popitem)
1185
1186         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1187         x = d.pop("d", "default", False)
1188         self.failUnlessEqual(x, "default")
1189         self.failUnlessRaises(KeyError, d.pop, "d", "default", True)
1190         x = d.pop("b")
1191         self.failUnlessEqual(x, 2)
1192         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1193
1194         d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1195         x = d.pop_from_list(1) # pop the second item, b/2
1196         self.failUnlessEqual(x, "b")
1197         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1198
1199 class FakeRemoteReference:
1200     def callRemote(self, methname, *args, **kwargs):
1201         return defer.maybeDeferred(self.oops)
1202     def oops(self):
1203         raise IndexError("remote missing key")
1204
1205 class RemoteFailures(unittest.TestCase):
1206     def test_check(self):
1207         try:
1208             raise IndexError("local missing key")
1209         except IndexError:
1210             localf = Failure()
1211         self.failUnlessEqual(localf.check(IndexError, KeyError), IndexError)
1212         self.failUnlessEqual(localf.check(ValueError, KeyError), None)
1213         self.failUnlessEqual(localf.check(ServerFailure), None)
1214
1215         frr = FakeRemoteReference()
1216         wrr = rrefutil.WrappedRemoteReference(frr)
1217         d = wrr.callRemote("oops")
1218         def _check(f):
1219             self.failUnlessEqual(f.check(IndexError, KeyError), None)
1220             self.failUnlessEqual(f.check(ServerFailure, KeyError),
1221                                  ServerFailure)
1222         d.addErrback(_check)
1223         return d
1224
1225     def test_is_remote(self):
1226         try:
1227             raise IndexError("local missing key")
1228         except IndexError:
1229             localf = Failure()
1230         self.failIf(rrefutil.is_remote(localf))
1231         self.failUnless(rrefutil.is_local(localf))
1232
1233         frr = FakeRemoteReference()
1234         wrr = rrefutil.WrappedRemoteReference(frr)
1235         d = wrr.callRemote("oops")
1236         def _check(f):
1237             self.failUnless(rrefutil.is_remote(f))
1238             self.failIf(rrefutil.is_local(f))
1239         d.addErrback(_check)
1240         return d
1241
1242     def test_trap(self):
1243         try:
1244             raise IndexError("local missing key")
1245         except IndexError:
1246             localf = Failure()
1247
1248         self.failUnlessRaises(Failure, localf.trap, ValueError, KeyError)
1249         self.failUnlessRaises(Failure, localf.trap, ServerFailure)
1250         self.failUnlessEqual(localf.trap(IndexError, KeyError), IndexError)
1251         self.failUnlessEqual(rrefutil.trap_local(localf, IndexError, KeyError),
1252                              IndexError)
1253         self.failUnlessRaises(Failure,
1254                               rrefutil.trap_remote, localf, ValueError, KeyError)
1255
1256         frr = FakeRemoteReference()
1257         wrr = rrefutil.WrappedRemoteReference(frr)
1258         d = wrr.callRemote("oops")
1259         def _check(f):
1260             self.failUnlessRaises(Failure,
1261                                   f.trap, ValueError, KeyError)
1262             self.failUnlessRaises(Failure,
1263                                   f.trap, IndexError)
1264             self.failUnlessEqual(f.trap(ServerFailure), ServerFailure)
1265             self.failUnlessRaises(Failure,
1266                                   rrefutil.trap_remote, f, ValueError, KeyError)
1267             self.failUnlessEqual(rrefutil.trap_remote(f, IndexError, KeyError),
1268                                  IndexError)
1269             self.failUnlessRaises(Failure,
1270                                   rrefutil.trap_local, f, ValueError, KeyError)
1271             self.failUnlessRaises(Failure,
1272                                   rrefutil.trap_local, f, IndexError)
1273         d.addErrback(_check)
1274         return d