src/allmydata/test/test_util.py

   1
   2 def foo(): pass # keep the line number constant
   3
   4 import os, time
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.python import failure
   8
   9 from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
  10 from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
  11 from allmydata.util import limiter, time_format, pollmixin, cachedir
  12
  13 class Base32(unittest.TestCase):
  14     def test_b2a_matches_Pythons(self):
  15         import base64
  16         y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0"
  17         x = base64.b32encode(y)
  18         while x and x[-1] == '=':
  19             x = x[:-1]
  20         x = x.lower()
  21         self.failUnlessEqual(base32.b2a(y), x)
  22     def test_b2a(self):
  23         self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a")
  24     def test_b2a_or_none(self):
  25         self.failUnlessEqual(base32.b2a_or_none(None), None)
  26         self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a")
  27     def test_a2b(self):
  28         self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34")
  29         self.failUnlessRaises(AssertionError, base32.a2b, "b0gus")
  30
  31 class IDLib(unittest.TestCase):
  32     def test_nodeid_b2a(self):
  33         self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
  34
  35 class NoArgumentException(Exception):
  36     def __init__(self):
  37         pass
  38
  39 class HumanReadable(unittest.TestCase):
  40     def test_repr(self):
  41         hr = humanreadable.hr
  42         self.failUnlessEqual(hr(foo), "<foo() at test_util.py:2>")
  43         self.failUnlessEqual(hr(self.test_repr),
  44                              "<bound method HumanReadable.test_repr of <allmydata.test.test_util.HumanReadable testMethod=test_repr>>")
  45         self.failUnlessEqual(hr(1L), "1")
  46         self.failUnlessEqual(hr(10**40),
  47                              "100000000000000000...000000000000000000")
  48         self.failUnlessEqual(hr(self), "<allmydata.test.test_util.HumanReadable testMethod=test_repr>")
  49         self.failUnlessEqual(hr([1,2]), "[1, 2]")
  50         self.failUnlessEqual(hr({1:2}), "{1:2}")
  51         try:
  52             raise RuntimeError
  53         except Exception, e:
  54             self.failUnless(
  55                 hr(e) == "<RuntimeError: ()>" # python-2.4
  56                 or hr(e) == "RuntimeError()") # python-2.5
  57         try:
  58             raise RuntimeError("oops")
  59         except Exception, e:
  60             self.failUnless(
  61                 hr(e) == "<RuntimeError: 'oops'>" # python-2.4
  62                 or hr(e) == "RuntimeError('oops',)") # python-2.5
  63         try:
  64             raise NoArgumentException
  65         except Exception, e:
  66             self.failUnless(
  67                 hr(e) == "<NoArgumentException>" # python-2.4
  68                 or hr(e) == "NoArgumentException()") # python-2.5
  69
  70
  71 class MyList(list):
  72     pass
  73
  74 class Math(unittest.TestCase):
  75     def test_div_ceil(self):
  76         f = mathutil.div_ceil
  77         self.failUnlessEqual(f(0, 1), 0)
  78         self.failUnlessEqual(f(0, 2), 0)
  79         self.failUnlessEqual(f(0, 3), 0)
  80         self.failUnlessEqual(f(1, 3), 1)
  81         self.failUnlessEqual(f(2, 3), 1)
  82         self.failUnlessEqual(f(3, 3), 1)
  83         self.failUnlessEqual(f(4, 3), 2)
  84         self.failUnlessEqual(f(5, 3), 2)
  85         self.failUnlessEqual(f(6, 3), 2)
  86         self.failUnlessEqual(f(7, 3), 3)
  87
  88     def test_next_multiple(self):
  89         f = mathutil.next_multiple
  90         self.failUnlessEqual(f(5, 1), 5)
  91         self.failUnlessEqual(f(5, 2), 6)
  92         self.failUnlessEqual(f(5, 3), 6)
  93         self.failUnlessEqual(f(5, 4), 8)
  94         self.failUnlessEqual(f(5, 5), 5)
  95         self.failUnlessEqual(f(5, 6), 6)
  96         self.failUnlessEqual(f(32, 1), 32)
  97         self.failUnlessEqual(f(32, 2), 32)
  98         self.failUnlessEqual(f(32, 3), 33)
  99         self.failUnlessEqual(f(32, 4), 32)
 100         self.failUnlessEqual(f(32, 5), 35)
 101         self.failUnlessEqual(f(32, 6), 36)
 102         self.failUnlessEqual(f(32, 7), 35)
 103         self.failUnlessEqual(f(32, 8), 32)
 104         self.failUnlessEqual(f(32, 9), 36)
 105         self.failUnlessEqual(f(32, 10), 40)
 106         self.failUnlessEqual(f(32, 11), 33)
 107         self.failUnlessEqual(f(32, 12), 36)
 108         self.failUnlessEqual(f(32, 13), 39)
 109         self.failUnlessEqual(f(32, 14), 42)
 110         self.failUnlessEqual(f(32, 15), 45)
 111         self.failUnlessEqual(f(32, 16), 32)
 112         self.failUnlessEqual(f(32, 17), 34)
 113         self.failUnlessEqual(f(32, 18), 36)
 114         self.failUnlessEqual(f(32, 589), 589)
 115
 116     def test_pad_size(self):
 117         f = mathutil.pad_size
 118         self.failUnlessEqual(f(0, 4), 0)
 119         self.failUnlessEqual(f(1, 4), 3)
 120         self.failUnlessEqual(f(2, 4), 2)
 121         self.failUnlessEqual(f(3, 4), 1)
 122         self.failUnlessEqual(f(4, 4), 0)
 123         self.failUnlessEqual(f(5, 4), 3)
 124
 125     def test_is_power_of_k(self):
 126         f = mathutil.is_power_of_k
 127         for i in range(1, 100):
 128             if i in (1, 2, 4, 8, 16, 32, 64):
 129                 self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
 130             else:
 131                 self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
 132         for i in range(1, 100):
 133             if i in (1, 3, 9, 27, 81):
 134                 self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
 135             else:
 136                 self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
 137
 138     def test_next_power_of_k(self):
 139         f = mathutil.next_power_of_k
 140         self.failUnlessEqual(f(0,2), 1)
 141         self.failUnlessEqual(f(1,2), 1)
 142         self.failUnlessEqual(f(2,2), 2)
 143         self.failUnlessEqual(f(3,2), 4)
 144         self.failUnlessEqual(f(4,2), 4)
 145         for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
 146         for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
 147         for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
 148         for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
 149         for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
 150
 151         self.failUnlessEqual(f(0,3), 1)
 152         self.failUnlessEqual(f(1,3), 1)
 153         self.failUnlessEqual(f(2,3), 3)
 154         self.failUnlessEqual(f(3,3), 3)
 155         for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
 156         for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
 157         for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
 158         for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
 159
 160     def test_ave(self):
 161         f = mathutil.ave
 162         self.failUnlessEqual(f([1,2,3]), 2)
 163         self.failUnlessEqual(f([0,0,0,4]), 1)
 164         self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
 165
 166
 167 class Asserts(unittest.TestCase):
 168     def should_assert(self, func, *args, **kwargs):
 169         try:
 170             func(*args, **kwargs)
 171         except AssertionError, e:
 172             return str(e)
 173         except Exception, e:
 174             self.fail("assert failed with non-AssertionError: %s" % e)
 175         self.fail("assert was not caught")
 176
 177     def should_not_assert(self, func, *args, **kwargs):
 178         if "re" in kwargs:
 179             regexp = kwargs["re"]
 180             del kwargs["re"]
 181         try:
 182             func(*args, **kwargs)
 183         except AssertionError, e:
 184             self.fail("assertion fired when it should not have: %s" % e)
 185         except Exception, e:
 186             self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e)
 187         return # we're happy
 188
 189
 190     def test_assert(self):
 191         f = assertutil._assert
 192         self.should_assert(f)
 193         self.should_assert(f, False)
 194         self.should_not_assert(f, True)
 195
 196         m = self.should_assert(f, False, "message")
 197         self.failUnlessEqual(m, "'message' <type 'str'>", m)
 198         m = self.should_assert(f, False, "message1", othermsg=12)
 199         self.failUnlessEqual("'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 200         m = self.should_assert(f, False, othermsg="message2")
 201         self.failUnlessEqual("othermsg: 'message2' <type 'str'>", m)
 202
 203     def test_precondition(self):
 204         f = assertutil.precondition
 205         self.should_assert(f)
 206         self.should_assert(f, False)
 207         self.should_not_assert(f, True)
 208
 209         m = self.should_assert(f, False, "message")
 210         self.failUnlessEqual("precondition: 'message' <type 'str'>", m)
 211         m = self.should_assert(f, False, "message1", othermsg=12)
 212         self.failUnlessEqual("precondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 213         m = self.should_assert(f, False, othermsg="message2")
 214         self.failUnlessEqual("precondition: othermsg: 'message2' <type 'str'>", m)
 215
 216     def test_postcondition(self):
 217         f = assertutil.postcondition
 218         self.should_assert(f)
 219         self.should_assert(f, False)
 220         self.should_not_assert(f, True)
 221
 222         m = self.should_assert(f, False, "message")
 223         self.failUnlessEqual("postcondition: 'message' <type 'str'>", m)
 224         m = self.should_assert(f, False, "message1", othermsg=12)
 225         self.failUnlessEqual("postcondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
 226         m = self.should_assert(f, False, othermsg="message2")
 227         self.failUnlessEqual("postcondition: othermsg: 'message2' <type 'str'>", m)
 228
 229 class FileUtil(unittest.TestCase):
 230     def mkdir(self, basedir, path, mode=0777):
 231         fn = os.path.join(basedir, path)
 232         fileutil.make_dirs(fn, mode)
 233
 234     def touch(self, basedir, path, mode=None, data="touch\n"):
 235         fn = os.path.join(basedir, path)
 236         f = open(fn, "w")
 237         f.write(data)
 238         f.close()
 239         if mode is not None:
 240             os.chmod(fn, mode)
 241
 242     def test_rm_dir(self):
 243         basedir = "util/FileUtil/test_rm_dir"
 244         fileutil.make_dirs(basedir)
 245         # create it again to test idempotency
 246         fileutil.make_dirs(basedir)
 247         d = os.path.join(basedir, "doomed")
 248         self.mkdir(d, "a/b")
 249         self.touch(d, "a/b/1.txt")
 250         self.touch(d, "a/b/2.txt", 0444)
 251         self.touch(d, "a/b/3.txt", 0)
 252         self.mkdir(d, "a/c")
 253         self.touch(d, "a/c/1.txt")
 254         self.touch(d, "a/c/2.txt", 0444)
 255         self.touch(d, "a/c/3.txt", 0)
 256         os.chmod(os.path.join(d, "a/c"), 0444)
 257         self.mkdir(d, "a/d")
 258         self.touch(d, "a/d/1.txt")
 259         self.touch(d, "a/d/2.txt", 0444)
 260         self.touch(d, "a/d/3.txt", 0)
 261         os.chmod(os.path.join(d, "a/d"), 0)
 262
 263         fileutil.rm_dir(d)
 264         self.failIf(os.path.exists(d))
 265         # remove it again to test idempotency
 266         fileutil.rm_dir(d)
 267
 268     def test_remove_if_possible(self):
 269         basedir = "util/FileUtil/test_remove_if_possible"
 270         fileutil.make_dirs(basedir)
 271         self.touch(basedir, "here")
 272         fn = os.path.join(basedir, "here")
 273         fileutil.remove_if_possible(fn)
 274         self.failIf(os.path.exists(fn))
 275         fileutil.remove_if_possible(fn) # should be idempotent
 276         fileutil.rm_dir(basedir)
 277         fileutil.remove_if_possible(fn) # should survive errors
 278
 279     def test_open_or_create(self):
 280         basedir = "util/FileUtil/test_open_or_create"
 281         fileutil.make_dirs(basedir)
 282         fn = os.path.join(basedir, "here")
 283         f = fileutil.open_or_create(fn)
 284         f.write("stuff.")
 285         f.close()
 286         f = fileutil.open_or_create(fn)
 287         f.seek(0, 2)
 288         f.write("more.")
 289         f.close()
 290         f = open(fn, "r")
 291         data = f.read()
 292         f.close()
 293         self.failUnlessEqual(data, "stuff.more.")
 294
 295     def test_NamedTemporaryDirectory(self):
 296         basedir = "util/FileUtil/test_NamedTemporaryDirectory"
 297         fileutil.make_dirs(basedir)
 298         td = fileutil.NamedTemporaryDirectory(dir=basedir)
 299         name = td.name
 300         self.failUnless(basedir in name)
 301         self.failUnless(basedir in repr(td))
 302         self.failUnless(os.path.isdir(name))
 303         del td
 304         # it is conceivable that we need to force gc here, but I'm not sure
 305         self.failIf(os.path.isdir(name))
 306
 307     def test_rename(self):
 308         basedir = "util/FileUtil/test_rename"
 309         fileutil.make_dirs(basedir)
 310         self.touch(basedir, "here")
 311         fn = os.path.join(basedir, "here")
 312         fn2 = os.path.join(basedir, "there")
 313         fileutil.rename(fn, fn2)
 314         self.failIf(os.path.exists(fn))
 315         self.failUnless(os.path.exists(fn2))
 316
 317     def test_du(self):
 318         basedir = "util/FileUtil/test_du"
 319         fileutil.make_dirs(basedir)
 320         d = os.path.join(basedir, "space-consuming")
 321         self.mkdir(d, "a/b")
 322         self.touch(d, "a/b/1.txt", data="a"*10)
 323         self.touch(d, "a/b/2.txt", data="b"*11)
 324         self.mkdir(d, "a/c")
 325         self.touch(d, "a/c/1.txt", data="c"*12)
 326         self.touch(d, "a/c/2.txt", data="d"*13)
 327
 328         used = fileutil.du(basedir)
 329         self.failUnlessEqual(10+11+12+13, used)
 330
 331 class PollMixinTests(unittest.TestCase):
 332     def setUp(self):
 333         self.pm = pollmixin.PollMixin()
 334
 335     def test_PollMixin_True(self):
 336         d = self.pm.poll(check_f=lambda : True,
 337                          pollinterval=0.1)
 338         return d
 339
 340     def test_PollMixin_False_then_True(self):
 341         i = iter([False, True])
 342         d = self.pm.poll(check_f=i.next,
 343                          pollinterval=0.1)
 344         return d
 345
 346     def test_timeout(self):
 347         d = self.pm.poll(check_f=lambda: False,
 348                          pollinterval=0.01,
 349                          timeout=1)
 350         def _suc(res):
 351             self.fail("poll should have failed, not returned %s" % (res,))
 352         def _err(f):
 353             f.trap(pollmixin.TimeoutError)
 354             return None # success
 355         d.addCallbacks(_suc, _err)
 356         return d
 357
 358 class DeferredUtilTests(unittest.TestCase):
 359     def test_success(self):
 360         d1, d2 = defer.Deferred(), defer.Deferred()
 361         good = []
 362         bad = []
 363         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 364         dlss.addCallbacks(good.append, bad.append)
 365         d1.callback(1)
 366         d2.callback(2)
 367         self.failUnlessEqual(good, [[1,2]])
 368         self.failUnlessEqual(bad, [])
 369
 370     def test_failure(self):
 371         d1, d2 = defer.Deferred(), defer.Deferred()
 372         good = []
 373         bad = []
 374         dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
 375         dlss.addCallbacks(good.append, bad.append)
 376         d1.addErrback(lambda _ignore: None)
 377         d2.addErrback(lambda _ignore: None)
 378         d1.callback(1)
 379         d2.errback(RuntimeError())
 380         self.failUnlessEqual(good, [])
 381         self.failUnlessEqual(len(bad), 1)
 382         f = bad[0]
 383         self.failUnless(isinstance(f, failure.Failure))
 384         self.failUnless(f.check(RuntimeError))
 385
 386 class HashUtilTests(unittest.TestCase):
 387
 388     def test_random_key(self):
 389         k = hashutil.random_key()
 390         self.failUnlessEqual(len(k), hashutil.KEYLEN)
 391
 392     def test_sha256d(self):
 393         h1 = hashutil.tagged_hash("tag1", "value")
 394         h2 = hashutil.tagged_hasher("tag1")
 395         h2.update("value")
 396         h2a = h2.digest()
 397         h2b = h2.digest()
 398         self.failUnlessEqual(h1, h2a)
 399         self.failUnlessEqual(h2a, h2b)
 400
 401     def test_sha256d_truncated(self):
 402         h1 = hashutil.tagged_hash("tag1", "value", 16)
 403         h2 = hashutil.tagged_hasher("tag1", 16)
 404         h2.update("value")
 405         h2 = h2.digest()
 406         self.failUnlessEqual(len(h1), 16)
 407         self.failUnlessEqual(len(h2), 16)
 408         self.failUnlessEqual(h1, h2)
 409
 410     def test_chk(self):
 411         h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret")
 412         h2 = hashutil.convergence_hasher(3, 10, 1000, "secret")
 413         h2.update("data")
 414         h2 = h2.digest()
 415         self.failUnlessEqual(h1, h2)
 416
 417     def test_hashers(self):
 418         h1 = hashutil.block_hash("foo")
 419         h2 = hashutil.block_hasher()
 420         h2.update("foo")
 421         self.failUnlessEqual(h1, h2.digest())
 422
 423         h1 = hashutil.uri_extension_hash("foo")
 424         h2 = hashutil.uri_extension_hasher()
 425         h2.update("foo")
 426         self.failUnlessEqual(h1, h2.digest())
 427
 428         h1 = hashutil.plaintext_hash("foo")
 429         h2 = hashutil.plaintext_hasher()
 430         h2.update("foo")
 431         self.failUnlessEqual(h1, h2.digest())
 432
 433         h1 = hashutil.crypttext_hash("foo")
 434         h2 = hashutil.crypttext_hasher()
 435         h2.update("foo")
 436         self.failUnlessEqual(h1, h2.digest())
 437
 438         h1 = hashutil.crypttext_segment_hash("foo")
 439         h2 = hashutil.crypttext_segment_hasher()
 440         h2.update("foo")
 441         self.failUnlessEqual(h1, h2.digest())
 442
 443         h1 = hashutil.plaintext_segment_hash("foo")
 444         h2 = hashutil.plaintext_segment_hasher()
 445         h2.update("foo")
 446         self.failUnlessEqual(h1, h2.digest())
 447
 448 class Abbreviate(unittest.TestCase):
 449     def test_time(self):
 450         a = abbreviate.abbreviate_time
 451         self.failUnlessEqual(a(None), "unknown")
 452         self.failUnlessEqual(a(0), "0 seconds")
 453         self.failUnlessEqual(a(1), "1 second")
 454         self.failUnlessEqual(a(2), "2 seconds")
 455         self.failUnlessEqual(a(119), "119 seconds")
 456         MIN = 60
 457         self.failUnlessEqual(a(2*MIN), "2 minutes")
 458         self.failUnlessEqual(a(60*MIN), "60 minutes")
 459         self.failUnlessEqual(a(179*MIN), "179 minutes")
 460         HOUR = 60*MIN
 461         self.failUnlessEqual(a(180*MIN), "3 hours")
 462         self.failUnlessEqual(a(4*HOUR), "4 hours")
 463         DAY = 24*HOUR
 464         MONTH = 30*DAY
 465         self.failUnlessEqual(a(2*DAY), "2 days")
 466         self.failUnlessEqual(a(2*MONTH), "2 months")
 467         YEAR = 365*DAY
 468         self.failUnlessEqual(a(5*YEAR), "5 years")
 469
 470     def test_space(self):
 471         tests_si = [(None, "unknown"),
 472                     (0, "0 B"),
 473                     (1, "1 B"),
 474                     (999, "999 B"),
 475                     (1000, "1000 B"),
 476                     (1023, "1023 B"),
 477                     (1024, "1.02 kB"),
 478                     (20*1000, "20.00 kB"),
 479                     (1024*1024, "1.05 MB"),
 480                     (1000*1000, "1.00 MB"),
 481                     (1000*1000*1000, "1.00 GB"),
 482                     (1000*1000*1000*1000, "1.00 TB"),
 483                     (1000*1000*1000*1000*1000, "1.00 PB"),
 484                     (1234567890123456, "1.23 PB"),
 485                     ]
 486         for (x, expected) in tests_si:
 487             got = abbreviate.abbreviate_space(x, SI=True)
 488             self.failUnlessEqual(got, expected)
 489
 490         tests_base1024 = [(None, "unknown"),
 491                           (0, "0 B"),
 492                           (1, "1 B"),
 493                           (999, "999 B"),
 494                           (1000, "1000 B"),
 495                           (1023, "1023 B"),
 496                           (1024, "1.00 kiB"),
 497                           (20*1024, "20.00 kiB"),
 498                           (1000*1000, "976.56 kiB"),
 499                           (1024*1024, "1.00 MiB"),
 500                           (1024*1024*1024, "1.00 GiB"),
 501                           (1024*1024*1024*1024, "1.00 TiB"),
 502                           (1000*1000*1000*1000*1000, "909.49 TiB"),
 503                           (1024*1024*1024*1024*1024, "1.00 PiB"),
 504                           (1234567890123456, "1.10 PiB"),
 505                     ]
 506         for (x, expected) in tests_base1024:
 507             got = abbreviate.abbreviate_space(x, SI=False)
 508             self.failUnlessEqual(got, expected)
 509
 510         self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567),
 511                              "(1.23 MB, 1.18 MiB)")
 512
 513     def test_parse_space(self):
 514         p = abbreviate.parse_abbreviated_size
 515         self.failUnlessEqual(p(""), None)
 516         self.failUnlessEqual(p(None), None)
 517         self.failUnlessEqual(p("123"), 123)
 518         self.failUnlessEqual(p("123B"), 123)
 519         self.failUnlessEqual(p("2K"), 2000)
 520         self.failUnlessEqual(p("2kb"), 2000)
 521         self.failUnlessEqual(p("2KiB"), 2048)
 522         self.failUnlessEqual(p("10MB"), 10*1000*1000)
 523         self.failUnlessEqual(p("10MiB"), 10*1024*1024)
 524         self.failUnlessEqual(p("5G"), 5*1000*1000*1000)
 525         self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024)
 526         e = self.failUnlessRaises(ValueError, p, "12 cubits")
 527         self.failUnless("12 cubits" in str(e))
 528
 529 class Limiter(unittest.TestCase):
 530     def job(self, i, foo):
 531         self.calls.append( (i, foo) )
 532         self.simultaneous += 1
 533         self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous)
 534         d = defer.Deferred()
 535         def _done():
 536             self.simultaneous -= 1
 537             d.callback("done %d" % i)
 538         reactor.callLater(1.0, _done)
 539         return d
 540
 541     def bad_job(self, i, foo):
 542         raise RuntimeError("bad_job %d" % i)
 543
 544     def test_limiter(self):
 545         self.calls = []
 546         self.simultaneous = 0
 547         self.peak_simultaneous = 0
 548         l = limiter.ConcurrencyLimiter()
 549         dl = []
 550         for i in range(20):
 551             dl.append(l.add(self.job, i, foo=str(i)))
 552         d = defer.DeferredList(dl, fireOnOneErrback=True)
 553         def _done(res):
 554             self.failUnlessEqual(self.simultaneous, 0)
 555             self.failUnless(self.peak_simultaneous <= 10)
 556             self.failUnlessEqual(len(self.calls), 20)
 557             for i in range(20):
 558                 self.failUnless( (i, str(i)) in self.calls)
 559         d.addCallback(_done)
 560         return d
 561
 562     def test_errors(self):
 563         self.calls = []
 564         self.simultaneous = 0
 565         self.peak_simultaneous = 0
 566         l = limiter.ConcurrencyLimiter()
 567         dl = []
 568         for i in range(20):
 569             dl.append(l.add(self.job, i, foo=str(i)))
 570         d2 = l.add(self.bad_job, 21, "21")
 571         d = defer.DeferredList(dl, fireOnOneErrback=True)
 572         def _most_done(res):
 573             results = []
 574             for (success, result) in res:
 575                 self.failUnlessEqual(success, True)
 576                 results.append(result)
 577             results.sort()
 578             expected_results = ["done %d" % i for i in range(20)]
 579             expected_results.sort()
 580             self.failUnlessEqual(results, expected_results)
 581             self.failUnless(self.peak_simultaneous <= 10)
 582             self.failUnlessEqual(len(self.calls), 20)
 583             for i in range(20):
 584                 self.failUnless( (i, str(i)) in self.calls)
 585             def _good(res):
 586                 self.fail("should have failed, not got %s" % (res,))
 587             def _err(f):
 588                 f.trap(RuntimeError)
 589                 self.failUnless("bad_job 21" in str(f))
 590             d2.addCallbacks(_good, _err)
 591             return d2
 592         d.addCallback(_most_done)
 593         def _all_done(res):
 594             self.failUnlessEqual(self.simultaneous, 0)
 595             self.failUnless(self.peak_simultaneous <= 10)
 596             self.failUnlessEqual(len(self.calls), 20)
 597             for i in range(20):
 598                 self.failUnless( (i, str(i)) in self.calls)
 599         d.addCallback(_all_done)
 600         return d
 601
 602 class TimeFormat(unittest.TestCase):
 603     def test_epoch(self):
 604         s = time_format.iso_utc_time_to_localseconds("1970-01-01T00:00:01")
 605         self.failUnlessEqual(s, 1.0)
 606         s = time_format.iso_utc_time_to_localseconds("1970-01-01_00:00:01")
 607         self.failUnlessEqual(s, 1.0)
 608         s = time_format.iso_utc_time_to_localseconds("1970-01-01 00:00:01")
 609         self.failUnlessEqual(s, 1.0)
 610
 611         self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01")
 612         self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "),
 613                              "1970-01-01 00:00:01")
 614         now = time.time()
 615         def my_time():
 616             return 1.0
 617         self.failUnlessEqual(time_format.iso_utc(t=my_time),
 618                              "1970-01-01_00:00:01")
 619         e = self.failUnlessRaises(ValueError,
 620                                   time_format.iso_utc_time_to_localseconds,
 621                                   "invalid timestring")
 622         self.failUnless("not a complete ISO8601 timestamp" in str(e))
 623         s = time_format.iso_utc_time_to_localseconds("1970-01-01_00:00:01.500")
 624         self.failUnlessEqual(s, 1.5)
 625
 626 class CacheDir(unittest.TestCase):
 627     def test_basic(self):
 628         basedir = "test_util/CacheDir/test_basic"
 629
 630         def _failIfExists(name):
 631             absfn = os.path.join(basedir, name)
 632             self.failIf(os.path.exists(absfn),
 633                         "%s exists but it shouldn't" % absfn)
 634
 635         def _failUnlessExists(name):
 636             absfn = os.path.join(basedir, name)
 637             self.failUnless(os.path.exists(absfn),
 638                             "%s doesn't exist but it should" % absfn)
 639
 640         cdm = cachedir.CacheDirectoryManager(basedir)
 641         a = cdm.get_file("a")
 642         b = cdm.get_file("b")
 643         c = cdm.get_file("c")
 644         f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f
 645         f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f
 646         f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f
 647
 648         _failUnlessExists("a")
 649         _failUnlessExists("b")
 650         _failUnlessExists("c")
 651
 652         cdm.check()
 653
 654         _failUnlessExists("a")
 655         _failUnlessExists("b")
 656         _failUnlessExists("c")
 657
 658         del a
 659         # this file won't be deleted yet, because it isn't old enough
 660         cdm.check()
 661         _failUnlessExists("a")
 662         _failUnlessExists("b")
 663         _failUnlessExists("c")
 664
 665         # we change the definition of "old" to make everything old
 666         cdm.old = -10
 667
 668         cdm.check()
 669         _failIfExists("a")
 670         _failUnlessExists("b")
 671         _failUnlessExists("c")
 672
 673         cdm.old = 60*60
 674
 675         del b
 676
 677         cdm.check()
 678         _failIfExists("a")
 679         _failUnlessExists("b")
 680         _failUnlessExists("c")
 681
 682         b2 = cdm.get_file("b")
 683
 684         cdm.check()
 685         _failIfExists("a")
 686         _failUnlessExists("b")
 687         _failUnlessExists("c")