2 def foo(): pass # keep the line number constant
5 from StringIO import StringIO
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.python.failure import Failure
9 from twisted.python import log
11 from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
12 from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
13 from allmydata.util import limiter, time_format, pollmixin, cachedir
14 from allmydata.util import statistics, dictutil, rrefutil, pipeline
15 from allmydata.util.rrefutil import ServerFailure
17 class Base32(unittest.TestCase):
18 def test_b2a_matches_Pythons(self):
20 y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0"
21 x = base64.b32encode(y)
22 while x and x[-1] == '=':
25 self.failUnlessEqual(base32.b2a(y), x)
27 self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a")
28 def test_b2a_or_none(self):
29 self.failUnlessEqual(base32.b2a_or_none(None), None)
30 self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a")
32 self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34")
33 self.failUnlessRaises(AssertionError, base32.a2b, "b0gus")
35 class IDLib(unittest.TestCase):
36 def test_nodeid_b2a(self):
37 self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
39 class NoArgumentException(Exception):
43 class HumanReadable(unittest.TestCase):
46 self.failUnlessEqual(hr(foo), "<foo() at test_util.py:2>")
47 self.failUnlessEqual(hr(self.test_repr),
48 "<bound method HumanReadable.test_repr of <allmydata.test.test_util.HumanReadable testMethod=test_repr>>")
49 self.failUnlessEqual(hr(1L), "1")
50 self.failUnlessEqual(hr(10**40),
51 "100000000000000000...000000000000000000")
52 self.failUnlessEqual(hr(self), "<allmydata.test.test_util.HumanReadable testMethod=test_repr>")
53 self.failUnlessEqual(hr([1,2]), "[1, 2]")
54 self.failUnlessEqual(hr({1:2}), "{1:2}")
59 hr(e) == "<ValueError: ()>" # python-2.4
60 or hr(e) == "ValueError()") # python-2.5
62 raise ValueError("oops")
65 hr(e) == "<ValueError: 'oops'>" # python-2.4
66 or hr(e) == "ValueError('oops',)") # python-2.5
68 raise NoArgumentException
71 hr(e) == "<NoArgumentException>" # python-2.4
72 or hr(e) == "NoArgumentException()") # python-2.5
78 class Math(unittest.TestCase):
79 def test_div_ceil(self):
81 self.failUnlessEqual(f(0, 1), 0)
82 self.failUnlessEqual(f(0, 2), 0)
83 self.failUnlessEqual(f(0, 3), 0)
84 self.failUnlessEqual(f(1, 3), 1)
85 self.failUnlessEqual(f(2, 3), 1)
86 self.failUnlessEqual(f(3, 3), 1)
87 self.failUnlessEqual(f(4, 3), 2)
88 self.failUnlessEqual(f(5, 3), 2)
89 self.failUnlessEqual(f(6, 3), 2)
90 self.failUnlessEqual(f(7, 3), 3)
92 def test_next_multiple(self):
93 f = mathutil.next_multiple
94 self.failUnlessEqual(f(5, 1), 5)
95 self.failUnlessEqual(f(5, 2), 6)
96 self.failUnlessEqual(f(5, 3), 6)
97 self.failUnlessEqual(f(5, 4), 8)
98 self.failUnlessEqual(f(5, 5), 5)
99 self.failUnlessEqual(f(5, 6), 6)
100 self.failUnlessEqual(f(32, 1), 32)
101 self.failUnlessEqual(f(32, 2), 32)
102 self.failUnlessEqual(f(32, 3), 33)
103 self.failUnlessEqual(f(32, 4), 32)
104 self.failUnlessEqual(f(32, 5), 35)
105 self.failUnlessEqual(f(32, 6), 36)
106 self.failUnlessEqual(f(32, 7), 35)
107 self.failUnlessEqual(f(32, 8), 32)
108 self.failUnlessEqual(f(32, 9), 36)
109 self.failUnlessEqual(f(32, 10), 40)
110 self.failUnlessEqual(f(32, 11), 33)
111 self.failUnlessEqual(f(32, 12), 36)
112 self.failUnlessEqual(f(32, 13), 39)
113 self.failUnlessEqual(f(32, 14), 42)
114 self.failUnlessEqual(f(32, 15), 45)
115 self.failUnlessEqual(f(32, 16), 32)
116 self.failUnlessEqual(f(32, 17), 34)
117 self.failUnlessEqual(f(32, 18), 36)
118 self.failUnlessEqual(f(32, 589), 589)
120 def test_pad_size(self):
121 f = mathutil.pad_size
122 self.failUnlessEqual(f(0, 4), 0)
123 self.failUnlessEqual(f(1, 4), 3)
124 self.failUnlessEqual(f(2, 4), 2)
125 self.failUnlessEqual(f(3, 4), 1)
126 self.failUnlessEqual(f(4, 4), 0)
127 self.failUnlessEqual(f(5, 4), 3)
129 def test_is_power_of_k(self):
130 f = mathutil.is_power_of_k
131 for i in range(1, 100):
132 if i in (1, 2, 4, 8, 16, 32, 64):
133 self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
135 self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
136 for i in range(1, 100):
137 if i in (1, 3, 9, 27, 81):
138 self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
140 self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
142 def test_next_power_of_k(self):
143 f = mathutil.next_power_of_k
144 self.failUnlessEqual(f(0,2), 1)
145 self.failUnlessEqual(f(1,2), 1)
146 self.failUnlessEqual(f(2,2), 2)
147 self.failUnlessEqual(f(3,2), 4)
148 self.failUnlessEqual(f(4,2), 4)
149 for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
150 for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
151 for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
152 for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
153 for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
155 self.failUnlessEqual(f(0,3), 1)
156 self.failUnlessEqual(f(1,3), 1)
157 self.failUnlessEqual(f(2,3), 3)
158 self.failUnlessEqual(f(3,3), 3)
159 for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
160 for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
161 for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
162 for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
166 self.failUnlessEqual(f([1,2,3]), 2)
167 self.failUnlessEqual(f([0,0,0,4]), 1)
168 self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
170 def test_round_sigfigs(self):
171 f = mathutil.round_sigfigs
172 self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002)
174 class Statistics(unittest.TestCase):
175 def should_assert(self, msg, func, *args, **kwargs):
177 func(*args, **kwargs)
179 except AssertionError, e:
182 def failUnlessListEqual(self, a, b, msg = None):
183 self.failUnlessEqual(len(a), len(b))
184 for i in range(len(a)):
185 self.failUnlessEqual(a[i], b[i], msg)
187 def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None):
188 self.failUnlessEqual(len(a), len(b))
189 for i in range(len(a)):
190 self.failUnlessAlmostEqual(a[i], b[i], places, msg)
192 def test_binomial_coeff(self):
193 f = statistics.binomial_coeff
194 self.failUnlessEqual(f(20, 0), 1)
195 self.failUnlessEqual(f(20, 1), 20)
196 self.failUnlessEqual(f(20, 2), 190)
197 self.failUnlessEqual(f(20, 8), f(20, 12))
198 self.should_assert("Should assert if n < k", f, 2, 3)
200 def test_binomial_distribution_pmf(self):
201 f = statistics.binomial_distribution_pmf
204 pmf_stat = [0.81, 0.18, 0.01]
205 self.failUnlessListAlmostEqual(pmf_comp, pmf_stat)
207 # Summing across a PMF should give the total probability 1
208 self.failUnlessAlmostEqual(sum(pmf_comp), 1)
209 self.should_assert("Should assert if not 0<=p<=1", f, 1, -1)
210 self.should_assert("Should assert if n < 1", f, 0, .1)
213 statistics.print_pmf(pmf_comp, out=out)
214 lines = out.getvalue().splitlines()
215 self.failUnlessEqual(lines[0], "i=0: 0.81")
216 self.failUnlessEqual(lines[1], "i=1: 0.18")
217 self.failUnlessEqual(lines[2], "i=2: 0.01")
219 def test_survival_pmf(self):
220 f = statistics.survival_pmf
221 # Cross-check binomial-distribution method against convolution
223 p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20
224 pmf1 = statistics.survival_pmf_via_conv(p_list)
225 pmf2 = statistics.survival_pmf_via_bd(p_list)
226 self.failUnlessListAlmostEqual(pmf1, pmf2)
227 self.failUnlessTrue(statistics.valid_pmf(pmf1))
228 self.should_assert("Should assert if p_i > 1", f, [1.1]);
229 self.should_assert("Should assert if p_i < 0", f, [-.1]);
231 def test_repair_count_pmf(self):
232 survival_pmf = statistics.binomial_distribution_pmf(5, .9)
233 repair_pmf = statistics.repair_count_pmf(survival_pmf, 3)
234 # repair_pmf[0] == sum(survival_pmf[0,1,2,5])
235 # repair_pmf[1] == survival_pmf[4]
236 # repair_pmf[2] = survival_pmf[3]
237 self.failUnlessListAlmostEqual(repair_pmf,
238 [0.00001 + 0.00045 + 0.0081 + 0.59049,
243 def test_repair_cost(self):
244 survival_pmf = statistics.binomial_distribution_pmf(5, .9)
245 bwcost = statistics.bandwidth_cost_function
246 cost = statistics.mean_repair_cost(bwcost, 1000,
247 survival_pmf, 3, ul_dl_ratio=1.0)
248 self.failUnlessAlmostEqual(cost, 558.90)
249 cost = statistics.mean_repair_cost(bwcost, 1000,
250 survival_pmf, 3, ul_dl_ratio=8.0)
251 self.failUnlessAlmostEqual(cost, 1664.55)
253 # I haven't manually checked the math beyond here -warner
254 cost = statistics.eternal_repair_cost(bwcost, 1000,
256 discount_rate=0, ul_dl_ratio=1.0)
257 self.failUnlessAlmostEqual(cost, 65292.056074766246)
258 cost = statistics.eternal_repair_cost(bwcost, 1000,
262 self.failUnlessAlmostEqual(cost, 9133.6097158191551)
264 def test_convolve(self):
265 f = statistics.convolve
269 v1v2result = [ 4, 13, 28, 27, 18 ]
270 # Convolution is commutative
273 self.failUnlessListEqual(r1, r2, "Convolution should be commutative")
274 self.failUnlessListEqual(r1, v1v2result, "Didn't match known result")
275 # Convolution is associative
276 r1 = f(f(v1, v2), v3)
277 r2 = f(v1, f(v2, v3))
278 self.failUnlessListEqual(r1, r2, "Convolution should be associative")
279 # Convolution is distributive
280 r1 = f(v3, [ a + b for a, b in zip(v1, v2) ])
283 r2 = [ a + b for a, b in zip(tmp1, tmp2) ]
284 self.failUnlessListEqual(r1, r2, "Convolution should be distributive")
285 # Convolution is scalar multiplication associative
287 r1 = [ a * 4 for a in tmp1 ]
288 tmp2 = [ a * 4 for a in v1 ]
290 self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative")
292 def test_find_k(self):
293 f = statistics.find_k
294 g = statistics.pr_file_loss
295 plist = [.9] * 10 + [.8] * 10 # N=20
298 self.failUnlessEqual(k, 10)
299 self.failUnless(g(plist, k) < t)
301 def test_pr_file_loss(self):
302 f = statistics.pr_file_loss
304 self.failUnlessEqual(f(plist, 3), .0546875)
306 def test_pr_backup_file_loss(self):
307 f = statistics.pr_backup_file_loss
309 self.failUnlessEqual(f(plist, .5, 3), .02734375)
312 class Asserts(unittest.TestCase):
313 def should_assert(self, func, *args, **kwargs):
315 func(*args, **kwargs)
316 except AssertionError, e:
319 self.fail("assert failed with non-AssertionError: %s" % e)
320 self.fail("assert was not caught")
322 def should_not_assert(self, func, *args, **kwargs):
324 regexp = kwargs["re"]
327 func(*args, **kwargs)
328 except AssertionError, e:
329 self.fail("assertion fired when it should not have: %s" % e)
331 self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e)
335 def test_assert(self):
336 f = assertutil._assert
337 self.should_assert(f)
338 self.should_assert(f, False)
339 self.should_not_assert(f, True)
341 m = self.should_assert(f, False, "message")
342 self.failUnlessEqual(m, "'message' <type 'str'>", m)
343 m = self.should_assert(f, False, "message1", othermsg=12)
344 self.failUnlessEqual("'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
345 m = self.should_assert(f, False, othermsg="message2")
346 self.failUnlessEqual("othermsg: 'message2' <type 'str'>", m)
348 def test_precondition(self):
349 f = assertutil.precondition
350 self.should_assert(f)
351 self.should_assert(f, False)
352 self.should_not_assert(f, True)
354 m = self.should_assert(f, False, "message")
355 self.failUnlessEqual("precondition: 'message' <type 'str'>", m)
356 m = self.should_assert(f, False, "message1", othermsg=12)
357 self.failUnlessEqual("precondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
358 m = self.should_assert(f, False, othermsg="message2")
359 self.failUnlessEqual("precondition: othermsg: 'message2' <type 'str'>", m)
361 def test_postcondition(self):
362 f = assertutil.postcondition
363 self.should_assert(f)
364 self.should_assert(f, False)
365 self.should_not_assert(f, True)
367 m = self.should_assert(f, False, "message")
368 self.failUnlessEqual("postcondition: 'message' <type 'str'>", m)
369 m = self.should_assert(f, False, "message1", othermsg=12)
370 self.failUnlessEqual("postcondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
371 m = self.should_assert(f, False, othermsg="message2")
372 self.failUnlessEqual("postcondition: othermsg: 'message2' <type 'str'>", m)
374 class FileUtil(unittest.TestCase):
375 def mkdir(self, basedir, path, mode=0777):
376 fn = os.path.join(basedir, path)
377 fileutil.make_dirs(fn, mode)
379 def touch(self, basedir, path, mode=None, data="touch\n"):
380 fn = os.path.join(basedir, path)
387 def test_rm_dir(self):
388 basedir = "util/FileUtil/test_rm_dir"
389 fileutil.make_dirs(basedir)
390 # create it again to test idempotency
391 fileutil.make_dirs(basedir)
392 d = os.path.join(basedir, "doomed")
394 self.touch(d, "a/b/1.txt")
395 self.touch(d, "a/b/2.txt", 0444)
396 self.touch(d, "a/b/3.txt", 0)
398 self.touch(d, "a/c/1.txt")
399 self.touch(d, "a/c/2.txt", 0444)
400 self.touch(d, "a/c/3.txt", 0)
401 os.chmod(os.path.join(d, "a/c"), 0444)
403 self.touch(d, "a/d/1.txt")
404 self.touch(d, "a/d/2.txt", 0444)
405 self.touch(d, "a/d/3.txt", 0)
406 os.chmod(os.path.join(d, "a/d"), 0)
409 self.failIf(os.path.exists(d))
410 # remove it again to test idempotency
413 def test_remove_if_possible(self):
414 basedir = "util/FileUtil/test_remove_if_possible"
415 fileutil.make_dirs(basedir)
416 self.touch(basedir, "here")
417 fn = os.path.join(basedir, "here")
418 fileutil.remove_if_possible(fn)
419 self.failIf(os.path.exists(fn))
420 fileutil.remove_if_possible(fn) # should be idempotent
421 fileutil.rm_dir(basedir)
422 fileutil.remove_if_possible(fn) # should survive errors
424 def test_open_or_create(self):
425 basedir = "util/FileUtil/test_open_or_create"
426 fileutil.make_dirs(basedir)
427 fn = os.path.join(basedir, "here")
428 f = fileutil.open_or_create(fn)
431 f = fileutil.open_or_create(fn)
438 self.failUnlessEqual(data, "stuff.more.")
440 def test_NamedTemporaryDirectory(self):
441 basedir = "util/FileUtil/test_NamedTemporaryDirectory"
442 fileutil.make_dirs(basedir)
443 td = fileutil.NamedTemporaryDirectory(dir=basedir)
445 self.failUnless(basedir in name)
446 self.failUnless(basedir in repr(td))
447 self.failUnless(os.path.isdir(name))
449 # it is conceivable that we need to force gc here, but I'm not sure
450 self.failIf(os.path.isdir(name))
452 def test_rename(self):
453 basedir = "util/FileUtil/test_rename"
454 fileutil.make_dirs(basedir)
455 self.touch(basedir, "here")
456 fn = os.path.join(basedir, "here")
457 fn2 = os.path.join(basedir, "there")
458 fileutil.rename(fn, fn2)
459 self.failIf(os.path.exists(fn))
460 self.failUnless(os.path.exists(fn2))
463 basedir = "util/FileUtil/test_du"
464 fileutil.make_dirs(basedir)
465 d = os.path.join(basedir, "space-consuming")
467 self.touch(d, "a/b/1.txt", data="a"*10)
468 self.touch(d, "a/b/2.txt", data="b"*11)
470 self.touch(d, "a/c/1.txt", data="c"*12)
471 self.touch(d, "a/c/2.txt", data="d"*13)
473 used = fileutil.du(basedir)
474 self.failUnlessEqual(10+11+12+13, used)
476 class PollMixinTests(unittest.TestCase):
478 self.pm = pollmixin.PollMixin()
480 def test_PollMixin_True(self):
481 d = self.pm.poll(check_f=lambda : True,
485 def test_PollMixin_False_then_True(self):
486 i = iter([False, True])
487 d = self.pm.poll(check_f=i.next,
491 def test_timeout(self):
492 d = self.pm.poll(check_f=lambda: False,
496 self.fail("poll should have failed, not returned %s" % (res,))
498 f.trap(pollmixin.TimeoutError)
499 return None # success
500 d.addCallbacks(_suc, _err)
503 class DeferredUtilTests(unittest.TestCase):
504 def test_gather_results(self):
505 d1 = defer.Deferred()
506 d2 = defer.Deferred()
507 res = deferredutil.gatherResults([d1, d2])
508 d1.errback(ValueError("BAD"))
510 self.fail("Should have errbacked, not resulted in %s" % (res,))
512 thef.trap(ValueError)
513 res.addCallbacks(_callb, _errb)
516 def test_success(self):
517 d1, d2 = defer.Deferred(), defer.Deferred()
520 dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
521 dlss.addCallbacks(good.append, bad.append)
524 self.failUnlessEqual(good, [[1,2]])
525 self.failUnlessEqual(bad, [])
527 def test_failure(self):
528 d1, d2 = defer.Deferred(), defer.Deferred()
531 dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
532 dlss.addCallbacks(good.append, bad.append)
533 d1.addErrback(lambda _ignore: None)
534 d2.addErrback(lambda _ignore: None)
536 d2.errback(ValueError())
537 self.failUnlessEqual(good, [])
538 self.failUnlessEqual(len(bad), 1)
540 self.failUnless(isinstance(f, Failure))
541 self.failUnless(f.check(ValueError))
543 class HashUtilTests(unittest.TestCase):
545 def test_random_key(self):
546 k = hashutil.random_key()
547 self.failUnlessEqual(len(k), hashutil.KEYLEN)
549 def test_sha256d(self):
550 h1 = hashutil.tagged_hash("tag1", "value")
551 h2 = hashutil.tagged_hasher("tag1")
555 self.failUnlessEqual(h1, h2a)
556 self.failUnlessEqual(h2a, h2b)
558 def test_sha256d_truncated(self):
559 h1 = hashutil.tagged_hash("tag1", "value", 16)
560 h2 = hashutil.tagged_hasher("tag1", 16)
563 self.failUnlessEqual(len(h1), 16)
564 self.failUnlessEqual(len(h2), 16)
565 self.failUnlessEqual(h1, h2)
568 h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret")
569 h2 = hashutil.convergence_hasher(3, 10, 1000, "secret")
572 self.failUnlessEqual(h1, h2)
574 def test_hashers(self):
575 h1 = hashutil.block_hash("foo")
576 h2 = hashutil.block_hasher()
578 self.failUnlessEqual(h1, h2.digest())
580 h1 = hashutil.uri_extension_hash("foo")
581 h2 = hashutil.uri_extension_hasher()
583 self.failUnlessEqual(h1, h2.digest())
585 h1 = hashutil.plaintext_hash("foo")
586 h2 = hashutil.plaintext_hasher()
588 self.failUnlessEqual(h1, h2.digest())
590 h1 = hashutil.crypttext_hash("foo")
591 h2 = hashutil.crypttext_hasher()
593 self.failUnlessEqual(h1, h2.digest())
595 h1 = hashutil.crypttext_segment_hash("foo")
596 h2 = hashutil.crypttext_segment_hasher()
598 self.failUnlessEqual(h1, h2.digest())
600 h1 = hashutil.plaintext_segment_hash("foo")
601 h2 = hashutil.plaintext_segment_hasher()
603 self.failUnlessEqual(h1, h2.digest())
605 def test_constant_time_compare(self):
606 self.failUnless(hashutil.constant_time_compare("a", "a"))
607 self.failUnless(hashutil.constant_time_compare("ab", "ab"))
608 self.failIf(hashutil.constant_time_compare("a", "b"))
609 self.failIf(hashutil.constant_time_compare("a", "aa"))
611 class Abbreviate(unittest.TestCase):
613 a = abbreviate.abbreviate_time
614 self.failUnlessEqual(a(None), "unknown")
615 self.failUnlessEqual(a(0), "0 seconds")
616 self.failUnlessEqual(a(1), "1 second")
617 self.failUnlessEqual(a(2), "2 seconds")
618 self.failUnlessEqual(a(119), "119 seconds")
620 self.failUnlessEqual(a(2*MIN), "2 minutes")
621 self.failUnlessEqual(a(60*MIN), "60 minutes")
622 self.failUnlessEqual(a(179*MIN), "179 minutes")
624 self.failUnlessEqual(a(180*MIN), "3 hours")
625 self.failUnlessEqual(a(4*HOUR), "4 hours")
628 self.failUnlessEqual(a(2*DAY), "2 days")
629 self.failUnlessEqual(a(2*MONTH), "2 months")
631 self.failUnlessEqual(a(5*YEAR), "5 years")
633 def test_space(self):
634 tests_si = [(None, "unknown"),
641 (20*1000, "20.00 kB"),
642 (1024*1024, "1.05 MB"),
643 (1000*1000, "1.00 MB"),
644 (1000*1000*1000, "1.00 GB"),
645 (1000*1000*1000*1000, "1.00 TB"),
646 (1000*1000*1000*1000*1000, "1.00 PB"),
647 (1234567890123456, "1.23 PB"),
649 for (x, expected) in tests_si:
650 got = abbreviate.abbreviate_space(x, SI=True)
651 self.failUnlessEqual(got, expected)
653 tests_base1024 = [(None, "unknown"),
660 (20*1024, "20.00 kiB"),
661 (1000*1000, "976.56 kiB"),
662 (1024*1024, "1.00 MiB"),
663 (1024*1024*1024, "1.00 GiB"),
664 (1024*1024*1024*1024, "1.00 TiB"),
665 (1000*1000*1000*1000*1000, "909.49 TiB"),
666 (1024*1024*1024*1024*1024, "1.00 PiB"),
667 (1234567890123456, "1.10 PiB"),
669 for (x, expected) in tests_base1024:
670 got = abbreviate.abbreviate_space(x, SI=False)
671 self.failUnlessEqual(got, expected)
673 self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567),
674 "(1.23 MB, 1.18 MiB)")
676 def test_parse_space(self):
677 p = abbreviate.parse_abbreviated_size
678 self.failUnlessEqual(p(""), None)
679 self.failUnlessEqual(p(None), None)
680 self.failUnlessEqual(p("123"), 123)
681 self.failUnlessEqual(p("123B"), 123)
682 self.failUnlessEqual(p("2K"), 2000)
683 self.failUnlessEqual(p("2kb"), 2000)
684 self.failUnlessEqual(p("2KiB"), 2048)
685 self.failUnlessEqual(p("10MB"), 10*1000*1000)
686 self.failUnlessEqual(p("10MiB"), 10*1024*1024)
687 self.failUnlessEqual(p("5G"), 5*1000*1000*1000)
688 self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024)
689 e = self.failUnlessRaises(ValueError, p, "12 cubits")
690 self.failUnless("12 cubits" in str(e))
692 class Limiter(unittest.TestCase):
693 def job(self, i, foo):
694 self.calls.append( (i, foo) )
695 self.simultaneous += 1
696 self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous)
699 self.simultaneous -= 1
700 d.callback("done %d" % i)
701 reactor.callLater(1.0, _done)
704 def bad_job(self, i, foo):
705 raise ValueError("bad_job %d" % i)
707 def test_limiter(self):
709 self.simultaneous = 0
710 self.peak_simultaneous = 0
711 l = limiter.ConcurrencyLimiter()
714 dl.append(l.add(self.job, i, foo=str(i)))
715 d = defer.DeferredList(dl, fireOnOneErrback=True)
717 self.failUnlessEqual(self.simultaneous, 0)
718 self.failUnless(self.peak_simultaneous <= 10)
719 self.failUnlessEqual(len(self.calls), 20)
721 self.failUnless( (i, str(i)) in self.calls)
725 def test_errors(self):
727 self.simultaneous = 0
728 self.peak_simultaneous = 0
729 l = limiter.ConcurrencyLimiter()
732 dl.append(l.add(self.job, i, foo=str(i)))
733 d2 = l.add(self.bad_job, 21, "21")
734 d = defer.DeferredList(dl, fireOnOneErrback=True)
737 for (success, result) in res:
738 self.failUnlessEqual(success, True)
739 results.append(result)
741 expected_results = ["done %d" % i for i in range(20)]
742 expected_results.sort()
743 self.failUnlessEqual(results, expected_results)
744 self.failUnless(self.peak_simultaneous <= 10)
745 self.failUnlessEqual(len(self.calls), 20)
747 self.failUnless( (i, str(i)) in self.calls)
749 self.fail("should have failed, not got %s" % (res,))
752 self.failUnless("bad_job 21" in str(f))
753 d2.addCallbacks(_good, _err)
755 d.addCallback(_most_done)
757 self.failUnlessEqual(self.simultaneous, 0)
758 self.failUnless(self.peak_simultaneous <= 10)
759 self.failUnlessEqual(len(self.calls), 20)
761 self.failUnless( (i, str(i)) in self.calls)
762 d.addCallback(_all_done)
765 class TimeFormat(unittest.TestCase):
766 def test_epoch(self):
767 s = time_format.iso_utc_time_to_seconds("1970-01-01T00:00:01")
768 self.failUnlessEqual(s, 1.0)
769 s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01")
770 self.failUnlessEqual(s, 1.0)
771 s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01")
772 self.failUnlessEqual(s, 1.0)
774 self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01")
775 self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "),
776 "1970-01-01 00:00:01")
779 isostr = time_format.iso_utc(now)
780 timestamp = time_format.iso_utc_time_to_seconds(isostr)
781 self.failUnlessEqual(int(timestamp), int(now))
785 self.failUnlessEqual(time_format.iso_utc(t=my_time),
786 "1970-01-01_00:00:01")
787 e = self.failUnlessRaises(ValueError,
788 time_format.iso_utc_time_to_seconds,
789 "invalid timestring")
790 self.failUnless("not a complete ISO8601 timestamp" in str(e))
791 s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01.500")
792 self.failUnlessEqual(s, 1.5)
794 # Look for daylight-savings-related errors.
795 thatmomentinmarch = time_format.iso_utc_time_to_seconds("2009-03-20 21:49:02.226536")
796 self.failUnlessEqual(thatmomentinmarch, 1237585742.226536)
798 class CacheDir(unittest.TestCase):
799 def test_basic(self):
800 basedir = "test_util/CacheDir/test_basic"
802 def _failIfExists(name):
803 absfn = os.path.join(basedir, name)
804 self.failIf(os.path.exists(absfn),
805 "%s exists but it shouldn't" % absfn)
807 def _failUnlessExists(name):
808 absfn = os.path.join(basedir, name)
809 self.failUnless(os.path.exists(absfn),
810 "%s doesn't exist but it should" % absfn)
812 cdm = cachedir.CacheDirectoryManager(basedir)
813 a = cdm.get_file("a")
814 b = cdm.get_file("b")
815 c = cdm.get_file("c")
816 f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f
817 f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f
818 f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f
820 _failUnlessExists("a")
821 _failUnlessExists("b")
822 _failUnlessExists("c")
826 _failUnlessExists("a")
827 _failUnlessExists("b")
828 _failUnlessExists("c")
831 # this file won't be deleted yet, because it isn't old enough
833 _failUnlessExists("a")
834 _failUnlessExists("b")
835 _failUnlessExists("c")
837 # we change the definition of "old" to make everything old
842 _failUnlessExists("b")
843 _failUnlessExists("c")
851 _failUnlessExists("b")
852 _failUnlessExists("c")
854 b2 = cdm.get_file("b")
858 _failUnlessExists("b")
859 _failUnlessExists("c")
863 def __init__(self, x):
868 return "<%s %s>" % (self.__class__.__name__, self.x,)
871 def __le__(self, other):
872 return self.x <= other
873 def __lt__(self, other):
874 return self.x < other
875 def __ge__(self, other):
876 return self.x >= other
877 def __gt__(self, other):
878 return self.x > other
879 def __ne__(self, other):
880 return self.x != other
881 def __eq__(self, other):
882 return self.x == other
884 class DictUtil(unittest.TestCase):
885 def _help_test_empty_dict(self, klass):
889 self.failUnless(d1 == d2, "d1: %r, d2: %r" % (d1, d2,))
890 self.failUnless(len(d1) == 0)
891 self.failUnless(len(d2) == 0)
893 def _help_test_nonempty_dict(self, klass):
894 d1 = klass({'a': 1, 'b': "eggs", 3: "spam",})
895 d2 = klass({'a': 1, 'b': "eggs", 3: "spam",})
897 self.failUnless(d1 == d2)
898 self.failUnless(len(d1) == 3, "%s, %s" % (len(d1), d1,))
899 self.failUnless(len(d2) == 3)
901 def _help_test_eq_but_notis(self, klass):
902 d = klass({'a': 3, 'b': EqButNotIs(3), 'c': 3})
907 d['b'] = EqButNotIs(3)
912 d['b'] = EqButNotIs(3)
918 d['a'] = EqButNotIs(3)
923 fake3 = EqButNotIs(3)
924 fake7 = EqButNotIs(7)
928 self.failUnless(filter(lambda x: x is 8, d.itervalues()))
929 self.failUnless(filter(lambda x: x is fake7, d.itervalues()))
930 # The real 7 should have been ejected by the d[3] = 8.
931 self.failUnless(not filter(lambda x: x is 7, d.itervalues()))
932 self.failUnless(filter(lambda x: x is fake3, d.iterkeys()))
933 self.failUnless(filter(lambda x: x is 3, d.iterkeys()))
938 fake3 = EqButNotIs(3)
939 fake7 = EqButNotIs(7)
942 self.failUnless(filter(lambda x: x is 8, d.itervalues()))
943 self.failUnless(filter(lambda x: x is fake7, d.itervalues()))
944 # The real 7 should have been ejected by the d[3] = 8.
945 self.failUnless(not filter(lambda x: x is 7, d.itervalues()))
946 self.failUnless(filter(lambda x: x is fake3, d.iterkeys()))
947 self.failUnless(filter(lambda x: x is 3, d.iterkeys()))
951 self._help_test_eq_but_notis(dictutil.UtilDict)
952 self._help_test_eq_but_notis(dictutil.NumDict)
953 self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
954 self._help_test_nonempty_dict(dictutil.UtilDict)
955 self._help_test_nonempty_dict(dictutil.NumDict)
956 self._help_test_nonempty_dict(dictutil.ValueOrderedDict)
957 self._help_test_eq_but_notis(dictutil.UtilDict)
958 self._help_test_eq_but_notis(dictutil.NumDict)
959 self._help_test_eq_but_notis(dictutil.ValueOrderedDict)
961 def test_dict_of_sets(self):
962 ds = dictutil.DictOfSets()
967 self.failUnlessEqual(ds[1], set(["a"]))
968 self.failUnlessEqual(ds[2], set(["b", "c"]))
969 ds.discard(3, "d") # should not raise an exception
971 self.failUnlessEqual(ds[2], set(["c"]))
975 ds.union(1, ["a", "e"])
977 self.failUnlessEqual(ds[1], set(["a","e"]))
978 self.failUnlessEqual(ds[3], set(["f"]))
979 ds2 = dictutil.DictOfSets()
984 self.failUnlessEqual(ds[1], set(["a","e"]))
985 self.failUnlessEqual(ds[3], set(["f", "g"]))
986 self.failUnlessEqual(ds[4], set(["h"]))
989 d1 = {1: "a", 2: "b"}
990 d2 = {2: "c", 3: "d"}
991 dictutil.move(1, d1, d2)
992 self.failUnlessEqual(d1, {2: "b"})
993 self.failUnlessEqual(d2, {1: "a", 2: "c", 3: "d"})
995 d1 = {1: "a", 2: "b"}
996 d2 = {2: "c", 3: "d"}
997 dictutil.move(2, d1, d2)
998 self.failUnlessEqual(d1, {1: "a"})
999 self.failUnlessEqual(d2, {2: "b", 3: "d"})
1001 d1 = {1: "a", 2: "b"}
1002 d2 = {2: "c", 3: "d"}
1003 self.failUnlessRaises(KeyError, dictutil.move, 5, d1, d2, strict=True)
1005 def test_subtract(self):
1006 d1 = {1: "a", 2: "b"}
1007 d2 = {2: "c", 3: "d"}
1008 d3 = dictutil.subtract(d1, d2)
1009 self.failUnlessEqual(d3, {1: "a"})
1011 d1 = {1: "a", 2: "b"}
1013 d3 = dictutil.subtract(d1, d2)
1014 self.failUnlessEqual(d3, {1: "a"})
1016 def test_utildict(self):
1017 d = dictutil.UtilDict({1: "a", 2: "b"})
1020 self.failUnlessEqual(d, {2: "b"})
1023 self.failUnlessRaises(TypeError, eq, d, "not a dict")
1025 d = dictutil.UtilDict({1: "b", 2: "a"})
1026 self.failUnlessEqual(d.items_sorted_by_value(),
1027 [(2, "a"), (1, "b")])
1028 self.failUnlessEqual(d.items_sorted_by_key(),
1029 [(1, "b"), (2, "a")])
1030 self.failUnlessEqual(repr(d), "{1: 'b', 2: 'a'}")
1031 self.failUnless(1 in d)
1033 d2 = dictutil.UtilDict({3: "c", 4: "d"})
1034 self.failUnless(d != d2)
1035 self.failUnless(d2 > d)
1036 self.failUnless(d2 >= d)
1037 self.failUnless(d <= d2)
1038 self.failUnless(d < d2)
1039 self.failUnlessEqual(d[1], "b")
1040 self.failUnlessEqual(sorted(list([k for k in d])), [1,2])
1043 self.failUnlessEqual(d, d3)
1044 self.failUnless(isinstance(d3, dictutil.UtilDict))
1046 d4 = d.fromkeys([3,4], "e")
1047 self.failUnlessEqual(d4, {3: "e", 4: "e"})
1049 self.failUnlessEqual(d.get(1), "b")
1050 self.failUnlessEqual(d.get(3), None)
1051 self.failUnlessEqual(d.get(3, "default"), "default")
1052 self.failUnlessEqual(sorted(list(d.items())),
1053 [(1, "b"), (2, "a")])
1054 self.failUnlessEqual(sorted(list(d.iteritems())),
1055 [(1, "b"), (2, "a")])
1056 self.failUnlessEqual(sorted(d.keys()), [1, 2])
1057 self.failUnlessEqual(sorted(d.values()), ["a", "b"])
1058 x = d.setdefault(1, "new")
1059 self.failUnlessEqual(x, "b")
1060 self.failUnlessEqual(d[1], "b")
1061 x = d.setdefault(3, "new")
1062 self.failUnlessEqual(x, "new")
1063 self.failUnlessEqual(d[3], "new")
1067 self.failUnless(x in [(1, "b"), (2, "a")])
1069 self.failUnless(x in [(1, "b"), (2, "a")])
1070 self.failUnlessRaises(KeyError, d.popitem)
1072 def test_numdict(self):
1073 d = dictutil.NumDict({"a": 1, "b": 2})
1075 d.add_num("a", 10, 5)
1076 d.add_num("c", 20, 5)
1078 self.failUnlessEqual(d, {"a": 11, "b": 2, "c": 25, "d": 30})
1080 d.subtract_num("a", 10)
1081 d.subtract_num("e", 10)
1082 d.subtract_num("f", 10, 15)
1083 self.failUnlessEqual(d, {"a": 1, "b": 2, "c": 25, "d": 30,
1086 self.failUnlessEqual(d.sum(), sum([1, 2, 25, 30, -10, 5]))
1088 d = dictutil.NumDict()
1092 self.failUnlessEqual(d, {"a": 2, "b": 6})
1096 self.failUnlessEqual(d, {"a": 1, "b": 6, "c": -1, "d": 4})
1097 self.failUnlessEqual(d.items_sorted_by_key(),
1098 [("a", 1), ("b", 6), ("c", -1), ("d", 4)])
1099 self.failUnlessEqual(d.items_sorted_by_value(),
1100 [("c", -1), ("a", 1), ("d", 4), ("b", 6)])
1101 self.failUnlessEqual(d.item_with_largest_value(), ("b", 6))
1103 d = dictutil.NumDict({"a": 1, "b": 2})
1104 self.failUnlessEqual(repr(d), "{'a': 1, 'b': 2}")
1105 self.failUnless("a" in d)
1107 d2 = dictutil.NumDict({"c": 3, "d": 4})
1108 self.failUnless(d != d2)
1109 self.failUnless(d2 > d)
1110 self.failUnless(d2 >= d)
1111 self.failUnless(d <= d2)
1112 self.failUnless(d < d2)
1113 self.failUnlessEqual(d["a"], 1)
1114 self.failUnlessEqual(sorted(list([k for k in d])), ["a","b"])
1117 self.failUnlessRaises(TypeError, eq, d, "not a dict")
1120 self.failUnlessEqual(d, d3)
1121 self.failUnless(isinstance(d3, dictutil.NumDict))
1123 d4 = d.fromkeys(["a","b"], 5)
1124 self.failUnlessEqual(d4, {"a": 5, "b": 5})
1126 self.failUnlessEqual(d.get("a"), 1)
1127 self.failUnlessEqual(d.get("c"), 0)
1128 self.failUnlessEqual(d.get("c", 5), 5)
1129 self.failUnlessEqual(sorted(list(d.items())),
1130 [("a", 1), ("b", 2)])
1131 self.failUnlessEqual(sorted(list(d.iteritems())),
1132 [("a", 1), ("b", 2)])
1133 self.failUnlessEqual(sorted(d.keys()), ["a", "b"])
1134 self.failUnlessEqual(sorted(d.values()), [1, 2])
1135 self.failUnless(d.has_key("a"))
1136 self.failIf(d.has_key("c"))
1138 x = d.setdefault("c", 3)
1139 self.failUnlessEqual(x, 3)
1140 self.failUnlessEqual(d["c"], 3)
1141 x = d.setdefault("c", 5)
1142 self.failUnlessEqual(x, 3)
1143 self.failUnlessEqual(d["c"], 3)
1147 self.failUnless(x in [("a", 1), ("b", 2)])
1149 self.failUnless(x in [("a", 1), ("b", 2)])
1150 self.failUnlessRaises(KeyError, d.popitem)
1153 d.update({"c": 4, "d": 5})
1154 self.failUnlessEqual(d, {"c": 4, "d": 5})
1156 def test_del_if_present(self):
1157 d = {1: "a", 2: "b"}
1158 dictutil.del_if_present(d, 1)
1159 dictutil.del_if_present(d, 3)
1160 self.failUnlessEqual(d, {2: "b"})
1162 def test_valueordereddict(self):
1163 d = dictutil.ValueOrderedDict()
1168 self.failUnlessEqual(d, {"a": 3, "b": 2, "c": 1})
1169 self.failUnlessEqual(d.items(), [("c", 1), ("b", 2), ("a", 3)])
1170 self.failUnlessEqual(d.values(), [1, 2, 3])
1171 self.failUnlessEqual(d.keys(), ["c", "b", "a"])
1172 self.failUnlessEqual(repr(d), "<ValueOrderedDict {c: 1, b: 2, a: 3}>")
1175 self.failIf(d == {"a": 4})
1176 self.failUnless(d != {"a": 4})
1178 x = d.setdefault("d", 0)
1179 self.failUnlessEqual(x, 0)
1180 self.failUnlessEqual(d["d"], 0)
1181 x = d.setdefault("d", -1)
1182 self.failUnlessEqual(x, 0)
1183 self.failUnlessEqual(d["d"], 0)
1185 x = d.remove("e", "default", False)
1186 self.failUnlessEqual(x, "default")
1187 self.failUnlessRaises(KeyError, d.remove, "e", "default", True)
1188 x = d.remove("d", 5)
1189 self.failUnlessEqual(x, 0)
1191 x = d.__getitem__("c")
1192 self.failUnlessEqual(x, 1)
1193 x = d.__getitem__("e", "default", False)
1194 self.failUnlessEqual(x, "default")
1195 self.failUnlessRaises(KeyError, d.__getitem__, "e", "default", True)
1197 self.failUnlessEqual(d.popitem(), ("c", 1))
1198 self.failUnlessEqual(d.popitem(), ("b", 2))
1199 self.failUnlessEqual(d.popitem(), ("a", 3))
1200 self.failUnlessRaises(KeyError, d.popitem)
1202 d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1203 x = d.pop("d", "default", False)
1204 self.failUnlessEqual(x, "default")
1205 self.failUnlessRaises(KeyError, d.pop, "d", "default", True)
1207 self.failUnlessEqual(x, 2)
1208 self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1210 d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1})
1211 x = d.pop_from_list(1) # pop the second item, b/2
1212 self.failUnlessEqual(x, "b")
1213 self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
1215 class FakeRemoteReference:
1216 def callRemote(self, methname, *args, **kwargs):
1217 return defer.maybeDeferred(self.oops)
1219 raise IndexError("remote missing key")
1221 class RemoteFailures(unittest.TestCase):
1222 def test_check(self):
1223 check_local = rrefutil.check_local
1224 check_remote = rrefutil.check_remote
1226 raise IndexError("local missing key")
1230 self.failUnlessEqual(localf.check(IndexError, KeyError), IndexError)
1231 self.failUnlessEqual(localf.check(ValueError, KeyError), None)
1232 self.failUnlessEqual(localf.check(ServerFailure), None)
1233 self.failUnlessEqual(check_local(localf, IndexError, KeyError),
1235 self.failUnlessEqual(check_local(localf, ValueError, KeyError), None)
1236 self.failUnlessEqual(check_remote(localf, IndexError, KeyError), None)
1237 self.failUnlessEqual(check_remote(localf, ValueError, KeyError), None)
1239 frr = FakeRemoteReference()
1240 wrr = rrefutil.WrappedRemoteReference(frr)
1241 d = wrr.callRemote("oops")
1243 self.failUnlessEqual(f.check(IndexError, KeyError), None)
1244 self.failUnlessEqual(f.check(ServerFailure, KeyError),
1246 self.failUnlessEqual(check_remote(f, IndexError, KeyError),
1248 self.failUnlessEqual(check_remote(f, ValueError, KeyError), None)
1249 self.failUnlessEqual(check_local(f, IndexError, KeyError), None)
1250 self.failUnlessEqual(check_local(f, ValueError, KeyError), None)
1251 d.addErrback(_check)
1254 def test_is_remote(self):
1256 raise IndexError("local missing key")
1259 self.failIf(rrefutil.is_remote(localf))
1260 self.failUnless(rrefutil.is_local(localf))
1262 frr = FakeRemoteReference()
1263 wrr = rrefutil.WrappedRemoteReference(frr)
1264 d = wrr.callRemote("oops")
1266 self.failUnless(rrefutil.is_remote(f))
1267 self.failIf(rrefutil.is_local(f))
1268 d.addErrback(_check)
1271 def test_trap(self):
1273 raise IndexError("local missing key")
1277 self.failUnlessRaises(Failure, localf.trap, ValueError, KeyError)
1278 self.failUnlessRaises(Failure, localf.trap, ServerFailure)
1279 self.failUnlessEqual(localf.trap(IndexError, KeyError), IndexError)
1280 self.failUnlessEqual(rrefutil.trap_local(localf, IndexError, KeyError),
1282 self.failUnlessRaises(Failure,
1283 rrefutil.trap_remote, localf, ValueError, KeyError)
1285 frr = FakeRemoteReference()
1286 wrr = rrefutil.WrappedRemoteReference(frr)
1287 d = wrr.callRemote("oops")
1289 self.failUnlessRaises(Failure,
1290 f.trap, ValueError, KeyError)
1291 self.failUnlessRaises(Failure,
1293 self.failUnlessEqual(f.trap(ServerFailure), ServerFailure)
1294 self.failUnlessRaises(Failure,
1295 rrefutil.trap_remote, f, ValueError, KeyError)
1296 self.failUnlessEqual(rrefutil.trap_remote(f, IndexError, KeyError),
1298 self.failUnlessRaises(Failure,
1299 rrefutil.trap_local, f, ValueError, KeyError)
1300 self.failUnlessRaises(Failure,
1301 rrefutil.trap_local, f, IndexError)
1302 d.addErrback(_check)
1305 class Pipeline(unittest.TestCase):
1306 def pause(self, *args, **kwargs):
1307 d = defer.Deferred()
1308 self.calls.append( (d, args, kwargs) )
1311 def failUnlessCallsAre(self, expected):
1314 self.failUnlessEqual(len(self.calls), len(expected), self.calls)
1315 for i,c in enumerate(self.calls):
1316 self.failUnlessEqual(c[1:], expected[i], str(i))
1318 def test_basic(self):
1321 p = pipeline.Pipeline(100)
1323 d = p.flush() # fires immediately
1324 d.addCallbacks(finished.append, log.err)
1325 self.failUnlessEqual(len(finished), 1)
1328 d = p.add(10, self.pause, "one")
1329 # the call should start right away, and our return Deferred should
1331 d.addCallbacks(finished.append, log.err)
1332 self.failUnlessEqual(len(finished), 1)
1333 self.failUnlessEqual(finished[0], None)
1334 self.failUnlessCallsAre([ ( ("one",) , {} ) ])
1335 self.failUnlessEqual(p.gauge, 10)
1340 d = p.add(20, self.pause, "two", kw=2)
1341 # pipeline: [one, two]
1343 # the call and the Deferred should fire right away
1344 d.addCallbacks(finished.append, log.err)
1345 self.failUnlessEqual(len(finished), 1)
1346 self.failUnlessEqual(finished[0], None)
1347 self.failUnlessCallsAre([ ( ("one",) , {} ),
1348 ( ("two",) , {"kw": 2} ),
1350 self.failUnlessEqual(p.gauge, 30)
1352 self.calls[0][0].callback("one-result")
1354 self.failUnlessEqual(p.gauge, 20)
1357 d = p.add(90, self.pause, "three", "posarg1")
1358 # pipeline: [two, three]
1361 fd.addCallbacks(flushed.append, log.err)
1362 self.failUnlessEqual(flushed, [])
1364 # the call will be made right away, but the return Deferred will not,
1365 # because the pipeline is now full.
1366 d.addCallbacks(finished.append, log.err)
1367 self.failUnlessEqual(len(finished), 0)
1368 self.failUnlessCallsAre([ ( ("one",) , {} ),
1369 ( ("two",) , {"kw": 2} ),
1370 ( ("three", "posarg1"), {} ),
1372 self.failUnlessEqual(p.gauge, 110)
1374 self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause)
1376 # retiring either call will unblock the pipeline, causing the #3
1378 self.calls[2][0].callback("three-result")
1381 self.failUnlessEqual(len(finished), 1)
1382 self.failUnlessEqual(finished[0], None)
1383 self.failUnlessEqual(flushed, [])
1385 # retiring call#2 will finally allow the flush() Deferred to fire
1386 self.calls[1][0].callback("two-result")
1387 self.failUnlessEqual(len(flushed), 1)
1389 def test_errors(self):
1391 p = pipeline.Pipeline(100)
1393 d1 = p.add(200, self.pause, "one")
1397 d1.addBoth(finished.append)
1398 self.failUnlessEqual(finished, [])
1401 d2.addBoth(flushed.append)
1402 self.failUnlessEqual(flushed, [])
1404 self.calls[0][0].errback(ValueError("oops"))
1406 self.failUnlessEqual(len(finished), 1)
1408 self.failUnless(isinstance(f, Failure))
1409 self.failUnless(f.check(pipeline.PipelineError))
1411 self.failUnless("ValueError" in r, r)
1413 self.failUnless(f2.check(ValueError))
1415 self.failUnlessEqual(len(flushed), 1)
1417 self.failUnless(isinstance(f, Failure))
1418 self.failUnless(f.check(pipeline.PipelineError))
1420 self.failUnless(f2.check(ValueError))
1422 # now that the pipeline is in the failed state, any new calls will
1425 d3 = p.add(20, self.pause, "two")
1428 d3.addBoth(finished.append)
1429 self.failUnlessEqual(len(finished), 1)
1431 self.failUnless(isinstance(f, Failure))
1432 self.failUnless(f.check(pipeline.PipelineError))
1434 self.failUnless("ValueError" in r, r)
1436 self.failUnless(f2.check(ValueError))
1440 d4.addBoth(flushed.append)
1441 self.failUnlessEqual(len(flushed), 1)
1443 self.failUnless(isinstance(f, Failure))
1444 self.failUnless(f.check(pipeline.PipelineError))
1446 self.failUnless(f2.check(ValueError))
1449 def test_errors2(self):
1451 p = pipeline.Pipeline(100)
1453 d1 = p.add(10, self.pause, "one")
1454 d2 = p.add(20, self.pause, "two")
1455 d3 = p.add(30, self.pause, "three")
1458 # one call fails, then the second one succeeds: make sure
1459 # ExpandableDeferredList tolerates the second one
1462 d4.addBoth(flushed.append)
1463 self.failUnlessEqual(flushed, [])
1465 self.calls[0][0].errback(ValueError("oops"))
1466 self.failUnlessEqual(len(flushed), 1)
1468 self.failUnless(isinstance(f, Failure))
1469 self.failUnless(f.check(pipeline.PipelineError))
1471 self.failUnless(f2.check(ValueError))
1473 self.calls[1][0].callback("two-result")
1474 self.calls[2][0].errback(ValueError("three-error"))