src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IPullProducer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.api import flushEventualQueue, fireEventually
   9 from allmydata import uri, dirnode, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IMutableFileNode, IImmutableFileNode,\
  12                                  NotEnoughSharesError, ICheckable, \
  13                                  IMutableUploadable, SDMF_VERSION, \
  14                                  MDMF_VERSION
  15 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  16      DeepCheckResults, DeepCheckAndRepairResults
  17 from allmydata.mutable.common import CorruptShareError
  18 from allmydata.mutable.layout import unpack_header
  19 from allmydata.mutable.publish import MutableData
  20 from allmydata.storage.mutable import MutableShareFile
  21 from allmydata.util import hashutil, log, fileutil, pollmixin
  22 from allmydata.util.assertutil import precondition
  23 from allmydata.util.consumer import download_to_data
  24 from allmydata.stats import StatsGathererService
  25 from allmydata.key_generator import KeyGeneratorService
  26 import allmydata.test.common_util as testutil
  27 from allmydata import immutable
  28
  29 TEST_RSA_KEY_SIZE = 522
  30
  31 def flush_but_dont_ignore(res):
  32     d = flushEventualQueue()
  33     def _done(ignored):
  34         return res
  35     d.addCallback(_done)
  36     return d
  37
  38 class DummyProducer:
  39     implements(IPullProducer)
  40     def resumeProducing(self):
  41         pass
  42
  43 class FakeCHKFileNode:
  44     """I provide IImmutableFileNode, but all of my data is stored in a
  45     class-level dictionary."""
  46     implements(IImmutableFileNode)
  47     all_contents = {}
  48     bad_shares = {}
  49
  50     def __init__(self, filecap):
  51         precondition(isinstance(filecap, (uri.CHKFileURI, uri.LiteralFileURI)), filecap)
  52         self.my_uri = filecap
  53         self.storage_index = self.my_uri.get_storage_index()
  54
  55     def get_uri(self):
  56         return self.my_uri.to_string()
  57     def get_write_uri(self):
  58         return None
  59     def get_readonly_uri(self):
  60         return self.my_uri.to_string()
  61     def get_cap(self):
  62         return self.my_uri
  63     def get_verify_cap(self):
  64         return self.my_uri.get_verify_cap()
  65     def get_repair_cap(self):
  66         return self.my_uri.get_verify_cap()
  67     def get_storage_index(self):
  68         return self.storage_index
  69
  70     def check(self, monitor, verify=False, add_lease=False):
  71         r = CheckResults(self.my_uri, self.storage_index)
  72         is_bad = self.bad_shares.get(self.storage_index, None)
  73         data = {}
  74         data["count-shares-needed"] = 3
  75         data["count-shares-expected"] = 10
  76         data["count-good-share-hosts"] = 10
  77         data["count-wrong-shares"] = 0
  78         nodeid = "\x00"*20
  79         data["list-corrupt-shares"] = []
  80         data["sharemap"] = {1: [nodeid]}
  81         data["servers-responding"] = [nodeid]
  82         data["count-recoverable-versions"] = 1
  83         data["count-unrecoverable-versions"] = 0
  84         if is_bad:
  85             r.set_healthy(False)
  86             r.set_recoverable(True)
  87             data["count-shares-good"] = 9
  88             data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
  89             r.problems = failure.Failure(CorruptShareError(is_bad))
  90         else:
  91             r.set_healthy(True)
  92             r.set_recoverable(True)
  93             data["count-shares-good"] = 10
  94             r.problems = []
  95         r.set_data(data)
  96         r.set_needs_rebalancing(False)
  97         return defer.succeed(r)
  98     def check_and_repair(self, monitor, verify=False, add_lease=False):
  99         d = self.check(verify)
 100         def _got(cr):
 101             r = CheckAndRepairResults(self.storage_index)
 102             r.pre_repair_results = r.post_repair_results = cr
 103             return r
 104         d.addCallback(_got)
 105         return d
 106
 107     def is_mutable(self):
 108         return False
 109     def is_readonly(self):
 110         return True
 111     def is_unknown(self):
 112         return False
 113     def is_allowed_in_immutable_directory(self):
 114         return True
 115     def raise_error(self):
 116         pass
 117
 118     def get_size(self):
 119         if isinstance(self.my_uri, uri.LiteralFileURI):
 120             return self.my_uri.get_size()
 121         try:
 122             data = self.all_contents[self.my_uri.to_string()]
 123         except KeyError, le:
 124             raise NotEnoughSharesError(le, 0, 3)
 125         return len(data)
 126
 127     def read(self, consumer, offset=0, size=None):
 128         # we don't bother to call registerProducer/unregisterProducer,
 129         # because it's a hassle to write a dummy Producer that does the right
 130         # thing (we have to make sure that DummyProducer.resumeProducing
 131         # writes the data into the consumer immediately, otherwise it will
 132         # loop forever).
 133
 134         d = defer.succeed(None)
 135         d.addCallback(self._read, consumer, offset, size)
 136         return d
 137
 138     def _read(self, ignored, consumer, offset, size):
 139         if isinstance(self.my_uri, uri.LiteralFileURI):
 140             data = self.my_uri.data
 141         else:
 142             if self.my_uri.to_string() not in self.all_contents:
 143                 raise NotEnoughSharesError(None, 0, 3)
 144             data = self.all_contents[self.my_uri.to_string()]
 145         start = offset
 146         if size is not None:
 147             end = offset + size
 148         else:
 149             end = len(data)
 150         consumer.write(data[start:end])
 151         return consumer
 152
 153
 154     def get_best_readable_version(self):
 155         return defer.succeed(self)
 156
 157
 158     def download_to_data(self):
 159         return download_to_data(self)
 160
 161
 162     download_best_version = download_to_data
 163
 164
 165     def get_size_of_best_version(self):
 166         return defer.succeed(self.get_size)
 167
 168
 169 def make_chk_file_cap(size):
 170     return uri.CHKFileURI(key=os.urandom(16),
 171                           uri_extension_hash=os.urandom(32),
 172                           needed_shares=3,
 173                           total_shares=10,
 174                           size=size)
 175 def make_chk_file_uri(size):
 176     return make_chk_file_cap(size).to_string()
 177
 178 def create_chk_filenode(contents):
 179     filecap = make_chk_file_cap(len(contents))
 180     n = FakeCHKFileNode(filecap)
 181     FakeCHKFileNode.all_contents[filecap.to_string()] = contents
 182     return n
 183
 184
 185 class FakeMutableFileNode:
 186     """I provide IMutableFileNode, but all of my data is stored in a
 187     class-level dictionary."""
 188
 189     implements(IMutableFileNode, ICheckable)
 190     MUTABLE_SIZELIMIT = 10000
 191     all_contents = {}
 192     bad_shares = {}
 193     file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION
 194
 195     def __init__(self, storage_broker, secret_holder,
 196                  default_encoding_parameters, history):
 197         self.init_from_cap(make_mutable_file_cap())
 198         self._k = default_encoding_parameters['k']
 199         self._segsize = default_encoding_parameters['max_segment_size']
 200     def create(self, contents, key_generator=None, keysize=None,
 201                version=SDMF_VERSION):
 202         if version == MDMF_VERSION and \
 203             isinstance(self.my_uri, (uri.ReadonlySSKFileURI,
 204                                  uri.WriteableSSKFileURI)):
 205             self.init_from_cap(make_mdmf_mutable_file_cap())
 206         self.file_types[self.storage_index] = version
 207         initial_contents = self._get_initial_contents(contents)
 208         data = initial_contents.read(initial_contents.get_size())
 209         data = "".join(data)
 210         self.all_contents[self.storage_index] = data
 211         self.my_uri.set_extension_params([self._k, self._segsize])
 212         return defer.succeed(self)
 213     def _get_initial_contents(self, contents):
 214         if contents is None:
 215             return MutableData("")
 216
 217         if IMutableUploadable.providedBy(contents):
 218             return contents
 219
 220         assert callable(contents), "%s should be callable, not %s" % \
 221                (contents, type(contents))
 222         return contents(self)
 223     def init_from_cap(self, filecap):
 224         assert isinstance(filecap, (uri.WriteableSSKFileURI,
 225                                     uri.ReadonlySSKFileURI,
 226                                     uri.WriteableMDMFFileURI,
 227                                     uri.ReadonlyMDMFFileURI))
 228         self.my_uri = filecap
 229         self.storage_index = self.my_uri.get_storage_index()
 230         if isinstance(filecap, (uri.WriteableMDMFFileURI,
 231                                 uri.ReadonlyMDMFFileURI)):
 232             self.file_types[self.storage_index] = MDMF_VERSION
 233
 234         else:
 235             self.file_types[self.storage_index] = SDMF_VERSION
 236
 237         return self
 238     def get_cap(self):
 239         return self.my_uri
 240     def get_readcap(self):
 241         return self.my_uri.get_readonly()
 242     def get_uri(self):
 243         return self.my_uri.to_string()
 244     def get_write_uri(self):
 245         if self.is_readonly():
 246             return None
 247         return self.my_uri.to_string()
 248     def get_readonly(self):
 249         return self.my_uri.get_readonly()
 250     def get_readonly_uri(self):
 251         return self.my_uri.get_readonly().to_string()
 252     def get_verify_cap(self):
 253         return self.my_uri.get_verify_cap()
 254     def get_repair_cap(self):
 255         if self.my_uri.is_readonly():
 256             return None
 257         return self.my_uri
 258     def is_readonly(self):
 259         return self.my_uri.is_readonly()
 260     def is_mutable(self):
 261         return self.my_uri.is_mutable()
 262     def is_unknown(self):
 263         return False
 264     def is_allowed_in_immutable_directory(self):
 265         return not self.my_uri.is_mutable()
 266     def raise_error(self):
 267         pass
 268     def get_writekey(self):
 269         return "\x00"*16
 270     def get_size(self):
 271         return len(self.all_contents[self.storage_index])
 272     def get_current_size(self):
 273         return self.get_size_of_best_version()
 274     def get_size_of_best_version(self):
 275         return defer.succeed(len(self.all_contents[self.storage_index]))
 276
 277     def get_storage_index(self):
 278         return self.storage_index
 279
 280     def get_servermap(self, mode):
 281         return defer.succeed(None)
 282
 283     def get_version(self):
 284         assert self.storage_index in self.file_types
 285         return self.file_types[self.storage_index]
 286
 287     def check(self, monitor, verify=False, add_lease=False):
 288         r = CheckResults(self.my_uri, self.storage_index)
 289         is_bad = self.bad_shares.get(self.storage_index, None)
 290         data = {}
 291         data["count-shares-needed"] = 3
 292         data["count-shares-expected"] = 10
 293         data["count-good-share-hosts"] = 10
 294         data["count-wrong-shares"] = 0
 295         data["list-corrupt-shares"] = []
 296         nodeid = "\x00"*20
 297         data["sharemap"] = {"seq1-abcd-sh0": [nodeid]}
 298         data["servers-responding"] = [nodeid]
 299         data["count-recoverable-versions"] = 1
 300         data["count-unrecoverable-versions"] = 0
 301         if is_bad:
 302             r.set_healthy(False)
 303             r.set_recoverable(True)
 304             data["count-shares-good"] = 9
 305             r.problems = failure.Failure(CorruptShareError("peerid",
 306                                                            0, # shnum
 307                                                            is_bad))
 308         else:
 309             r.set_healthy(True)
 310             r.set_recoverable(True)
 311             data["count-shares-good"] = 10
 312             r.problems = []
 313         r.set_data(data)
 314         r.set_needs_rebalancing(False)
 315         return defer.succeed(r)
 316
 317     def check_and_repair(self, monitor, verify=False, add_lease=False):
 318         d = self.check(verify)
 319         def _got(cr):
 320             r = CheckAndRepairResults(self.storage_index)
 321             r.pre_repair_results = r.post_repair_results = cr
 322             return r
 323         d.addCallback(_got)
 324         return d
 325
 326     def deep_check(self, verify=False, add_lease=False):
 327         d = self.check(verify)
 328         def _done(r):
 329             dr = DeepCheckResults(self.storage_index)
 330             dr.add_check(r, [])
 331             return dr
 332         d.addCallback(_done)
 333         return d
 334
 335     def deep_check_and_repair(self, verify=False, add_lease=False):
 336         d = self.check_and_repair(verify)
 337         def _done(r):
 338             dr = DeepCheckAndRepairResults(self.storage_index)
 339             dr.add_check(r, [])
 340             return dr
 341         d.addCallback(_done)
 342         return d
 343
 344     def download_best_version(self):
 345         return defer.succeed(self._download_best_version())
 346
 347
 348     def _download_best_version(self, ignored=None):
 349         if isinstance(self.my_uri, uri.LiteralFileURI):
 350             return self.my_uri.data
 351         if self.storage_index not in self.all_contents:
 352             raise NotEnoughSharesError(None, 0, 3)
 353         return self.all_contents[self.storage_index]
 354
 355
 356     def overwrite(self, new_contents):
 357         assert not self.is_readonly()
 358         new_data = new_contents.read(new_contents.get_size())
 359         new_data = "".join(new_data)
 360         self.all_contents[self.storage_index] = new_data
 361         self.my_uri.set_extension_params([self._k, self._segsize])
 362         return defer.succeed(None)
 363     def modify(self, modifier):
 364         # this does not implement FileTooLargeError, but the real one does
 365         return defer.maybeDeferred(self._modify, modifier)
 366     def _modify(self, modifier):
 367         assert not self.is_readonly()
 368         old_contents = self.all_contents[self.storage_index]
 369         new_data = modifier(old_contents, None, True)
 370         self.all_contents[self.storage_index] = new_data
 371         self.my_uri.set_extension_params([self._k, self._segsize])
 372         return None
 373
 374     # As actually implemented, MutableFilenode and MutableFileVersion
 375     # are distinct. However, nothing in the webapi uses (yet) that
 376     # distinction -- it just uses the unified download interface
 377     # provided by get_best_readable_version and read. When we start
 378     # doing cooler things like LDMF, we will want to revise this code to
 379     # be less simplistic.
 380     def get_best_readable_version(self):
 381         return defer.succeed(self)
 382
 383
 384     def get_best_mutable_version(self):
 385         return defer.succeed(self)
 386
 387     # Ditto for this, which is an implementation of IWriteable.
 388     # XXX: Declare that the same is implemented.
 389     def update(self, data, offset):
 390         assert not self.is_readonly()
 391         def modifier(old, servermap, first_time):
 392             new = old[:offset] + "".join(data.read(data.get_size()))
 393             new += old[len(new):]
 394             return new
 395         return self.modify(modifier)
 396
 397
 398     def read(self, consumer, offset=0, size=None):
 399         data = self._download_best_version()
 400         if size:
 401             data = data[offset:offset+size]
 402         consumer.write(data)
 403         return defer.succeed(consumer)
 404
 405
 406 def make_mutable_file_cap():
 407     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 408                                    fingerprint=os.urandom(32))
 409
 410 def make_mdmf_mutable_file_cap():
 411     return uri.WriteableMDMFFileURI(writekey=os.urandom(16),
 412                                    fingerprint=os.urandom(32))
 413
 414 def make_mutable_file_uri(mdmf=False):
 415     if mdmf:
 416         uri = make_mdmf_mutable_file_cap()
 417     else:
 418         uri = make_mutable_file_cap()
 419
 420     return uri.to_string()
 421
 422 def make_verifier_uri():
 423     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 424                               fingerprint=os.urandom(32)).to_string()
 425
 426 def create_mutable_filenode(contents, mdmf=False):
 427     # XXX: All of these arguments are kind of stupid.
 428     if mdmf:
 429         cap = make_mdmf_mutable_file_cap()
 430     else:
 431         cap = make_mutable_file_cap()
 432
 433     encoding_params = {}
 434     encoding_params['k'] = 3
 435     encoding_params['max_segment_size'] = 128*1024
 436
 437     filenode = FakeMutableFileNode(None, None, encoding_params, None)
 438     filenode.init_from_cap(cap)
 439     if mdmf:
 440         filenode.create(MutableData(contents), version=MDMF_VERSION)
 441     else:
 442         filenode.create(MutableData(contents), version=SDMF_VERSION)
 443     return filenode
 444
 445
 446 class FakeDirectoryNode(dirnode.DirectoryNode):
 447     """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
 448     backing store, so it doesn't go to the grid. The child data is still
 449     encrypted and serialized, so this isn't useful for tests that want to
 450     look inside the dirnodes and check their contents.
 451     """
 452     filenode_class = FakeMutableFileNode
 453
 454 class LoggingServiceParent(service.MultiService):
 455     def log(self, *args, **kwargs):
 456         return log.msg(*args, **kwargs)
 457
 458 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 459
 460     # SystemTestMixin tests tend to be a lot of work, and we have a few
 461     # buildslaves that are pretty slow, and every once in a while these tests
 462     # run up against the default 120 second timeout. So increase the default
 463     # timeout. Individual test cases can override this, of course.
 464     timeout = 300
 465
 466     def setUp(self):
 467         self.sparent = service.MultiService()
 468         self.sparent.startService()
 469
 470         self.stats_gatherer = None
 471         self.stats_gatherer_furl = None
 472         self.key_generator_svc = None
 473         self.key_generator_furl = None
 474
 475     def tearDown(self):
 476         log.msg("shutting down SystemTest services")
 477         d = self.sparent.stopService()
 478         d.addBoth(flush_but_dont_ignore)
 479         return d
 480
 481     def getdir(self, subdir):
 482         return os.path.join(self.basedir, subdir)
 483
 484     def add_service(self, s):
 485         s.setServiceParent(self.sparent)
 486         return s
 487
 488     def set_up_nodes(self, NUMCLIENTS=5,
 489                      use_stats_gatherer=False, use_key_generator=False):
 490         self.numclients = NUMCLIENTS
 491         iv_dir = self.getdir("introducer")
 492         if not os.path.isdir(iv_dir):
 493             fileutil.make_dirs(iv_dir)
 494             fileutil.write(os.path.join(iv_dir, 'tahoe.cfg'), \
 495                                "[node]\n" + \
 496                                "web.port = tcp:0:interface=127.0.0.1\n")
 497             if SYSTEM_TEST_CERTS:
 498                 os.mkdir(os.path.join(iv_dir, "private"))
 499                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 500                 f.write(SYSTEM_TEST_CERTS[0])
 501                 f.close()
 502         iv = IntroducerNode(basedir=iv_dir)
 503         self.introducer = self.add_service(iv)
 504         d = self.introducer.when_tub_ready()
 505         d.addCallback(self._get_introducer_web)
 506         if use_stats_gatherer:
 507             d.addCallback(self._set_up_stats_gatherer)
 508         if use_key_generator:
 509             d.addCallback(self._set_up_key_generator)
 510         d.addCallback(self._set_up_nodes_2)
 511         if use_stats_gatherer:
 512             d.addCallback(self._grab_stats)
 513         return d
 514
 515     def _get_introducer_web(self, res):
 516         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 517         self.introweb_url = f.read().strip()
 518         f.close()
 519
 520     def _set_up_stats_gatherer(self, res):
 521         statsdir = self.getdir("stats_gatherer")
 522         fileutil.make_dirs(statsdir)
 523         self.stats_gatherer_svc = StatsGathererService(statsdir)
 524         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 525         self.add_service(self.stats_gatherer_svc)
 526
 527         d = fireEventually()
 528         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 529         def check_for_furl():
 530             return os.path.exists(sgf)
 531         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 532         def get_furl(junk):
 533             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 534         d.addCallback(get_furl)
 535         return d
 536
 537     def _set_up_key_generator(self, res):
 538         kgsdir = self.getdir("key_generator")
 539         fileutil.make_dirs(kgsdir)
 540
 541         self.key_generator_svc = KeyGeneratorService(kgsdir,
 542                                                      display_furl=False,
 543                                                      default_key_size=TEST_RSA_KEY_SIZE)
 544         self.key_generator_svc.key_generator.pool_size = 4
 545         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 546         self.add_service(self.key_generator_svc)
 547
 548         d = fireEventually()
 549         def check_for_furl():
 550             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 551         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 552         def get_furl(junk):
 553             kgf = os.path.join(kgsdir, 'key_generator.furl')
 554             self.key_generator_furl = file(kgf, 'rb').read().strip()
 555         d.addCallback(get_furl)
 556         return d
 557
 558     def _set_up_nodes_2(self, res):
 559         q = self.introducer
 560         self.introducer_furl = q.introducer_url
 561         self.clients = []
 562         basedirs = []
 563         for i in range(self.numclients):
 564             basedir = self.getdir("client%d" % i)
 565             basedirs.append(basedir)
 566             fileutil.make_dirs(os.path.join(basedir, "private"))
 567             if len(SYSTEM_TEST_CERTS) > (i+1):
 568                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 569                 f.write(SYSTEM_TEST_CERTS[i+1])
 570                 f.close()
 571
 572             config = "[client]\n"
 573             config += "introducer.furl = %s\n" % self.introducer_furl
 574             if self.stats_gatherer_furl:
 575                 config += "stats_gatherer.furl = %s\n" % self.stats_gatherer_furl
 576
 577             if i == 0:
 578                 # clients[0] runs a webserver and a helper, no key_generator
 579                 config += "[node]\n"
 580                 config += "web.port = tcp:0:interface=127.0.0.1\n"
 581                 config += "timeout.keepalive = 600\n"
 582                 config += "[helper]\n"
 583                 config += "enabled = True\n"
 584             if i == 3:
 585                 # clients[3] runs a webserver and uses a helper, uses
 586                 # key_generator
 587                 if self.key_generator_furl:
 588                     config += "key_generator.furl = %s\n" % self.key_generator_furl
 589                 config += "[node]\n"
 590                 config += "web.port = tcp:0:interface=127.0.0.1\n"
 591                 config += "timeout.disconnect = 1800\n"
 592
 593             fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
 594
 595         # give subclasses a chance to append lines to the node's tahoe.cfg
 596         # files before they are launched.
 597         self._set_up_nodes_extra_config()
 598
 599         # start clients[0], wait for it's tub to be ready (at which point it
 600         # will have registered the helper furl).
 601         c = self.add_service(client.Client(basedir=basedirs[0]))
 602         self.clients.append(c)
 603         c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 604         d = c.when_tub_ready()
 605         def _ready(res):
 606             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 607             helper_furl = f.read()
 608             f.close()
 609             self.helper_furl = helper_furl
 610             if self.numclients >= 4:
 611                 f = open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+')
 612                 f.write(
 613                       "[client]\n"
 614                       "helper.furl = %s\n" % helper_furl)
 615                 f.close()
 616
 617             # this starts the rest of the clients
 618             for i in range(1, self.numclients):
 619                 c = self.add_service(client.Client(basedir=basedirs[i]))
 620                 self.clients.append(c)
 621                 c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 622             log.msg("STARTING")
 623             return self.wait_for_connections()
 624         d.addCallback(_ready)
 625         def _connected(res):
 626             log.msg("CONNECTED")
 627             # now find out where the web port was
 628             self.webish_url = self.clients[0].getServiceNamed("webish").getURL()
 629             if self.numclients >=4:
 630                 # and the helper-using webport
 631                 self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL()
 632         d.addCallback(_connected)
 633         return d
 634
 635     def _set_up_nodes_extra_config(self):
 636         # for overriding by subclasses
 637         pass
 638
 639     def _grab_stats(self, res):
 640         d = self.stats_gatherer.poll()
 641         return d
 642
 643     def bounce_client(self, num):
 644         c = self.clients[num]
 645         d = c.disownServiceParent()
 646         # I think windows requires a moment to let the connection really stop
 647         # and the port number made available for re-use. TODO: examine the
 648         # behavior, see if this is really the problem, see if we can do
 649         # better than blindly waiting for a second.
 650         d.addCallback(self.stall, 1.0)
 651         def _stopped(res):
 652             new_c = client.Client(basedir=self.getdir("client%d" % num))
 653             self.clients[num] = new_c
 654             new_c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 655             self.add_service(new_c)
 656             return new_c.when_tub_ready()
 657         d.addCallback(_stopped)
 658         d.addCallback(lambda res: self.wait_for_connections())
 659         def _maybe_get_webport(res):
 660             if num == 0:
 661                 # now find out where the web port was
 662                 self.webish_url = self.clients[0].getServiceNamed("webish").getURL()
 663         d.addCallback(_maybe_get_webport)
 664         return d
 665
 666     def add_extra_node(self, client_num, helper_furl=None,
 667                        add_to_sparent=False):
 668         # usually this node is *not* parented to our self.sparent, so we can
 669         # shut it down separately from the rest, to exercise the
 670         # connection-lost code
 671         basedir = self.getdir("client%d" % client_num)
 672         if not os.path.isdir(basedir):
 673             fileutil.make_dirs(basedir)
 674         config = "[client]\n"
 675         config += "introducer.furl = %s\n" % self.introducer_furl
 676         if helper_furl:
 677             config += "helper.furl = %s\n" % helper_furl
 678         fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
 679
 680         c = client.Client(basedir=basedir)
 681         self.clients.append(c)
 682         c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 683         self.numclients += 1
 684         if add_to_sparent:
 685             c.setServiceParent(self.sparent)
 686         else:
 687             c.startService()
 688         d = self.wait_for_connections()
 689         d.addCallback(lambda res: c)
 690         return d
 691
 692     def _check_connections(self):
 693         for c in self.clients:
 694             if not c.connected_to_introducer():
 695                 return False
 696             sb = c.get_storage_broker()
 697             if len(sb.get_connected_servers()) != self.numclients:
 698                 return False
 699         return True
 700
 701     def wait_for_connections(self, ignored=None):
 702         # TODO: replace this with something that takes a list of peerids and
 703         # fires when they've all been heard from, instead of using a count
 704         # and a threshold
 705         return self.poll(self._check_connections, timeout=200)
 706
 707
 708 # our system test uses the same Tub certificates each time, to avoid the
 709 # overhead of key generation
 710 SYSTEM_TEST_CERTS = [
 711 """-----BEGIN CERTIFICATE-----
 712 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 713 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 714 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 715 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 716 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 717 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 718 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 719 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 720 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 721 -----END CERTIFICATE-----
 722 -----BEGIN RSA PRIVATE KEY-----
 723 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 724 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 725 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 726 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 727 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 728 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 729 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 730 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 731 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 732 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 733 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 734 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 735 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 736 -----END RSA PRIVATE KEY-----
 737 """, # 0
 738 """-----BEGIN CERTIFICATE-----
 739 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 740 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 741 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 742 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 743 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 744 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 745 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 746 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 747 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 748 -----END CERTIFICATE-----
 749 -----BEGIN RSA PRIVATE KEY-----
 750 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 751 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 752 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 753 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 754 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 755 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 756 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 757 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 758 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 759 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 760 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 761 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 762 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 763 -----END RSA PRIVATE KEY-----
 764 """, # 1
 765 """-----BEGIN CERTIFICATE-----
 766 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 767 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 768 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 769 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 770 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 771 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 772 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 773 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 774 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 775 -----END CERTIFICATE-----
 776 -----BEGIN RSA PRIVATE KEY-----
 777 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 778 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 779 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 780 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 781 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 782 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 783 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 784 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 785 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 786 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 787 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 788 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 789 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 790 -----END RSA PRIVATE KEY-----
 791 """, # 2
 792 """-----BEGIN CERTIFICATE-----
 793 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 794 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 795 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 796 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 797 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 798 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 799 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 800 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 801 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 802 -----END CERTIFICATE-----
 803 -----BEGIN RSA PRIVATE KEY-----
 804 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 805 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 806 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 807 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 808 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 809 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 810 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 811 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 812 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 813 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 814 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 815 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 816 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 817 -----END RSA PRIVATE KEY-----
 818 """, # 3
 819 """-----BEGIN CERTIFICATE-----
 820 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 821 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 822 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 823 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 824 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 825 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 826 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 827 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 828 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 829 -----END CERTIFICATE-----
 830 -----BEGIN RSA PRIVATE KEY-----
 831 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 832 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 833 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 834 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 835 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 836 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 837 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 838 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 839 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 840 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 841 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 842 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 843 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 844 -----END RSA PRIVATE KEY-----
 845 """, # 4
 846 """-----BEGIN CERTIFICATE-----
 847 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 848 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 849 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 850 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 851 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 852 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 853 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 854 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 855 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 856 -----END CERTIFICATE-----
 857 -----BEGIN RSA PRIVATE KEY-----
 858 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 859 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 860 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 861 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 862 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 863 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 864 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 865 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 866 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 867 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 868 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 869 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 870 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 871 -----END RSA PRIVATE KEY-----
 872 """, # 5
 873 """-----BEGIN CERTIFICATE-----
 874 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 875 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 876 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 877 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 878 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 879 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 880 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 881 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 882 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 883 -----END CERTIFICATE-----
 884 -----BEGIN RSA PRIVATE KEY-----
 885 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 886 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 887 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 888 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 889 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 890 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 891 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 892 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 893 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 894 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 895 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 896 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 897 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 898 -----END RSA PRIVATE KEY-----
 899 """, #6
 900 """-----BEGIN CERTIFICATE-----
 901 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 902 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 903 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 904 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 905 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 906 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 907 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 908 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 909 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 910 -----END CERTIFICATE-----
 911 -----BEGIN RSA PRIVATE KEY-----
 912 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 913 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 914 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 915 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 916 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 917 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 918 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 919 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 920 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 921 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 922 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 923 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 924 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 925 -----END RSA PRIVATE KEY-----
 926 """, #7
 927 """-----BEGIN CERTIFICATE-----
 928 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 929 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 930 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 931 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 932 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 933 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 934 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 935 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 936 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 937 -----END CERTIFICATE-----
 938 -----BEGIN RSA PRIVATE KEY-----
 939 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 940 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 941 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 942 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 943 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 944 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 945 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 946 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 947 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 948 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 949 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 950 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 951 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 952 -----END RSA PRIVATE KEY-----
 953 """, #8
 954 """-----BEGIN CERTIFICATE-----
 955 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 956 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 957 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 958 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 959 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 960 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 961 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 962 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 963 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 964 -----END CERTIFICATE-----
 965 -----BEGIN RSA PRIVATE KEY-----
 966 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 967 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 968 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 969 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 970 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 971 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 972 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 973 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 974 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 975 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 976 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 977 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 978 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 979 -----END RSA PRIVATE KEY-----
 980 """, #9
 981 """-----BEGIN CERTIFICATE-----
 982 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 983 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 984 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 985 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 986 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 987 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 988 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 989 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 990 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 991 -----END CERTIFICATE-----
 992 -----BEGIN RSA PRIVATE KEY-----
 993 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 994 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 995 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 996 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 997 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 998 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 999 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
1000 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
1001 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
1002 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
1003 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
1004 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
1005 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
1006 -----END RSA PRIVATE KEY-----
1007 """, #10
1008 """-----BEGIN CERTIFICATE-----
1009 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
1010 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
1011 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
1012 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
1013 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
1014 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
1015 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
1016 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
1017 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
1018 -----END CERTIFICATE-----
1019 -----BEGIN RSA PRIVATE KEY-----
1020 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
1021 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
1022 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
1023 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
1024 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
1025 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
1026 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
1027 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
1028 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
1029 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
1030 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
1031 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
1032 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
1033 -----END RSA PRIVATE KEY-----
1034 """, #11
1035 ]
1036
1037 # To disable the pre-computed tub certs, uncomment this line.
1038 #SYSTEM_TEST_CERTS = []
1039
1040 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
1041
1042 class ShouldFailMixin:
1043     def shouldFail(self, expected_failure, which, substring,
1044                    callable, *args, **kwargs):
1045         """Assert that a function call raises some exception. This is a
1046         Deferred-friendly version of TestCase.assertRaises() .
1047
1048         Suppose you want to verify the following function:
1049
1050          def broken(a, b, c):
1051              if a < 0:
1052                  raise TypeError('a must not be negative')
1053              return defer.succeed(b+c)
1054
1055         You can use:
1056             d = self.shouldFail(TypeError, 'test name',
1057                                 'a must not be negative',
1058                                 broken, -4, 5, c=12)
1059         in your test method. The 'test name' string will be included in the
1060         error message, if any, because Deferred chains frequently make it
1061         difficult to tell which assertion was tripped.
1062
1063         The substring= argument, if not None, must appear in the 'repr'
1064         of the message wrapped by this Failure, or the test will fail.
1065         """
1066
1067         assert substring is None or isinstance(substring, str)
1068         d = defer.maybeDeferred(callable, *args, **kwargs)
1069         def done(res):
1070             if isinstance(res, failure.Failure):
1071                 res.trap(expected_failure)
1072                 if substring:
1073                     message = repr(res.value.args[0])
1074                     self.failUnless(substring in message,
1075                                     "substring '%s' not in '%s'"
1076                                     % (substring, message))
1077             else:
1078                 self.fail("%s was supposed to raise %s, not get '%s'" %
1079                           (which, expected_failure, res))
1080         d.addBoth(done)
1081         return d
1082
1083 class WebErrorMixin:
1084     def explain_web_error(self, f):
1085         # an error on the server side causes the client-side getPage() to
1086         # return a failure(t.web.error.Error), and its str() doesn't show the
1087         # response body, which is where the useful information lives. Attach
1088         # this method as an errback handler, and it will reveal the hidden
1089         # message.
1090         f.trap(WebError)
1091         print "Web Error:", f.value, ":", f.value.response
1092         return f
1093
1094     def _shouldHTTPError(self, res, which, validator):
1095         if isinstance(res, failure.Failure):
1096             res.trap(WebError)
1097             return validator(res)
1098         else:
1099             self.fail("%s was supposed to Error, not get '%s'" % (which, res))
1100
1101     def shouldHTTPError(self, which,
1102                         code=None, substring=None, response_substring=None,
1103                         callable=None, *args, **kwargs):
1104         # returns a Deferred with the response body
1105         assert substring is None or isinstance(substring, str)
1106         assert callable
1107         def _validate(f):
1108             if code is not None:
1109                 self.failUnlessEqual(f.value.status, str(code))
1110             if substring:
1111                 code_string = str(f)
1112                 self.failUnless(substring in code_string,
1113                                 "substring '%s' not in '%s'"
1114                                 % (substring, code_string))
1115             response_body = f.value.response
1116             if response_substring:
1117                 self.failUnless(response_substring in response_body,
1118                                 "response substring '%s' not in '%s'"
1119                                 % (response_substring, response_body))
1120             return response_body
1121         d = defer.maybeDeferred(callable, *args, **kwargs)
1122         d.addBoth(self._shouldHTTPError, which, _validate)
1123         return d
1124
1125 class ErrorMixin(WebErrorMixin):
1126     def explain_error(self, f):
1127         if f.check(defer.FirstError):
1128             print "First Error:", f.value.subFailure
1129         return f
1130
1131 def corrupt_field(data, offset, size, debug=False):
1132     if random.random() < 0.5:
1133         newdata = testutil.flip_one_bit(data, offset, size)
1134         if debug:
1135             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1136         return newdata
1137     else:
1138         newval = testutil.insecurerandstr(size)
1139         if debug:
1140             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1141         return data[:offset]+newval+data[offset+size:]
1142
1143 def _corrupt_nothing(data, debug=False):
1144     """Leave the data pristine. """
1145     return data
1146
1147 def _corrupt_file_version_number(data, debug=False):
1148     """Scramble the file data -- the share file version number have one bit
1149     flipped or else will be changed to a random value."""
1150     return corrupt_field(data, 0x00, 4)
1151
1152 def _corrupt_size_of_file_data(data, debug=False):
1153     """Scramble the file data -- the field showing the size of the share data
1154     within the file will be set to one smaller."""
1155     return corrupt_field(data, 0x04, 4)
1156
1157 def _corrupt_sharedata_version_number(data, debug=False):
1158     """Scramble the file data -- the share data version number will have one
1159     bit flipped or else will be changed to a random value, but not 1 or 2."""
1160     return corrupt_field(data, 0x0c, 4)
1161     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1162     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1163     newsharevernum = sharevernum
1164     while newsharevernum in (1, 2):
1165         newsharevernum = random.randrange(0, 2**32)
1166     newsharevernumbytes = struct.pack(">L", newsharevernum)
1167     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1168
1169 def _corrupt_sharedata_version_number_to_plausible_version(data, debug=False):
1170     """Scramble the file data -- the share data version number will be
1171     changed to 2 if it is 1 or else to 1 if it is 2."""
1172     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1173     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1174     if sharevernum == 1:
1175         newsharevernum = 2
1176     else:
1177         newsharevernum = 1
1178     newsharevernumbytes = struct.pack(">L", newsharevernum)
1179     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1180
1181 def _corrupt_segment_size(data, debug=False):
1182     """Scramble the file data -- the field showing the size of the segment
1183     will have one bit flipped or else be changed to a random value."""
1184     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1185     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1186     if sharevernum == 1:
1187         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1188     else:
1189         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1190
1191 def _corrupt_size_of_sharedata(data, debug=False):
1192     """Scramble the file data -- the field showing the size of the data
1193     within the share data will have one bit flipped or else will be changed
1194     to a random value."""
1195     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1196     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1197     if sharevernum == 1:
1198         return corrupt_field(data, 0x0c+0x08, 4)
1199     else:
1200         return corrupt_field(data, 0x0c+0x0c, 8)
1201
1202 def _corrupt_offset_of_sharedata(data, debug=False):
1203     """Scramble the file data -- the field showing the offset of the data
1204     within the share data will have one bit flipped or else be changed to a
1205     random value."""
1206     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1207     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1208     if sharevernum == 1:
1209         return corrupt_field(data, 0x0c+0x0c, 4)
1210     else:
1211         return corrupt_field(data, 0x0c+0x14, 8)
1212
1213 def _corrupt_offset_of_ciphertext_hash_tree(data, debug=False):
1214     """Scramble the file data -- the field showing the offset of the
1215     ciphertext hash tree within the share data will have one bit flipped or
1216     else be changed to a random value.
1217     """
1218     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1219     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1220     if sharevernum == 1:
1221         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1222     else:
1223         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1224
1225 def _corrupt_offset_of_block_hashes(data, debug=False):
1226     """Scramble the file data -- the field showing the offset of the block
1227     hash tree within the share data will have one bit flipped or else will be
1228     changed to a random value."""
1229     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1230     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1231     if sharevernum == 1:
1232         return corrupt_field(data, 0x0c+0x18, 4)
1233     else:
1234         return corrupt_field(data, 0x0c+0x2c, 8)
1235
1236 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data, debug=False):
1237     """Scramble the file data -- the field showing the offset of the block
1238     hash tree within the share data will have a multiple of hash size
1239     subtracted from it, thus causing the downloader to download an incomplete
1240     crypttext hash tree."""
1241     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1242     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1243     if sharevernum == 1:
1244         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1245         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1246         newvalstr = struct.pack(">L", newval)
1247         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1248     else:
1249         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1250         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1251         newvalstr = struct.pack(">Q", newval)
1252         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1253
1254 def _corrupt_offset_of_share_hashes(data, debug=False):
1255     """Scramble the file data -- the field showing the offset of the share
1256     hash tree within the share data will have one bit flipped or else will be
1257     changed to a random value."""
1258     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1259     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1260     if sharevernum == 1:
1261         return corrupt_field(data, 0x0c+0x1c, 4)
1262     else:
1263         return corrupt_field(data, 0x0c+0x34, 8)
1264
1265 def _corrupt_offset_of_uri_extension(data, debug=False):
1266     """Scramble the file data -- the field showing the offset of the uri
1267     extension will have one bit flipped or else will be changed to a random
1268     value."""
1269     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1270     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1271     if sharevernum == 1:
1272         return corrupt_field(data, 0x0c+0x20, 4)
1273     else:
1274         return corrupt_field(data, 0x0c+0x3c, 8)
1275
1276 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1277     """Scramble the file data -- the field showing the offset of the uri
1278     extension will be set to the size of the file minus 3. This means when
1279     the client tries to read the length field from that location it will get
1280     a short read -- the result string will be only 3 bytes long, not the 4 or
1281     8 bytes necessary to do a successful struct.unpack."""
1282     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1283     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1284     # The "-0x0c" in here is to skip the server-side header in the share
1285     # file, which the client doesn't see when seeking and reading.
1286     if sharevernum == 1:
1287         if debug:
1288             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1289         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1290     else:
1291         if debug:
1292             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1293         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1294
1295 def _corrupt_mutable_share_data(data, debug=False):
1296     prefix = data[:32]
1297     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
1298     data_offset = MutableShareFile.DATA_OFFSET
1299     sharetype = data[data_offset:data_offset+1]
1300     assert sharetype == "\x00", "non-SDMF mutable shares not supported"
1301     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
1302      ig_datalen, offsets) = unpack_header(data[data_offset:])
1303     assert version == 0, "this function only handles v0 SDMF files"
1304     start = data_offset + offsets["share_data"]
1305     length = data_offset + offsets["enc_privkey"] - start
1306     return corrupt_field(data, start, length)
1307
1308 def _corrupt_share_data(data, debug=False):
1309     """Scramble the file data -- the field containing the share data itself
1310     will have one bit flipped or else will be changed to a random value."""
1311     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1312     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1313     if sharevernum == 1:
1314         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1315
1316         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1317     else:
1318         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1319
1320         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1321
1322 def _corrupt_share_data_last_byte(data, debug=False):
1323     """Scramble the file data -- flip all bits of the last byte."""
1324     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1325     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1326     if sharevernum == 1:
1327         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1328         offset = 0x0c+0x24+sharedatasize-1
1329     else:
1330         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1331         offset = 0x0c+0x44+sharedatasize-1
1332
1333     newdata = data[:offset] + chr(ord(data[offset])^0xFF) + data[offset+1:]
1334     if debug:
1335         log.msg("testing: flipping all bits of byte at offset %d: %r, newdata: %r" % (offset, data[offset], newdata[offset]))
1336     return newdata
1337
1338 def _corrupt_crypttext_hash_tree(data, debug=False):
1339     """Scramble the file data -- the field containing the crypttext hash tree
1340     will have one bit flipped or else will be changed to a random value.
1341     """
1342     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1343     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1344     if sharevernum == 1:
1345         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1346         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1347     else:
1348         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1349         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1350
1351     return corrupt_field(data, 0x0c+crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset, debug=debug)
1352
1353 def _corrupt_crypttext_hash_tree_byte_x221(data, debug=False):
1354     """Scramble the file data -- the byte at offset 0x221 will have its 7th
1355     (b1) bit flipped.
1356     """
1357     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1358     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1359     if debug:
1360         log.msg("original data: %r" % (data,))
1361     return data[:0x0c+0x221] + chr(ord(data[0x0c+0x221])^0x02) + data[0x0c+0x2210+1:]
1362
1363 def _corrupt_block_hashes(data, debug=False):
1364     """Scramble the file data -- the field containing the block hash tree
1365     will have one bit flipped or else will be changed to a random value.
1366     """
1367     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1368     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1369     if sharevernum == 1:
1370         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1371         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1372     else:
1373         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1374         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1375
1376     return corrupt_field(data, 0x0c+blockhashesoffset, sharehashesoffset-blockhashesoffset)
1377
1378 def _corrupt_share_hashes(data, debug=False):
1379     """Scramble the file data -- the field containing the share hash chain
1380     will have one bit flipped or else will be changed to a random value.
1381     """
1382     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1383     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1384     if sharevernum == 1:
1385         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1386         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1387     else:
1388         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1389         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1390
1391     return corrupt_field(data, 0x0c+sharehashesoffset, uriextoffset-sharehashesoffset)
1392
1393 def _corrupt_length_of_uri_extension(data, debug=False):
1394     """Scramble the file data -- the field showing the length of the uri
1395     extension will have one bit flipped or else will be changed to a random
1396     value."""
1397     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1398     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1399     if sharevernum == 1:
1400         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1401         return corrupt_field(data, uriextoffset, 4)
1402     else:
1403         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1404         return corrupt_field(data, 0x0c+uriextoffset, 8)
1405
1406 def _corrupt_uri_extension(data, debug=False):
1407     """Scramble the file data -- the field containing the uri extension will
1408     have one bit flipped or else will be changed to a random value."""
1409     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1410     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1411     if sharevernum == 1:
1412         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1413         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1414     else:
1415         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1416         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1417
1418     return corrupt_field(data, 0x0c+uriextoffset, uriextlen)