src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IPullProducer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.api import flushEventualQueue, fireEventually
   9 from allmydata import uri, dirnode, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IMutableFileNode, IImmutableFileNode,\
  12                                  NotEnoughSharesError, ICheckable, \
  13                                  IMutableUploadable, SDMF_VERSION, \
  14                                  MDMF_VERSION
  15 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  16      DeepCheckResults, DeepCheckAndRepairResults
  17 from allmydata.mutable.common import CorruptShareError
  18 from allmydata.mutable.layout import unpack_header
  19 from allmydata.mutable.publish import MutableData
  20 from allmydata.storage.mutable import MutableShareFile
  21 from allmydata.util import hashutil, log, fileutil, pollmixin
  22 from allmydata.util.assertutil import precondition
  23 from allmydata.util.consumer import download_to_data
  24 from allmydata.stats import StatsGathererService
  25 from allmydata.key_generator import KeyGeneratorService
  26 import allmydata.test.common_util as testutil
  27 from allmydata import immutable
  28
  29 TEST_RSA_KEY_SIZE = 522
  30
  31 def flush_but_dont_ignore(res):
  32     d = flushEventualQueue()
  33     def _done(ignored):
  34         return res
  35     d.addCallback(_done)
  36     return d
  37
  38 class DummyProducer:
  39     implements(IPullProducer)
  40     def resumeProducing(self):
  41         pass
  42
  43 class FakeCHKFileNode:
  44     """I provide IImmutableFileNode, but all of my data is stored in a
  45     class-level dictionary."""
  46     implements(IImmutableFileNode)
  47     all_contents = {}
  48     bad_shares = {}
  49
  50     def __init__(self, filecap):
  51         precondition(isinstance(filecap, (uri.CHKFileURI, uri.LiteralFileURI)), filecap)
  52         self.my_uri = filecap
  53         self.storage_index = self.my_uri.get_storage_index()
  54
  55     def get_uri(self):
  56         return self.my_uri.to_string()
  57     def get_write_uri(self):
  58         return None
  59     def get_readonly_uri(self):
  60         return self.my_uri.to_string()
  61     def get_cap(self):
  62         return self.my_uri
  63     def get_verify_cap(self):
  64         return self.my_uri.get_verify_cap()
  65     def get_repair_cap(self):
  66         return self.my_uri.get_verify_cap()
  67     def get_storage_index(self):
  68         return self.storage_index
  69
  70     def check(self, monitor, verify=False, add_lease=False):
  71         r = CheckResults(self.my_uri, self.storage_index)
  72         is_bad = self.bad_shares.get(self.storage_index, None)
  73         data = {}
  74         data["count-shares-needed"] = 3
  75         data["count-shares-expected"] = 10
  76         data["count-good-share-hosts"] = 10
  77         data["count-wrong-shares"] = 0
  78         nodeid = "\x00"*20
  79         data["list-corrupt-shares"] = []
  80         data["sharemap"] = {1: [nodeid]}
  81         data["servers-responding"] = [nodeid]
  82         data["count-recoverable-versions"] = 1
  83         data["count-unrecoverable-versions"] = 0
  84         if is_bad:
  85             r.set_healthy(False)
  86             r.set_recoverable(True)
  87             data["count-shares-good"] = 9
  88             data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
  89             r.problems = failure.Failure(CorruptShareError(is_bad))
  90         else:
  91             r.set_healthy(True)
  92             r.set_recoverable(True)
  93             data["count-shares-good"] = 10
  94             r.problems = []
  95         r.set_data(data)
  96         r.set_needs_rebalancing(False)
  97         return defer.succeed(r)
  98     def check_and_repair(self, monitor, verify=False, add_lease=False):
  99         d = self.check(verify)
 100         def _got(cr):
 101             r = CheckAndRepairResults(self.storage_index)
 102             r.pre_repair_results = r.post_repair_results = cr
 103             return r
 104         d.addCallback(_got)
 105         return d
 106
 107     def is_mutable(self):
 108         return False
 109     def is_readonly(self):
 110         return True
 111     def is_unknown(self):
 112         return False
 113     def is_allowed_in_immutable_directory(self):
 114         return True
 115     def raise_error(self):
 116         pass
 117
 118     def get_size(self):
 119         if isinstance(self.my_uri, uri.LiteralFileURI):
 120             return self.my_uri.get_size()
 121         try:
 122             data = self.all_contents[self.my_uri.to_string()]
 123         except KeyError, le:
 124             raise NotEnoughSharesError(le, 0, 3)
 125         return len(data)
 126
 127     def read(self, consumer, offset=0, size=None):
 128         # we don't bother to call registerProducer/unregisterProducer,
 129         # because it's a hassle to write a dummy Producer that does the right
 130         # thing (we have to make sure that DummyProducer.resumeProducing
 131         # writes the data into the consumer immediately, otherwise it will
 132         # loop forever).
 133
 134         d = defer.succeed(None)
 135         d.addCallback(self._read, consumer, offset, size)
 136         return d
 137
 138     def _read(self, ignored, consumer, offset, size):
 139         if isinstance(self.my_uri, uri.LiteralFileURI):
 140             data = self.my_uri.data
 141         else:
 142             if self.my_uri.to_string() not in self.all_contents:
 143                 raise NotEnoughSharesError(None, 0, 3)
 144             data = self.all_contents[self.my_uri.to_string()]
 145         start = offset
 146         if size is not None:
 147             end = offset + size
 148         else:
 149             end = len(data)
 150         consumer.write(data[start:end])
 151         return consumer
 152
 153
 154     def get_best_readable_version(self):
 155         return defer.succeed(self)
 156
 157
 158     def download_to_data(self):
 159         return download_to_data(self)
 160
 161
 162     download_best_version = download_to_data
 163
 164
 165     def get_size_of_best_version(self):
 166         return defer.succeed(self.get_size)
 167
 168
 169 def make_chk_file_cap(size):
 170     return uri.CHKFileURI(key=os.urandom(16),
 171                           uri_extension_hash=os.urandom(32),
 172                           needed_shares=3,
 173                           total_shares=10,
 174                           size=size)
 175 def make_chk_file_uri(size):
 176     return make_chk_file_cap(size).to_string()
 177
 178 def create_chk_filenode(contents):
 179     filecap = make_chk_file_cap(len(contents))
 180     n = FakeCHKFileNode(filecap)
 181     FakeCHKFileNode.all_contents[filecap.to_string()] = contents
 182     return n
 183
 184
 185 class FakeMutableFileNode:
 186     """I provide IMutableFileNode, but all of my data is stored in a
 187     class-level dictionary."""
 188
 189     implements(IMutableFileNode, ICheckable)
 190     MUTABLE_SIZELIMIT = 10000
 191     all_contents = {}
 192     bad_shares = {}
 193     file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION
 194
 195     def __init__(self, storage_broker, secret_holder,
 196                  default_encoding_parameters, history):
 197         self.init_from_cap(make_mutable_file_cap())
 198         self._k = default_encoding_parameters['k']
 199         self._segsize = default_encoding_parameters['max_segment_size']
 200     def create(self, contents, key_generator=None, keysize=None,
 201                version=SDMF_VERSION):
 202         if version == MDMF_VERSION and \
 203             isinstance(self.my_uri, (uri.ReadonlySSKFileURI,
 204                                  uri.WriteableSSKFileURI)):
 205             self.init_from_cap(make_mdmf_mutable_file_cap())
 206         self.file_types[self.storage_index] = version
 207         initial_contents = self._get_initial_contents(contents)
 208         data = initial_contents.read(initial_contents.get_size())
 209         data = "".join(data)
 210         self.all_contents[self.storage_index] = data
 211         return defer.succeed(self)
 212     def _get_initial_contents(self, contents):
 213         if contents is None:
 214             return MutableData("")
 215
 216         if IMutableUploadable.providedBy(contents):
 217             return contents
 218
 219         assert callable(contents), "%s should be callable, not %s" % \
 220                (contents, type(contents))
 221         return contents(self)
 222     def init_from_cap(self, filecap):
 223         assert isinstance(filecap, (uri.WriteableSSKFileURI,
 224                                     uri.ReadonlySSKFileURI,
 225                                     uri.WriteableMDMFFileURI,
 226                                     uri.ReadonlyMDMFFileURI))
 227         self.my_uri = filecap
 228         self.storage_index = self.my_uri.get_storage_index()
 229         if isinstance(filecap, (uri.WriteableMDMFFileURI,
 230                                 uri.ReadonlyMDMFFileURI)):
 231             self.file_types[self.storage_index] = MDMF_VERSION
 232
 233         else:
 234             self.file_types[self.storage_index] = SDMF_VERSION
 235
 236         return self
 237     def get_cap(self):
 238         return self.my_uri
 239     def get_readcap(self):
 240         return self.my_uri.get_readonly()
 241     def get_uri(self):
 242         return self.my_uri.to_string()
 243     def get_write_uri(self):
 244         if self.is_readonly():
 245             return None
 246         return self.my_uri.to_string()
 247     def get_readonly(self):
 248         return self.my_uri.get_readonly()
 249     def get_readonly_uri(self):
 250         return self.my_uri.get_readonly().to_string()
 251     def get_verify_cap(self):
 252         return self.my_uri.get_verify_cap()
 253     def get_repair_cap(self):
 254         if self.my_uri.is_readonly():
 255             return None
 256         return self.my_uri
 257     def is_readonly(self):
 258         return self.my_uri.is_readonly()
 259     def is_mutable(self):
 260         return self.my_uri.is_mutable()
 261     def is_unknown(self):
 262         return False
 263     def is_allowed_in_immutable_directory(self):
 264         return not self.my_uri.is_mutable()
 265     def raise_error(self):
 266         pass
 267     def get_writekey(self):
 268         return "\x00"*16
 269     def get_size(self):
 270         return len(self.all_contents[self.storage_index])
 271     def get_current_size(self):
 272         return self.get_size_of_best_version()
 273     def get_size_of_best_version(self):
 274         return defer.succeed(len(self.all_contents[self.storage_index]))
 275
 276     def get_storage_index(self):
 277         return self.storage_index
 278
 279     def get_servermap(self, mode):
 280         return defer.succeed(None)
 281
 282     def get_version(self):
 283         assert self.storage_index in self.file_types
 284         return self.file_types[self.storage_index]
 285
 286     def check(self, monitor, verify=False, add_lease=False):
 287         r = CheckResults(self.my_uri, self.storage_index)
 288         is_bad = self.bad_shares.get(self.storage_index, None)
 289         data = {}
 290         data["count-shares-needed"] = 3
 291         data["count-shares-expected"] = 10
 292         data["count-good-share-hosts"] = 10
 293         data["count-wrong-shares"] = 0
 294         data["list-corrupt-shares"] = []
 295         nodeid = "\x00"*20
 296         data["sharemap"] = {"seq1-abcd-sh0": [nodeid]}
 297         data["servers-responding"] = [nodeid]
 298         data["count-recoverable-versions"] = 1
 299         data["count-unrecoverable-versions"] = 0
 300         if is_bad:
 301             r.set_healthy(False)
 302             r.set_recoverable(True)
 303             data["count-shares-good"] = 9
 304             r.problems = failure.Failure(CorruptShareError("peerid",
 305                                                            0, # shnum
 306                                                            is_bad))
 307         else:
 308             r.set_healthy(True)
 309             r.set_recoverable(True)
 310             data["count-shares-good"] = 10
 311             r.problems = []
 312         r.set_data(data)
 313         r.set_needs_rebalancing(False)
 314         return defer.succeed(r)
 315
 316     def check_and_repair(self, monitor, verify=False, add_lease=False):
 317         d = self.check(verify)
 318         def _got(cr):
 319             r = CheckAndRepairResults(self.storage_index)
 320             r.pre_repair_results = r.post_repair_results = cr
 321             return r
 322         d.addCallback(_got)
 323         return d
 324
 325     def deep_check(self, verify=False, add_lease=False):
 326         d = self.check(verify)
 327         def _done(r):
 328             dr = DeepCheckResults(self.storage_index)
 329             dr.add_check(r, [])
 330             return dr
 331         d.addCallback(_done)
 332         return d
 333
 334     def deep_check_and_repair(self, verify=False, add_lease=False):
 335         d = self.check_and_repair(verify)
 336         def _done(r):
 337             dr = DeepCheckAndRepairResults(self.storage_index)
 338             dr.add_check(r, [])
 339             return dr
 340         d.addCallback(_done)
 341         return d
 342
 343     def download_best_version(self):
 344         return defer.succeed(self._download_best_version())
 345
 346
 347     def _download_best_version(self, ignored=None):
 348         if isinstance(self.my_uri, uri.LiteralFileURI):
 349             return self.my_uri.data
 350         if self.storage_index not in self.all_contents:
 351             raise NotEnoughSharesError(None, 0, 3)
 352         return self.all_contents[self.storage_index]
 353
 354
 355     def overwrite(self, new_contents):
 356         assert not self.is_readonly()
 357         new_data = new_contents.read(new_contents.get_size())
 358         new_data = "".join(new_data)
 359         self.all_contents[self.storage_index] = new_data
 360         return defer.succeed(None)
 361     def modify(self, modifier):
 362         # this does not implement FileTooLargeError, but the real one does
 363         return defer.maybeDeferred(self._modify, modifier)
 364     def _modify(self, modifier):
 365         assert not self.is_readonly()
 366         old_contents = self.all_contents[self.storage_index]
 367         new_data = modifier(old_contents, None, True)
 368         self.all_contents[self.storage_index] = new_data
 369         return None
 370
 371     # As actually implemented, MutableFilenode and MutableFileVersion
 372     # are distinct. However, nothing in the webapi uses (yet) that
 373     # distinction -- it just uses the unified download interface
 374     # provided by get_best_readable_version and read. When we start
 375     # doing cooler things like LDMF, we will want to revise this code to
 376     # be less simplistic.
 377     def get_best_readable_version(self):
 378         return defer.succeed(self)
 379
 380
 381     def get_best_mutable_version(self):
 382         return defer.succeed(self)
 383
 384     # Ditto for this, which is an implementation of IWriteable.
 385     # XXX: Declare that the same is implemented.
 386     def update(self, data, offset):
 387         assert not self.is_readonly()
 388         def modifier(old, servermap, first_time):
 389             new = old[:offset] + "".join(data.read(data.get_size()))
 390             new += old[len(new):]
 391             return new
 392         return self.modify(modifier)
 393
 394
 395     def read(self, consumer, offset=0, size=None):
 396         data = self._download_best_version()
 397         if size:
 398             data = data[offset:offset+size]
 399         consumer.write(data)
 400         return defer.succeed(consumer)
 401
 402
 403 def make_mutable_file_cap():
 404     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 405                                    fingerprint=os.urandom(32))
 406
 407 def make_mdmf_mutable_file_cap():
 408     return uri.WriteableMDMFFileURI(writekey=os.urandom(16),
 409                                    fingerprint=os.urandom(32))
 410
 411 def make_mutable_file_uri(mdmf=False):
 412     if mdmf:
 413         uri = make_mdmf_mutable_file_cap()
 414     else:
 415         uri = make_mutable_file_cap()
 416
 417     return uri.to_string()
 418
 419 def make_verifier_uri():
 420     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 421                               fingerprint=os.urandom(32)).to_string()
 422
 423 def create_mutable_filenode(contents, mdmf=False):
 424     # XXX: All of these arguments are kind of stupid.
 425     if mdmf:
 426         cap = make_mdmf_mutable_file_cap()
 427     else:
 428         cap = make_mutable_file_cap()
 429
 430     encoding_params = {}
 431     encoding_params['k'] = 3
 432     encoding_params['max_segment_size'] = 128*1024
 433
 434     filenode = FakeMutableFileNode(None, None, encoding_params, None)
 435     filenode.init_from_cap(cap)
 436     if mdmf:
 437         filenode.create(MutableData(contents), version=MDMF_VERSION)
 438     else:
 439         filenode.create(MutableData(contents), version=SDMF_VERSION)
 440     return filenode
 441
 442
 443 class FakeDirectoryNode(dirnode.DirectoryNode):
 444     """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
 445     backing store, so it doesn't go to the grid. The child data is still
 446     encrypted and serialized, so this isn't useful for tests that want to
 447     look inside the dirnodes and check their contents.
 448     """
 449     filenode_class = FakeMutableFileNode
 450
 451 class LoggingServiceParent(service.MultiService):
 452     def log(self, *args, **kwargs):
 453         return log.msg(*args, **kwargs)
 454
 455 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 456
 457     # SystemTestMixin tests tend to be a lot of work, and we have a few
 458     # buildslaves that are pretty slow, and every once in a while these tests
 459     # run up against the default 120 second timeout. So increase the default
 460     # timeout. Individual test cases can override this, of course.
 461     timeout = 300
 462
 463     def setUp(self):
 464         self.sparent = service.MultiService()
 465         self.sparent.startService()
 466
 467         self.stats_gatherer = None
 468         self.stats_gatherer_furl = None
 469         self.key_generator_svc = None
 470         self.key_generator_furl = None
 471
 472     def tearDown(self):
 473         log.msg("shutting down SystemTest services")
 474         d = self.sparent.stopService()
 475         d.addBoth(flush_but_dont_ignore)
 476         return d
 477
 478     def getdir(self, subdir):
 479         return os.path.join(self.basedir, subdir)
 480
 481     def add_service(self, s):
 482         s.setServiceParent(self.sparent)
 483         return s
 484
 485     def set_up_nodes(self, NUMCLIENTS=5,
 486                      use_stats_gatherer=False, use_key_generator=False):
 487         self.numclients = NUMCLIENTS
 488         iv_dir = self.getdir("introducer")
 489         if not os.path.isdir(iv_dir):
 490             fileutil.make_dirs(iv_dir)
 491             fileutil.write(os.path.join(iv_dir, 'tahoe.cfg'), \
 492                                "[node]\n" + \
 493                                "web.port = tcp:0:interface=127.0.0.1\n")
 494             if SYSTEM_TEST_CERTS:
 495                 os.mkdir(os.path.join(iv_dir, "private"))
 496                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 497                 f.write(SYSTEM_TEST_CERTS[0])
 498                 f.close()
 499         iv = IntroducerNode(basedir=iv_dir)
 500         self.introducer = self.add_service(iv)
 501         d = self.introducer.when_tub_ready()
 502         d.addCallback(self._get_introducer_web)
 503         if use_stats_gatherer:
 504             d.addCallback(self._set_up_stats_gatherer)
 505         if use_key_generator:
 506             d.addCallback(self._set_up_key_generator)
 507         d.addCallback(self._set_up_nodes_2)
 508         if use_stats_gatherer:
 509             d.addCallback(self._grab_stats)
 510         return d
 511
 512     def _get_introducer_web(self, res):
 513         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 514         self.introweb_url = f.read().strip()
 515         f.close()
 516
 517     def _set_up_stats_gatherer(self, res):
 518         statsdir = self.getdir("stats_gatherer")
 519         fileutil.make_dirs(statsdir)
 520         self.stats_gatherer_svc = StatsGathererService(statsdir)
 521         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 522         self.add_service(self.stats_gatherer_svc)
 523
 524         d = fireEventually()
 525         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 526         def check_for_furl():
 527             return os.path.exists(sgf)
 528         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 529         def get_furl(junk):
 530             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 531         d.addCallback(get_furl)
 532         return d
 533
 534     def _set_up_key_generator(self, res):
 535         kgsdir = self.getdir("key_generator")
 536         fileutil.make_dirs(kgsdir)
 537
 538         self.key_generator_svc = KeyGeneratorService(kgsdir,
 539                                                      display_furl=False,
 540                                                      default_key_size=TEST_RSA_KEY_SIZE)
 541         self.key_generator_svc.key_generator.pool_size = 4
 542         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 543         self.add_service(self.key_generator_svc)
 544
 545         d = fireEventually()
 546         def check_for_furl():
 547             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 548         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 549         def get_furl(junk):
 550             kgf = os.path.join(kgsdir, 'key_generator.furl')
 551             self.key_generator_furl = file(kgf, 'rb').read().strip()
 552         d.addCallback(get_furl)
 553         return d
 554
 555     def _set_up_nodes_2(self, res):
 556         q = self.introducer
 557         self.introducer_furl = q.introducer_url
 558         self.clients = []
 559         basedirs = []
 560         for i in range(self.numclients):
 561             basedir = self.getdir("client%d" % i)
 562             basedirs.append(basedir)
 563             fileutil.make_dirs(os.path.join(basedir, "private"))
 564             if len(SYSTEM_TEST_CERTS) > (i+1):
 565                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 566                 f.write(SYSTEM_TEST_CERTS[i+1])
 567                 f.close()
 568
 569             config = "[client]\n"
 570             config += "introducer.furl = %s\n" % self.introducer_furl
 571             if self.stats_gatherer_furl:
 572                 config += "stats_gatherer.furl = %s\n" % self.stats_gatherer_furl
 573
 574             if i == 0:
 575                 # clients[0] runs a webserver and a helper, no key_generator
 576                 config += "[node]\n"
 577                 config += "web.port = tcp:0:interface=127.0.0.1\n"
 578                 config += "timeout.keepalive = 600\n"
 579                 config += "[helper]\n"
 580                 config += "enabled = True\n"
 581             if i == 3:
 582                 # clients[3] runs a webserver and uses a helper, uses
 583                 # key_generator
 584                 if self.key_generator_furl:
 585                     config += "key_generator.furl = %s\n" % self.key_generator_furl
 586                 config += "[node]\n"
 587                 config += "web.port = tcp:0:interface=127.0.0.1\n"
 588                 config += "timeout.disconnect = 1800\n"
 589
 590             fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
 591
 592         # give subclasses a chance to append lines to the node's tahoe.cfg
 593         # files before they are launched.
 594         self._set_up_nodes_extra_config()
 595
 596         # start clients[0], wait for it's tub to be ready (at which point it
 597         # will have registered the helper furl).
 598         c = self.add_service(client.Client(basedir=basedirs[0]))
 599         self.clients.append(c)
 600         c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 601         d = c.when_tub_ready()
 602         def _ready(res):
 603             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 604             helper_furl = f.read()
 605             f.close()
 606             self.helper_furl = helper_furl
 607             if self.numclients >= 4:
 608                 f = open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+')
 609                 f.write(
 610                       "[client]\n"
 611                       "helper.furl = %s\n" % helper_furl)
 612                 f.close()
 613
 614             # this starts the rest of the clients
 615             for i in range(1, self.numclients):
 616                 c = self.add_service(client.Client(basedir=basedirs[i]))
 617                 self.clients.append(c)
 618                 c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 619             log.msg("STARTING")
 620             return self.wait_for_connections()
 621         d.addCallback(_ready)
 622         def _connected(res):
 623             log.msg("CONNECTED")
 624             # now find out where the web port was
 625             self.webish_url = self.clients[0].getServiceNamed("webish").getURL()
 626             if self.numclients >=4:
 627                 # and the helper-using webport
 628                 self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL()
 629         d.addCallback(_connected)
 630         return d
 631
 632     def _set_up_nodes_extra_config(self):
 633         # for overriding by subclasses
 634         pass
 635
 636     def _grab_stats(self, res):
 637         d = self.stats_gatherer.poll()
 638         return d
 639
 640     def bounce_client(self, num):
 641         c = self.clients[num]
 642         d = c.disownServiceParent()
 643         # I think windows requires a moment to let the connection really stop
 644         # and the port number made available for re-use. TODO: examine the
 645         # behavior, see if this is really the problem, see if we can do
 646         # better than blindly waiting for a second.
 647         d.addCallback(self.stall, 1.0)
 648         def _stopped(res):
 649             new_c = client.Client(basedir=self.getdir("client%d" % num))
 650             self.clients[num] = new_c
 651             new_c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 652             self.add_service(new_c)
 653             return new_c.when_tub_ready()
 654         d.addCallback(_stopped)
 655         d.addCallback(lambda res: self.wait_for_connections())
 656         def _maybe_get_webport(res):
 657             if num == 0:
 658                 # now find out where the web port was
 659                 self.webish_url = self.clients[0].getServiceNamed("webish").getURL()
 660         d.addCallback(_maybe_get_webport)
 661         return d
 662
 663     def add_extra_node(self, client_num, helper_furl=None,
 664                        add_to_sparent=False):
 665         # usually this node is *not* parented to our self.sparent, so we can
 666         # shut it down separately from the rest, to exercise the
 667         # connection-lost code
 668         basedir = self.getdir("client%d" % client_num)
 669         if not os.path.isdir(basedir):
 670             fileutil.make_dirs(basedir)
 671         config = "[client]\n"
 672         config += "introducer.furl = %s\n" % self.introducer_furl
 673         if helper_furl:
 674             config += "helper.furl = %s\n" % helper_furl
 675         fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
 676
 677         c = client.Client(basedir=basedir)
 678         self.clients.append(c)
 679         c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 680         self.numclients += 1
 681         if add_to_sparent:
 682             c.setServiceParent(self.sparent)
 683         else:
 684             c.startService()
 685         d = self.wait_for_connections()
 686         d.addCallback(lambda res: c)
 687         return d
 688
 689     def _check_connections(self):
 690         for c in self.clients:
 691             if not c.connected_to_introducer():
 692                 return False
 693             sb = c.get_storage_broker()
 694             if len(sb.get_connected_servers()) != self.numclients:
 695                 return False
 696         return True
 697
 698     def wait_for_connections(self, ignored=None):
 699         # TODO: replace this with something that takes a list of peerids and
 700         # fires when they've all been heard from, instead of using a count
 701         # and a threshold
 702         return self.poll(self._check_connections, timeout=200)
 703
 704
 705 # our system test uses the same Tub certificates each time, to avoid the
 706 # overhead of key generation
 707 SYSTEM_TEST_CERTS = [
 708 """-----BEGIN CERTIFICATE-----
 709 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 710 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 711 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 712 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 713 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 714 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 715 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 716 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 717 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 718 -----END CERTIFICATE-----
 719 -----BEGIN RSA PRIVATE KEY-----
 720 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 721 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 722 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 723 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 724 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 725 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 726 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 727 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 728 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 729 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 730 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 731 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 732 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 733 -----END RSA PRIVATE KEY-----
 734 """, # 0
 735 """-----BEGIN CERTIFICATE-----
 736 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 737 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 738 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 739 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 740 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 741 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 742 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 743 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 744 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 745 -----END CERTIFICATE-----
 746 -----BEGIN RSA PRIVATE KEY-----
 747 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 748 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 749 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 750 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 751 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 752 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 753 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 754 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 755 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 756 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 757 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 758 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 759 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 760 -----END RSA PRIVATE KEY-----
 761 """, # 1
 762 """-----BEGIN CERTIFICATE-----
 763 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 764 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 765 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 766 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 767 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 768 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 769 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 770 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 771 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 772 -----END CERTIFICATE-----
 773 -----BEGIN RSA PRIVATE KEY-----
 774 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 775 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 776 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 777 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 778 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 779 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 780 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 781 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 782 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 783 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 784 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 785 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 786 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 787 -----END RSA PRIVATE KEY-----
 788 """, # 2
 789 """-----BEGIN CERTIFICATE-----
 790 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 791 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 792 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 793 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 794 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 795 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 796 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 797 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 798 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 799 -----END CERTIFICATE-----
 800 -----BEGIN RSA PRIVATE KEY-----
 801 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 802 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 803 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 804 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 805 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 806 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 807 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 808 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 809 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 810 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 811 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 812 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 813 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 814 -----END RSA PRIVATE KEY-----
 815 """, # 3
 816 """-----BEGIN CERTIFICATE-----
 817 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 818 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 819 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 820 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 821 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 822 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 823 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 824 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 825 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 826 -----END CERTIFICATE-----
 827 -----BEGIN RSA PRIVATE KEY-----
 828 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 829 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 830 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 831 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 832 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 833 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 834 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 835 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 836 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 837 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 838 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 839 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 840 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 841 -----END RSA PRIVATE KEY-----
 842 """, # 4
 843 """-----BEGIN CERTIFICATE-----
 844 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 845 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 846 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 847 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 848 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 849 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 850 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 851 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 852 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 853 -----END CERTIFICATE-----
 854 -----BEGIN RSA PRIVATE KEY-----
 855 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 856 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 857 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 858 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 859 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 860 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 861 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 862 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 863 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 864 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 865 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 866 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 867 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 868 -----END RSA PRIVATE KEY-----
 869 """, # 5
 870 """-----BEGIN CERTIFICATE-----
 871 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 872 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 873 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 874 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 875 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 876 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 877 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 878 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 879 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 880 -----END CERTIFICATE-----
 881 -----BEGIN RSA PRIVATE KEY-----
 882 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 883 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 884 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 885 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 886 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 887 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 888 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 889 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 890 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 891 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 892 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 893 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 894 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 895 -----END RSA PRIVATE KEY-----
 896 """, #6
 897 """-----BEGIN CERTIFICATE-----
 898 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 899 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 900 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 901 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 902 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 903 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 904 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 905 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 906 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 907 -----END CERTIFICATE-----
 908 -----BEGIN RSA PRIVATE KEY-----
 909 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 910 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 911 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 912 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 913 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 914 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 915 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 916 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 917 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 918 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 919 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 920 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 921 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 922 -----END RSA PRIVATE KEY-----
 923 """, #7
 924 """-----BEGIN CERTIFICATE-----
 925 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 926 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 927 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 928 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 929 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 930 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 931 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 932 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 933 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 934 -----END CERTIFICATE-----
 935 -----BEGIN RSA PRIVATE KEY-----
 936 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 937 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 938 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 939 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 940 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 941 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 942 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 943 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 944 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 945 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 946 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 947 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 948 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 949 -----END RSA PRIVATE KEY-----
 950 """, #8
 951 """-----BEGIN CERTIFICATE-----
 952 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 953 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 954 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 955 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 956 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 957 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 958 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 959 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 960 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 961 -----END CERTIFICATE-----
 962 -----BEGIN RSA PRIVATE KEY-----
 963 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 964 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 965 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 966 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 967 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 968 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 969 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 970 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 971 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 972 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 973 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 974 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 975 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 976 -----END RSA PRIVATE KEY-----
 977 """, #9
 978 """-----BEGIN CERTIFICATE-----
 979 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 980 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 981 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 982 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 983 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 984 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 985 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 986 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 987 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 988 -----END CERTIFICATE-----
 989 -----BEGIN RSA PRIVATE KEY-----
 990 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 991 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 992 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 993 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 994 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 995 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 996 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
 997 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
 998 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
 999 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
1000 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
1001 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
1002 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
1003 -----END RSA PRIVATE KEY-----
1004 """, #10
1005 """-----BEGIN CERTIFICATE-----
1006 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
1007 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
1008 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
1009 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
1010 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
1011 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
1012 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
1013 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
1014 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
1015 -----END CERTIFICATE-----
1016 -----BEGIN RSA PRIVATE KEY-----
1017 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
1018 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
1019 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
1020 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
1021 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
1022 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
1023 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
1024 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
1025 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
1026 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
1027 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
1028 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
1029 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
1030 -----END RSA PRIVATE KEY-----
1031 """, #11
1032 ]
1033
1034 # To disable the pre-computed tub certs, uncomment this line.
1035 #SYSTEM_TEST_CERTS = []
1036
1037 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
1038
1039 class ShouldFailMixin:
1040     def shouldFail(self, expected_failure, which, substring,
1041                    callable, *args, **kwargs):
1042         """Assert that a function call raises some exception. This is a
1043         Deferred-friendly version of TestCase.assertRaises() .
1044
1045         Suppose you want to verify the following function:
1046
1047          def broken(a, b, c):
1048              if a < 0:
1049                  raise TypeError('a must not be negative')
1050              return defer.succeed(b+c)
1051
1052         You can use:
1053             d = self.shouldFail(TypeError, 'test name',
1054                                 'a must not be negative',
1055                                 broken, -4, 5, c=12)
1056         in your test method. The 'test name' string will be included in the
1057         error message, if any, because Deferred chains frequently make it
1058         difficult to tell which assertion was tripped.
1059
1060         The substring= argument, if not None, must appear in the 'repr'
1061         of the message wrapped by this Failure, or the test will fail.
1062         """
1063
1064         assert substring is None or isinstance(substring, str)
1065         d = defer.maybeDeferred(callable, *args, **kwargs)
1066         def done(res):
1067             if isinstance(res, failure.Failure):
1068                 res.trap(expected_failure)
1069                 if substring:
1070                     message = repr(res.value.args[0])
1071                     self.failUnless(substring in message,
1072                                     "substring '%s' not in '%s'"
1073                                     % (substring, message))
1074             else:
1075                 self.fail("%s was supposed to raise %s, not get '%s'" %
1076                           (which, expected_failure, res))
1077         d.addBoth(done)
1078         return d
1079
1080 class WebErrorMixin:
1081     def explain_web_error(self, f):
1082         # an error on the server side causes the client-side getPage() to
1083         # return a failure(t.web.error.Error), and its str() doesn't show the
1084         # response body, which is where the useful information lives. Attach
1085         # this method as an errback handler, and it will reveal the hidden
1086         # message.
1087         f.trap(WebError)
1088         print "Web Error:", f.value, ":", f.value.response
1089         return f
1090
1091     def _shouldHTTPError(self, res, which, validator):
1092         if isinstance(res, failure.Failure):
1093             res.trap(WebError)
1094             return validator(res)
1095         else:
1096             self.fail("%s was supposed to Error, not get '%s'" % (which, res))
1097
1098     def shouldHTTPError(self, which,
1099                         code=None, substring=None, response_substring=None,
1100                         callable=None, *args, **kwargs):
1101         # returns a Deferred with the response body
1102         assert substring is None or isinstance(substring, str)
1103         assert callable
1104         def _validate(f):
1105             if code is not None:
1106                 self.failUnlessEqual(f.value.status, str(code))
1107             if substring:
1108                 code_string = str(f)
1109                 self.failUnless(substring in code_string,
1110                                 "substring '%s' not in '%s'"
1111                                 % (substring, code_string))
1112             response_body = f.value.response
1113             if response_substring:
1114                 self.failUnless(response_substring in response_body,
1115                                 "response substring '%s' not in '%s'"
1116                                 % (response_substring, response_body))
1117             return response_body
1118         d = defer.maybeDeferred(callable, *args, **kwargs)
1119         d.addBoth(self._shouldHTTPError, which, _validate)
1120         return d
1121
1122 class ErrorMixin(WebErrorMixin):
1123     def explain_error(self, f):
1124         if f.check(defer.FirstError):
1125             print "First Error:", f.value.subFailure
1126         return f
1127
1128 def corrupt_field(data, offset, size, debug=False):
1129     if random.random() < 0.5:
1130         newdata = testutil.flip_one_bit(data, offset, size)
1131         if debug:
1132             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1133         return newdata
1134     else:
1135         newval = testutil.insecurerandstr(size)
1136         if debug:
1137             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1138         return data[:offset]+newval+data[offset+size:]
1139
1140 def _corrupt_nothing(data, debug=False):
1141     """Leave the data pristine. """
1142     return data
1143
1144 def _corrupt_file_version_number(data, debug=False):
1145     """Scramble the file data -- the share file version number have one bit
1146     flipped or else will be changed to a random value."""
1147     return corrupt_field(data, 0x00, 4)
1148
1149 def _corrupt_size_of_file_data(data, debug=False):
1150     """Scramble the file data -- the field showing the size of the share data
1151     within the file will be set to one smaller."""
1152     return corrupt_field(data, 0x04, 4)
1153
1154 def _corrupt_sharedata_version_number(data, debug=False):
1155     """Scramble the file data -- the share data version number will have one
1156     bit flipped or else will be changed to a random value, but not 1 or 2."""
1157     return corrupt_field(data, 0x0c, 4)
1158     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1159     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1160     newsharevernum = sharevernum
1161     while newsharevernum in (1, 2):
1162         newsharevernum = random.randrange(0, 2**32)
1163     newsharevernumbytes = struct.pack(">L", newsharevernum)
1164     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1165
1166 def _corrupt_sharedata_version_number_to_plausible_version(data, debug=False):
1167     """Scramble the file data -- the share data version number will be
1168     changed to 2 if it is 1 or else to 1 if it is 2."""
1169     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1170     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1171     if sharevernum == 1:
1172         newsharevernum = 2
1173     else:
1174         newsharevernum = 1
1175     newsharevernumbytes = struct.pack(">L", newsharevernum)
1176     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1177
1178 def _corrupt_segment_size(data, debug=False):
1179     """Scramble the file data -- the field showing the size of the segment
1180     will have one bit flipped or else be changed to a random value."""
1181     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1182     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1183     if sharevernum == 1:
1184         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1185     else:
1186         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1187
1188 def _corrupt_size_of_sharedata(data, debug=False):
1189     """Scramble the file data -- the field showing the size of the data
1190     within the share data will have one bit flipped or else will be changed
1191     to a random value."""
1192     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1193     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1194     if sharevernum == 1:
1195         return corrupt_field(data, 0x0c+0x08, 4)
1196     else:
1197         return corrupt_field(data, 0x0c+0x0c, 8)
1198
1199 def _corrupt_offset_of_sharedata(data, debug=False):
1200     """Scramble the file data -- the field showing the offset of the data
1201     within the share data will have one bit flipped or else be changed to a
1202     random value."""
1203     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1204     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1205     if sharevernum == 1:
1206         return corrupt_field(data, 0x0c+0x0c, 4)
1207     else:
1208         return corrupt_field(data, 0x0c+0x14, 8)
1209
1210 def _corrupt_offset_of_ciphertext_hash_tree(data, debug=False):
1211     """Scramble the file data -- the field showing the offset of the
1212     ciphertext hash tree within the share data will have one bit flipped or
1213     else be changed to a random value.
1214     """
1215     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1216     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1217     if sharevernum == 1:
1218         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1219     else:
1220         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1221
1222 def _corrupt_offset_of_block_hashes(data, debug=False):
1223     """Scramble the file data -- the field showing the offset of the block
1224     hash tree within the share data will have one bit flipped or else will be
1225     changed to a random value."""
1226     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1227     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1228     if sharevernum == 1:
1229         return corrupt_field(data, 0x0c+0x18, 4)
1230     else:
1231         return corrupt_field(data, 0x0c+0x2c, 8)
1232
1233 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data, debug=False):
1234     """Scramble the file data -- the field showing the offset of the block
1235     hash tree within the share data will have a multiple of hash size
1236     subtracted from it, thus causing the downloader to download an incomplete
1237     crypttext hash tree."""
1238     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1239     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1240     if sharevernum == 1:
1241         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1242         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1243         newvalstr = struct.pack(">L", newval)
1244         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1245     else:
1246         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1247         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1248         newvalstr = struct.pack(">Q", newval)
1249         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1250
1251 def _corrupt_offset_of_share_hashes(data, debug=False):
1252     """Scramble the file data -- the field showing the offset of the share
1253     hash tree within the share data will have one bit flipped or else will be
1254     changed to a random value."""
1255     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1256     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1257     if sharevernum == 1:
1258         return corrupt_field(data, 0x0c+0x1c, 4)
1259     else:
1260         return corrupt_field(data, 0x0c+0x34, 8)
1261
1262 def _corrupt_offset_of_uri_extension(data, debug=False):
1263     """Scramble the file data -- the field showing the offset of the uri
1264     extension will have one bit flipped or else will be changed to a random
1265     value."""
1266     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1267     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1268     if sharevernum == 1:
1269         return corrupt_field(data, 0x0c+0x20, 4)
1270     else:
1271         return corrupt_field(data, 0x0c+0x3c, 8)
1272
1273 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1274     """Scramble the file data -- the field showing the offset of the uri
1275     extension will be set to the size of the file minus 3. This means when
1276     the client tries to read the length field from that location it will get
1277     a short read -- the result string will be only 3 bytes long, not the 4 or
1278     8 bytes necessary to do a successful struct.unpack."""
1279     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1280     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1281     # The "-0x0c" in here is to skip the server-side header in the share
1282     # file, which the client doesn't see when seeking and reading.
1283     if sharevernum == 1:
1284         if debug:
1285             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1286         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1287     else:
1288         if debug:
1289             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1290         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1291
1292 def _corrupt_mutable_share_data(data, debug=False):
1293     prefix = data[:32]
1294     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
1295     data_offset = MutableShareFile.DATA_OFFSET
1296     sharetype = data[data_offset:data_offset+1]
1297     assert sharetype == "\x00", "non-SDMF mutable shares not supported"
1298     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
1299      ig_datalen, offsets) = unpack_header(data[data_offset:])
1300     assert version == 0, "this function only handles v0 SDMF files"
1301     start = data_offset + offsets["share_data"]
1302     length = data_offset + offsets["enc_privkey"] - start
1303     return corrupt_field(data, start, length)
1304
1305 def _corrupt_share_data(data, debug=False):
1306     """Scramble the file data -- the field containing the share data itself
1307     will have one bit flipped or else will be changed to a random value."""
1308     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1309     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1310     if sharevernum == 1:
1311         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1312
1313         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1314     else:
1315         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1316
1317         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1318
1319 def _corrupt_share_data_last_byte(data, debug=False):
1320     """Scramble the file data -- flip all bits of the last byte."""
1321     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1322     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1323     if sharevernum == 1:
1324         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1325         offset = 0x0c+0x24+sharedatasize-1
1326     else:
1327         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1328         offset = 0x0c+0x44+sharedatasize-1
1329
1330     newdata = data[:offset] + chr(ord(data[offset])^0xFF) + data[offset+1:]
1331     if debug:
1332         log.msg("testing: flipping all bits of byte at offset %d: %r, newdata: %r" % (offset, data[offset], newdata[offset]))
1333     return newdata
1334
1335 def _corrupt_crypttext_hash_tree(data, debug=False):
1336     """Scramble the file data -- the field containing the crypttext hash tree
1337     will have one bit flipped or else will be changed to a random value.
1338     """
1339     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1340     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1341     if sharevernum == 1:
1342         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1343         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1344     else:
1345         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1346         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1347
1348     return corrupt_field(data, 0x0c+crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset, debug=debug)
1349
1350 def _corrupt_crypttext_hash_tree_byte_x221(data, debug=False):
1351     """Scramble the file data -- the byte at offset 0x221 will have its 7th
1352     (b1) bit flipped.
1353     """
1354     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1355     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1356     if debug:
1357         log.msg("original data: %r" % (data,))
1358     return data[:0x0c+0x221] + chr(ord(data[0x0c+0x221])^0x02) + data[0x0c+0x2210+1:]
1359
1360 def _corrupt_block_hashes(data, debug=False):
1361     """Scramble the file data -- the field containing the block hash tree
1362     will have one bit flipped or else will be changed to a random value.
1363     """
1364     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1365     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1366     if sharevernum == 1:
1367         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1368         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1369     else:
1370         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1371         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1372
1373     return corrupt_field(data, 0x0c+blockhashesoffset, sharehashesoffset-blockhashesoffset)
1374
1375 def _corrupt_share_hashes(data, debug=False):
1376     """Scramble the file data -- the field containing the share hash chain
1377     will have one bit flipped or else will be changed to a random value.
1378     """
1379     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1380     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1381     if sharevernum == 1:
1382         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1383         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1384     else:
1385         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1386         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1387
1388     return corrupt_field(data, 0x0c+sharehashesoffset, uriextoffset-sharehashesoffset)
1389
1390 def _corrupt_length_of_uri_extension(data, debug=False):
1391     """Scramble the file data -- the field showing the length of the uri
1392     extension will have one bit flipped or else will be changed to a random
1393     value."""
1394     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1395     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1396     if sharevernum == 1:
1397         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1398         return corrupt_field(data, uriextoffset, 4)
1399     else:
1400         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1401         return corrupt_field(data, 0x0c+uriextoffset, 8)
1402
1403 def _corrupt_uri_extension(data, debug=False):
1404     """Scramble the file data -- the field containing the uri extension will
1405     have one bit flipped or else will be changed to a random value."""
1406     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1407     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1408     if sharevernum == 1:
1409         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1410         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1411     else:
1412         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1413         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1414
1415     return corrupt_field(data, 0x0c+uriextoffset, uriextlen)