src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IPullProducer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.api import flushEventualQueue, fireEventually
   9 from allmydata import uri, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IMutableFileNode, IImmutableFileNode,\
  12                                  NotEnoughSharesError, ICheckable, \
  13                                  IMutableUploadable, SDMF_VERSION, \
  14                                  MDMF_VERSION
  15 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  16      DeepCheckResults, DeepCheckAndRepairResults
  17 from allmydata.storage_client import StubServer
  18 from allmydata.mutable.layout import unpack_header
  19 from allmydata.mutable.publish import MutableData
  20 from allmydata.storage.mutable import MutableShareFile
  21 from allmydata.util import hashutil, log, fileutil, pollmixin
  22 from allmydata.util.assertutil import precondition
  23 from allmydata.util.consumer import download_to_data
  24 from allmydata.stats import StatsGathererService
  25 from allmydata.key_generator import KeyGeneratorService
  26 import allmydata.test.common_util as testutil
  27 from allmydata import immutable
  28
  29 TEST_RSA_KEY_SIZE = 522
  30
  31 def flush_but_dont_ignore(res):
  32     d = flushEventualQueue()
  33     def _done(ignored):
  34         return res
  35     d.addCallback(_done)
  36     return d
  37
  38 class DummyProducer:
  39     implements(IPullProducer)
  40     def resumeProducing(self):
  41         pass
  42
  43 class FakeCHKFileNode:
  44     """I provide IImmutableFileNode, but all of my data is stored in a
  45     class-level dictionary."""
  46     implements(IImmutableFileNode)
  47
  48     def __init__(self, filecap, all_contents):
  49         precondition(isinstance(filecap, (uri.CHKFileURI, uri.LiteralFileURI)), filecap)
  50         self.all_contents = all_contents
  51         self.my_uri = filecap
  52         self.storage_index = self.my_uri.get_storage_index()
  53
  54     def get_uri(self):
  55         return self.my_uri.to_string()
  56     def get_write_uri(self):
  57         return None
  58     def get_readonly_uri(self):
  59         return self.my_uri.to_string()
  60     def get_cap(self):
  61         return self.my_uri
  62     def get_verify_cap(self):
  63         return self.my_uri.get_verify_cap()
  64     def get_repair_cap(self):
  65         return self.my_uri.get_verify_cap()
  66     def get_storage_index(self):
  67         return self.storage_index
  68
  69     def check(self, monitor, verify=False, add_lease=False):
  70         s = StubServer("\x00"*20)
  71         r = CheckResults(self.my_uri, self.storage_index,
  72                          healthy=True, recoverable=True,
  73                          count_happiness=10,
  74                          count_shares_needed=3,
  75                          count_shares_expected=10,
  76                          count_shares_good=10,
  77                          count_good_share_hosts=10,
  78                          count_recoverable_versions=1,
  79                          count_unrecoverable_versions=0,
  80                          servers_responding=[s],
  81                          sharemap={1: [s]},
  82                          count_wrong_shares=0,
  83                          list_corrupt_shares=[],
  84                          count_corrupt_shares=0,
  85                          list_incompatible_shares=[],
  86                          count_incompatible_shares=0,
  87                          summary="",
  88                          report=[],
  89                          share_problems=[],
  90                          servermap=None)
  91         return defer.succeed(r)
  92     def check_and_repair(self, monitor, verify=False, add_lease=False):
  93         d = self.check(verify)
  94         def _got(cr):
  95             r = CheckAndRepairResults(self.storage_index)
  96             r.pre_repair_results = r.post_repair_results = cr
  97             return r
  98         d.addCallback(_got)
  99         return d
 100
 101     def is_mutable(self):
 102         return False
 103     def is_readonly(self):
 104         return True
 105     def is_unknown(self):
 106         return False
 107     def is_allowed_in_immutable_directory(self):
 108         return True
 109     def raise_error(self):
 110         pass
 111
 112     def get_size(self):
 113         if isinstance(self.my_uri, uri.LiteralFileURI):
 114             return self.my_uri.get_size()
 115         try:
 116             data = self.all_contents[self.my_uri.to_string()]
 117         except KeyError, le:
 118             raise NotEnoughSharesError(le, 0, 3)
 119         return len(data)
 120     def get_current_size(self):
 121         return defer.succeed(self.get_size())
 122
 123     def read(self, consumer, offset=0, size=None):
 124         # we don't bother to call registerProducer/unregisterProducer,
 125         # because it's a hassle to write a dummy Producer that does the right
 126         # thing (we have to make sure that DummyProducer.resumeProducing
 127         # writes the data into the consumer immediately, otherwise it will
 128         # loop forever).
 129
 130         d = defer.succeed(None)
 131         d.addCallback(self._read, consumer, offset, size)
 132         return d
 133
 134     def _read(self, ignored, consumer, offset, size):
 135         if isinstance(self.my_uri, uri.LiteralFileURI):
 136             data = self.my_uri.data
 137         else:
 138             if self.my_uri.to_string() not in self.all_contents:
 139                 raise NotEnoughSharesError(None, 0, 3)
 140             data = self.all_contents[self.my_uri.to_string()]
 141         start = offset
 142         if size is not None:
 143             end = offset + size
 144         else:
 145             end = len(data)
 146         consumer.write(data[start:end])
 147         return consumer
 148
 149
 150     def get_best_readable_version(self):
 151         return defer.succeed(self)
 152
 153
 154     def download_to_data(self, progress=None):
 155         return download_to_data(self, progress=progress)
 156
 157
 158     download_best_version = download_to_data
 159
 160
 161     def get_size_of_best_version(self):
 162         return defer.succeed(self.get_size)
 163
 164
 165 def make_chk_file_cap(size):
 166     return uri.CHKFileURI(key=os.urandom(16),
 167                           uri_extension_hash=os.urandom(32),
 168                           needed_shares=3,
 169                           total_shares=10,
 170                           size=size)
 171 def make_chk_file_uri(size):
 172     return make_chk_file_cap(size).to_string()
 173
 174 def create_chk_filenode(contents, all_contents):
 175     filecap = make_chk_file_cap(len(contents))
 176     n = FakeCHKFileNode(filecap, all_contents)
 177     all_contents[filecap.to_string()] = contents
 178     return n
 179
 180
 181 class FakeMutableFileNode:
 182     """I provide IMutableFileNode, but all of my data is stored in a
 183     class-level dictionary."""
 184
 185     implements(IMutableFileNode, ICheckable)
 186     MUTABLE_SIZELIMIT = 10000
 187
 188     def __init__(self, storage_broker, secret_holder,
 189                  default_encoding_parameters, history, all_contents):
 190         self.all_contents = all_contents
 191         self.file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION
 192         self.init_from_cap(make_mutable_file_cap())
 193         self._k = default_encoding_parameters['k']
 194         self._segsize = default_encoding_parameters['max_segment_size']
 195     def create(self, contents, key_generator=None, keysize=None,
 196                version=SDMF_VERSION):
 197         if version == MDMF_VERSION and \
 198             isinstance(self.my_uri, (uri.ReadonlySSKFileURI,
 199                                  uri.WriteableSSKFileURI)):
 200             self.init_from_cap(make_mdmf_mutable_file_cap())
 201         self.file_types[self.storage_index] = version
 202         initial_contents = self._get_initial_contents(contents)
 203         data = initial_contents.read(initial_contents.get_size())
 204         data = "".join(data)
 205         self.all_contents[self.storage_index] = data
 206         return defer.succeed(self)
 207     def _get_initial_contents(self, contents):
 208         if contents is None:
 209             return MutableData("")
 210
 211         if IMutableUploadable.providedBy(contents):
 212             return contents
 213
 214         assert callable(contents), "%s should be callable, not %s" % \
 215                (contents, type(contents))
 216         return contents(self)
 217     def init_from_cap(self, filecap):
 218         assert isinstance(filecap, (uri.WriteableSSKFileURI,
 219                                     uri.ReadonlySSKFileURI,
 220                                     uri.WriteableMDMFFileURI,
 221                                     uri.ReadonlyMDMFFileURI))
 222         self.my_uri = filecap
 223         self.storage_index = self.my_uri.get_storage_index()
 224         if isinstance(filecap, (uri.WriteableMDMFFileURI,
 225                                 uri.ReadonlyMDMFFileURI)):
 226             self.file_types[self.storage_index] = MDMF_VERSION
 227
 228         else:
 229             self.file_types[self.storage_index] = SDMF_VERSION
 230
 231         return self
 232     def get_cap(self):
 233         return self.my_uri
 234     def get_readcap(self):
 235         return self.my_uri.get_readonly()
 236     def get_uri(self):
 237         return self.my_uri.to_string()
 238     def get_write_uri(self):
 239         if self.is_readonly():
 240             return None
 241         return self.my_uri.to_string()
 242     def get_readonly(self):
 243         return self.my_uri.get_readonly()
 244     def get_readonly_uri(self):
 245         return self.my_uri.get_readonly().to_string()
 246     def get_verify_cap(self):
 247         return self.my_uri.get_verify_cap()
 248     def get_repair_cap(self):
 249         if self.my_uri.is_readonly():
 250             return None
 251         return self.my_uri
 252     def is_readonly(self):
 253         return self.my_uri.is_readonly()
 254     def is_mutable(self):
 255         return self.my_uri.is_mutable()
 256     def is_unknown(self):
 257         return False
 258     def is_allowed_in_immutable_directory(self):
 259         return not self.my_uri.is_mutable()
 260     def raise_error(self):
 261         pass
 262     def get_writekey(self):
 263         return "\x00"*16
 264     def get_size(self):
 265         return len(self.all_contents[self.storage_index])
 266     def get_current_size(self):
 267         return self.get_size_of_best_version()
 268     def get_size_of_best_version(self):
 269         return defer.succeed(len(self.all_contents[self.storage_index]))
 270
 271     def get_storage_index(self):
 272         return self.storage_index
 273
 274     def get_servermap(self, mode):
 275         return defer.succeed(None)
 276
 277     def get_version(self):
 278         assert self.storage_index in self.file_types
 279         return self.file_types[self.storage_index]
 280
 281     def check(self, monitor, verify=False, add_lease=False):
 282         s = StubServer("\x00"*20)
 283         r = CheckResults(self.my_uri, self.storage_index,
 284                          healthy=True, recoverable=True,
 285                          count_happiness=10,
 286                          count_shares_needed=3,
 287                          count_shares_expected=10,
 288                          count_shares_good=10,
 289                          count_good_share_hosts=10,
 290                          count_recoverable_versions=1,
 291                          count_unrecoverable_versions=0,
 292                          servers_responding=[s],
 293                          sharemap={"seq1-abcd-sh0": [s]},
 294                          count_wrong_shares=0,
 295                          list_corrupt_shares=[],
 296                          count_corrupt_shares=0,
 297                          list_incompatible_shares=[],
 298                          count_incompatible_shares=0,
 299                          summary="",
 300                          report=[],
 301                          share_problems=[],
 302                          servermap=None)
 303         return defer.succeed(r)
 304
 305     def check_and_repair(self, monitor, verify=False, add_lease=False):
 306         d = self.check(verify)
 307         def _got(cr):
 308             r = CheckAndRepairResults(self.storage_index)
 309             r.pre_repair_results = r.post_repair_results = cr
 310             return r
 311         d.addCallback(_got)
 312         return d
 313
 314     def deep_check(self, verify=False, add_lease=False):
 315         d = self.check(verify)
 316         def _done(r):
 317             dr = DeepCheckResults(self.storage_index)
 318             dr.add_check(r, [])
 319             return dr
 320         d.addCallback(_done)
 321         return d
 322
 323     def deep_check_and_repair(self, verify=False, add_lease=False):
 324         d = self.check_and_repair(verify)
 325         def _done(r):
 326             dr = DeepCheckAndRepairResults(self.storage_index)
 327             dr.add_check(r, [])
 328             return dr
 329         d.addCallback(_done)
 330         return d
 331
 332     def download_best_version(self, progress=None):
 333         return defer.succeed(self._download_best_version(progress=progress))
 334
 335
 336     def _download_best_version(self, ignored=None, progress=None):
 337         if isinstance(self.my_uri, uri.LiteralFileURI):
 338             return self.my_uri.data
 339         if self.storage_index not in self.all_contents:
 340             raise NotEnoughSharesError(None, 0, 3)
 341         return self.all_contents[self.storage_index]
 342
 343
 344     def overwrite(self, new_contents):
 345         assert not self.is_readonly()
 346         new_data = new_contents.read(new_contents.get_size())
 347         new_data = "".join(new_data)
 348         self.all_contents[self.storage_index] = new_data
 349         return defer.succeed(None)
 350     def modify(self, modifier):
 351         # this does not implement FileTooLargeError, but the real one does
 352         return defer.maybeDeferred(self._modify, modifier)
 353     def _modify(self, modifier):
 354         assert not self.is_readonly()
 355         old_contents = self.all_contents[self.storage_index]
 356         new_data = modifier(old_contents, None, True)
 357         self.all_contents[self.storage_index] = new_data
 358         return None
 359
 360     # As actually implemented, MutableFilenode and MutableFileVersion
 361     # are distinct. However, nothing in the webapi uses (yet) that
 362     # distinction -- it just uses the unified download interface
 363     # provided by get_best_readable_version and read. When we start
 364     # doing cooler things like LDMF, we will want to revise this code to
 365     # be less simplistic.
 366     def get_best_readable_version(self):
 367         return defer.succeed(self)
 368
 369
 370     def get_best_mutable_version(self):
 371         return defer.succeed(self)
 372
 373     # Ditto for this, which is an implementation of IWriteable.
 374     # XXX: Declare that the same is implemented.
 375     def update(self, data, offset):
 376         assert not self.is_readonly()
 377         def modifier(old, servermap, first_time):
 378             new = old[:offset] + "".join(data.read(data.get_size()))
 379             new += old[len(new):]
 380             return new
 381         return self.modify(modifier)
 382
 383
 384     def read(self, consumer, offset=0, size=None):
 385         data = self._download_best_version()
 386         if size:
 387             data = data[offset:offset+size]
 388         consumer.write(data)
 389         return defer.succeed(consumer)
 390
 391
 392 def make_mutable_file_cap():
 393     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 394                                    fingerprint=os.urandom(32))
 395
 396 def make_mdmf_mutable_file_cap():
 397     return uri.WriteableMDMFFileURI(writekey=os.urandom(16),
 398                                    fingerprint=os.urandom(32))
 399
 400 def make_mutable_file_uri(mdmf=False):
 401     if mdmf:
 402         uri = make_mdmf_mutable_file_cap()
 403     else:
 404         uri = make_mutable_file_cap()
 405
 406     return uri.to_string()
 407
 408 def make_verifier_uri():
 409     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 410                               fingerprint=os.urandom(32)).to_string()
 411
 412 def create_mutable_filenode(contents, mdmf=False, all_contents=None):
 413     # XXX: All of these arguments are kind of stupid.
 414     if mdmf:
 415         cap = make_mdmf_mutable_file_cap()
 416     else:
 417         cap = make_mutable_file_cap()
 418
 419     encoding_params = {}
 420     encoding_params['k'] = 3
 421     encoding_params['max_segment_size'] = 128*1024
 422
 423     filenode = FakeMutableFileNode(None, None, encoding_params, None,
 424                                    all_contents)
 425     filenode.init_from_cap(cap)
 426     if mdmf:
 427         filenode.create(MutableData(contents), version=MDMF_VERSION)
 428     else:
 429         filenode.create(MutableData(contents), version=SDMF_VERSION)
 430     return filenode
 431
 432
 433 class LoggingServiceParent(service.MultiService):
 434     def log(self, *args, **kwargs):
 435         return log.msg(*args, **kwargs)
 436
 437 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 438
 439     # SystemTestMixin tests tend to be a lot of work, and we have a few
 440     # buildslaves that are pretty slow, and every once in a while these tests
 441     # run up against the default 120 second timeout. So increase the default
 442     # timeout. Individual test cases can override this, of course.
 443     timeout = 300
 444
 445     def setUp(self):
 446         self.sparent = service.MultiService()
 447         self.sparent.startService()
 448
 449         self.stats_gatherer = None
 450         self.stats_gatherer_furl = None
 451         self.key_generator_svc = None
 452         self.key_generator_furl = None
 453
 454     def tearDown(self):
 455         log.msg("shutting down SystemTest services")
 456         d = self.sparent.stopService()
 457         d.addBoth(flush_but_dont_ignore)
 458         return d
 459
 460     def getdir(self, subdir):
 461         return os.path.join(self.basedir, subdir)
 462
 463     def add_service(self, s):
 464         s.setServiceParent(self.sparent)
 465         return s
 466
 467     def set_up_nodes(self, NUMCLIENTS=5,
 468                      use_stats_gatherer=False, use_key_generator=False):
 469         self.numclients = NUMCLIENTS
 470         iv_dir = self.getdir("introducer")
 471         if not os.path.isdir(iv_dir):
 472             fileutil.make_dirs(iv_dir)
 473             fileutil.write(os.path.join(iv_dir, 'tahoe.cfg'),
 474                            "[node]\n" +
 475                            u"nickname = introducer \u263A\n".encode('utf-8') +
 476                            "web.port = tcp:0:interface=127.0.0.1\n")
 477             if SYSTEM_TEST_CERTS:
 478                 os.mkdir(os.path.join(iv_dir, "private"))
 479                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 480                 f.write(SYSTEM_TEST_CERTS[0])
 481                 f.close()
 482         iv = IntroducerNode(basedir=iv_dir)
 483         self.introducer = self.add_service(iv)
 484         d = self.introducer.when_tub_ready()
 485         d.addCallback(self._get_introducer_web)
 486         if use_stats_gatherer:
 487             d.addCallback(self._set_up_stats_gatherer)
 488         if use_key_generator:
 489             d.addCallback(self._set_up_key_generator)
 490         d.addCallback(self._set_up_nodes_2)
 491         if use_stats_gatherer:
 492             d.addCallback(self._grab_stats)
 493         return d
 494
 495     def _get_introducer_web(self, res):
 496         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 497         self.introweb_url = f.read().strip()
 498         f.close()
 499
 500     def _set_up_stats_gatherer(self, res):
 501         statsdir = self.getdir("stats_gatherer")
 502         fileutil.make_dirs(statsdir)
 503         self.stats_gatherer_svc = StatsGathererService(statsdir)
 504         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 505         self.add_service(self.stats_gatherer_svc)
 506
 507         d = fireEventually()
 508         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 509         def check_for_furl():
 510             return os.path.exists(sgf)
 511         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 512         def get_furl(junk):
 513             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 514         d.addCallback(get_furl)
 515         return d
 516
 517     def _set_up_key_generator(self, res):
 518         kgsdir = self.getdir("key_generator")
 519         fileutil.make_dirs(kgsdir)
 520
 521         self.key_generator_svc = KeyGeneratorService(kgsdir,
 522                                                      display_furl=False,
 523                                                      default_key_size=TEST_RSA_KEY_SIZE)
 524         self.key_generator_svc.key_generator.pool_size = 4
 525         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 526         self.add_service(self.key_generator_svc)
 527
 528         d = fireEventually()
 529         def check_for_furl():
 530             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 531         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 532         def get_furl(junk):
 533             kgf = os.path.join(kgsdir, 'key_generator.furl')
 534             self.key_generator_furl = file(kgf, 'rb').read().strip()
 535         d.addCallback(get_furl)
 536         return d
 537
 538     def _set_up_nodes_2(self, res):
 539         q = self.introducer
 540         self.introducer_furl = q.introducer_url
 541         self.clients = []
 542         basedirs = []
 543         for i in range(self.numclients):
 544             basedir = self.getdir("client%d" % i)
 545             basedirs.append(basedir)
 546             fileutil.make_dirs(os.path.join(basedir, "private"))
 547             if len(SYSTEM_TEST_CERTS) > (i+1):
 548                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 549                 f.write(SYSTEM_TEST_CERTS[i+1])
 550                 f.close()
 551
 552             config = "[client]\n"
 553             config += "introducer.furl = %s\n" % self.introducer_furl
 554             if self.stats_gatherer_furl:
 555                 config += "stats_gatherer.furl = %s\n" % self.stats_gatherer_furl
 556
 557             nodeconfig = "[node]\n"
 558             nodeconfig += (u"nickname = client %d \u263A\n" % (i,)).encode('utf-8')
 559
 560             if i == 0:
 561                 # clients[0] runs a webserver and a helper, no key_generator
 562                 config += nodeconfig
 563                 config += "web.port = tcp:0:interface=127.0.0.1\n"
 564                 config += "timeout.keepalive = 600\n"
 565                 config += "[helper]\n"
 566                 config += "enabled = True\n"
 567             elif i == 3:
 568                 # clients[3] runs a webserver and uses a helper, uses
 569                 # key_generator
 570                 if self.key_generator_furl:
 571                     config += "key_generator.furl = %s\n" % self.key_generator_furl
 572                 config += nodeconfig
 573                 config += "web.port = tcp:0:interface=127.0.0.1\n"
 574                 config += "timeout.disconnect = 1800\n"
 575             else:
 576                 config += nodeconfig
 577
 578             fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
 579
 580         # give subclasses a chance to append lines to the node's tahoe.cfg
 581         # files before they are launched.
 582         self._set_up_nodes_extra_config()
 583
 584         # start clients[0], wait for it's tub to be ready (at which point it
 585         # will have registered the helper furl).
 586         c = self.add_service(client.Client(basedir=basedirs[0]))
 587         self.clients.append(c)
 588         c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 589         d = c.when_tub_ready()
 590         def _ready(res):
 591             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 592             helper_furl = f.read()
 593             f.close()
 594             self.helper_furl = helper_furl
 595             if self.numclients >= 4:
 596                 f = open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+')
 597                 f.write(
 598                       "[client]\n"
 599                       "helper.furl = %s\n" % helper_furl)
 600                 f.close()
 601
 602             # this starts the rest of the clients
 603             for i in range(1, self.numclients):
 604                 c = self.add_service(client.Client(basedir=basedirs[i]))
 605                 self.clients.append(c)
 606                 c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 607             log.msg("STARTING")
 608             return self.wait_for_connections()
 609         d.addCallback(_ready)
 610         def _connected(res):
 611             log.msg("CONNECTED")
 612             # now find out where the web port was
 613             self.webish_url = self.clients[0].getServiceNamed("webish").getURL()
 614             if self.numclients >=4:
 615                 # and the helper-using webport
 616                 self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL()
 617         d.addCallback(_connected)
 618         return d
 619
 620     def _set_up_nodes_extra_config(self):
 621         # for overriding by subclasses
 622         pass
 623
 624     def _grab_stats(self, res):
 625         d = self.stats_gatherer.poll()
 626         return d
 627
 628     def bounce_client(self, num):
 629         c = self.clients[num]
 630         d = c.disownServiceParent()
 631         # I think windows requires a moment to let the connection really stop
 632         # and the port number made available for re-use. TODO: examine the
 633         # behavior, see if this is really the problem, see if we can do
 634         # better than blindly waiting for a second.
 635         d.addCallback(self.stall, 1.0)
 636         def _stopped(res):
 637             new_c = client.Client(basedir=self.getdir("client%d" % num))
 638             self.clients[num] = new_c
 639             new_c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 640             self.add_service(new_c)
 641             return new_c.when_tub_ready()
 642         d.addCallback(_stopped)
 643         d.addCallback(lambda res: self.wait_for_connections())
 644         def _maybe_get_webport(res):
 645             if num == 0:
 646                 # now find out where the web port was
 647                 self.webish_url = self.clients[0].getServiceNamed("webish").getURL()
 648         d.addCallback(_maybe_get_webport)
 649         return d
 650
 651     def add_extra_node(self, client_num, helper_furl=None,
 652                        add_to_sparent=False):
 653         # usually this node is *not* parented to our self.sparent, so we can
 654         # shut it down separately from the rest, to exercise the
 655         # connection-lost code
 656         basedir = self.getdir("client%d" % client_num)
 657         if not os.path.isdir(basedir):
 658             fileutil.make_dirs(basedir)
 659         config = "[client]\n"
 660         config += "introducer.furl = %s\n" % self.introducer_furl
 661         if helper_furl:
 662             config += "helper.furl = %s\n" % helper_furl
 663         fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
 664
 665         c = client.Client(basedir=basedir)
 666         self.clients.append(c)
 667         c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
 668         self.numclients += 1
 669         if add_to_sparent:
 670             c.setServiceParent(self.sparent)
 671         else:
 672             c.startService()
 673         d = self.wait_for_connections()
 674         d.addCallback(lambda res: c)
 675         return d
 676
 677     def _check_connections(self):
 678         for c in self.clients:
 679             if not c.connected_to_introducer():
 680                 return False
 681             sb = c.get_storage_broker()
 682             if len(sb.get_connected_servers()) != self.numclients:
 683                 return False
 684             up = c.getServiceNamed("uploader")
 685             if up._helper_furl and not up._helper:
 686                 return False
 687         return True
 688
 689     def wait_for_connections(self, ignored=None):
 690         return self.poll(self._check_connections, timeout=200)
 691
 692
 693 # our system test uses the same Tub certificates each time, to avoid the
 694 # overhead of key generation
 695 SYSTEM_TEST_CERTS = [
 696 """-----BEGIN CERTIFICATE-----
 697 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 698 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 699 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 700 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 701 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 702 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 703 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 704 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 705 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 706 -----END CERTIFICATE-----
 707 -----BEGIN RSA PRIVATE KEY-----
 708 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 709 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 710 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 711 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 712 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 713 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 714 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 715 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 716 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 717 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 718 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 719 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 720 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 721 -----END RSA PRIVATE KEY-----
 722 """, # 0
 723 """-----BEGIN CERTIFICATE-----
 724 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 725 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 726 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 727 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 728 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 729 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 730 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 731 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 732 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 733 -----END CERTIFICATE-----
 734 -----BEGIN RSA PRIVATE KEY-----
 735 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 736 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 737 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 738 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 739 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 740 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 741 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 742 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 743 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 744 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 745 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 746 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 747 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 748 -----END RSA PRIVATE KEY-----
 749 """, # 1
 750 """-----BEGIN CERTIFICATE-----
 751 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 752 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 753 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 754 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 755 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 756 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 757 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 758 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 759 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 760 -----END CERTIFICATE-----
 761 -----BEGIN RSA PRIVATE KEY-----
 762 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 763 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 764 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 765 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 766 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 767 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 768 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 769 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 770 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 771 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 772 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 773 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 774 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 775 -----END RSA PRIVATE KEY-----
 776 """, # 2
 777 """-----BEGIN CERTIFICATE-----
 778 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 779 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 780 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 781 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 782 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 783 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 784 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 785 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 786 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 787 -----END CERTIFICATE-----
 788 -----BEGIN RSA PRIVATE KEY-----
 789 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 790 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 791 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 792 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 793 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 794 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 795 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 796 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 797 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 798 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 799 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 800 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 801 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 802 -----END RSA PRIVATE KEY-----
 803 """, # 3
 804 """-----BEGIN CERTIFICATE-----
 805 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 806 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 807 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 808 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 809 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 810 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 811 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 812 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 813 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 814 -----END CERTIFICATE-----
 815 -----BEGIN RSA PRIVATE KEY-----
 816 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 817 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 818 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 819 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 820 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 821 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 822 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 823 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 824 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 825 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 826 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 827 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 828 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 829 -----END RSA PRIVATE KEY-----
 830 """, # 4
 831 """-----BEGIN CERTIFICATE-----
 832 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 833 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 834 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 835 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 836 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 837 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 838 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 839 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 840 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 841 -----END CERTIFICATE-----
 842 -----BEGIN RSA PRIVATE KEY-----
 843 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 844 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 845 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 846 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 847 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 848 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 849 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 850 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 851 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 852 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 853 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 854 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 855 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 856 -----END RSA PRIVATE KEY-----
 857 """, # 5
 858 """-----BEGIN CERTIFICATE-----
 859 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 860 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 861 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 862 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 863 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 864 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 865 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 866 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 867 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 868 -----END CERTIFICATE-----
 869 -----BEGIN RSA PRIVATE KEY-----
 870 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 871 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 872 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 873 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 874 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 875 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 876 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 877 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 878 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 879 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 880 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 881 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 882 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 883 -----END RSA PRIVATE KEY-----
 884 """, #6
 885 """-----BEGIN CERTIFICATE-----
 886 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 887 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 888 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 889 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 890 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 891 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 892 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 893 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 894 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 895 -----END CERTIFICATE-----
 896 -----BEGIN RSA PRIVATE KEY-----
 897 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 898 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 899 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 900 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 901 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 902 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 903 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 904 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 905 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 906 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 907 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 908 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 909 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 910 -----END RSA PRIVATE KEY-----
 911 """, #7
 912 """-----BEGIN CERTIFICATE-----
 913 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 914 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 915 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 916 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 917 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 918 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 919 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 920 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 921 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 922 -----END CERTIFICATE-----
 923 -----BEGIN RSA PRIVATE KEY-----
 924 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 925 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 926 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 927 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 928 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 929 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 930 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 931 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 932 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 933 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 934 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 935 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 936 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 937 -----END RSA PRIVATE KEY-----
 938 """, #8
 939 """-----BEGIN CERTIFICATE-----
 940 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 941 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 942 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 943 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 944 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 945 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 946 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 947 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 948 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 949 -----END CERTIFICATE-----
 950 -----BEGIN RSA PRIVATE KEY-----
 951 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 952 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 953 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 954 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 955 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 956 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 957 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 958 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 959 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 960 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 961 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 962 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 963 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 964 -----END RSA PRIVATE KEY-----
 965 """, #9
 966 """-----BEGIN CERTIFICATE-----
 967 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 968 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 969 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 970 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 971 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 972 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 973 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 974 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 975 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 976 -----END CERTIFICATE-----
 977 -----BEGIN RSA PRIVATE KEY-----
 978 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 979 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 980 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 981 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 982 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 983 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 984 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
 985 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
 986 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
 987 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
 988 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
 989 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
 990 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
 991 -----END RSA PRIVATE KEY-----
 992 """, #10
 993 """-----BEGIN CERTIFICATE-----
 994 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 995 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 996 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
 997 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
 998 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
 999 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
1000 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
1001 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
1002 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
1003 -----END CERTIFICATE-----
1004 -----BEGIN RSA PRIVATE KEY-----
1005 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
1006 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
1007 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
1008 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
1009 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
1010 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
1011 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
1012 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
1013 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
1014 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
1015 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
1016 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
1017 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
1018 -----END RSA PRIVATE KEY-----
1019 """, #11
1020 ]
1021
1022 # To disable the pre-computed tub certs, uncomment this line.
1023 #SYSTEM_TEST_CERTS = []
1024
1025 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
1026
1027 class ShouldFailMixin:
1028     def shouldFail(self, expected_failure, which, substring,
1029                    callable, *args, **kwargs):
1030         """Assert that a function call raises some exception. This is a
1031         Deferred-friendly version of TestCase.assertRaises() .
1032
1033         Suppose you want to verify the following function:
1034
1035          def broken(a, b, c):
1036              if a < 0:
1037                  raise TypeError('a must not be negative')
1038              return defer.succeed(b+c)
1039
1040         You can use:
1041             d = self.shouldFail(TypeError, 'test name',
1042                                 'a must not be negative',
1043                                 broken, -4, 5, c=12)
1044         in your test method. The 'test name' string will be included in the
1045         error message, if any, because Deferred chains frequently make it
1046         difficult to tell which assertion was tripped.
1047
1048         The substring= argument, if not None, must appear in the 'repr'
1049         of the message wrapped by this Failure, or the test will fail.
1050         """
1051
1052         assert substring is None or isinstance(substring, str)
1053         d = defer.maybeDeferred(callable, *args, **kwargs)
1054         def done(res):
1055             if isinstance(res, failure.Failure):
1056                 res.trap(expected_failure)
1057                 if substring:
1058                     message = repr(res.value.args[0])
1059                     self.failUnless(substring in message,
1060                                     "%s: substring '%s' not in '%s'"
1061                                     % (which, substring, message))
1062             else:
1063                 self.fail("%s was supposed to raise %s, not get '%s'" %
1064                           (which, expected_failure, res))
1065         d.addBoth(done)
1066         return d
1067
1068 class WebErrorMixin:
1069     def explain_web_error(self, f):
1070         # an error on the server side causes the client-side getPage() to
1071         # return a failure(t.web.error.Error), and its str() doesn't show the
1072         # response body, which is where the useful information lives. Attach
1073         # this method as an errback handler, and it will reveal the hidden
1074         # message.
1075         f.trap(WebError)
1076         print "Web Error:", f.value, ":", f.value.response
1077         return f
1078
1079     def _shouldHTTPError(self, res, which, validator):
1080         if isinstance(res, failure.Failure):
1081             res.trap(WebError)
1082             return validator(res)
1083         else:
1084             self.fail("%s was supposed to Error, not get '%s'" % (which, res))
1085
1086     def shouldHTTPError(self, which,
1087                         code=None, substring=None, response_substring=None,
1088                         callable=None, *args, **kwargs):
1089         # returns a Deferred with the response body
1090         assert substring is None or isinstance(substring, str)
1091         assert callable
1092         def _validate(f):
1093             if code is not None:
1094                 self.failUnlessEqual(f.value.status, str(code), which)
1095             if substring:
1096                 code_string = str(f)
1097                 self.failUnless(substring in code_string,
1098                                 "%s: substring '%s' not in '%s'"
1099                                 % (which, substring, code_string))
1100             response_body = f.value.response
1101             if response_substring:
1102                 self.failUnless(response_substring in response_body,
1103                                 "%s: response substring '%s' not in '%s'"
1104                                 % (which, response_substring, response_body))
1105             return response_body
1106         d = defer.maybeDeferred(callable, *args, **kwargs)
1107         d.addBoth(self._shouldHTTPError, which, _validate)
1108         return d
1109
1110 class ErrorMixin(WebErrorMixin):
1111     def explain_error(self, f):
1112         if f.check(defer.FirstError):
1113             print "First Error:", f.value.subFailure
1114         return f
1115
1116 def corrupt_field(data, offset, size, debug=False):
1117     if random.random() < 0.5:
1118         newdata = testutil.flip_one_bit(data, offset, size)
1119         if debug:
1120             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1121         return newdata
1122     else:
1123         newval = testutil.insecurerandstr(size)
1124         if debug:
1125             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1126         return data[:offset]+newval+data[offset+size:]
1127
1128 def _corrupt_nothing(data, debug=False):
1129     """Leave the data pristine. """
1130     return data
1131
1132 def _corrupt_file_version_number(data, debug=False):
1133     """Scramble the file data -- the share file version number have one bit
1134     flipped or else will be changed to a random value."""
1135     return corrupt_field(data, 0x00, 4)
1136
1137 def _corrupt_size_of_file_data(data, debug=False):
1138     """Scramble the file data -- the field showing the size of the share data
1139     within the file will be set to one smaller."""
1140     return corrupt_field(data, 0x04, 4)
1141
1142 def _corrupt_sharedata_version_number(data, debug=False):
1143     """Scramble the file data -- the share data version number will have one
1144     bit flipped or else will be changed to a random value, but not 1 or 2."""
1145     return corrupt_field(data, 0x0c, 4)
1146     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1147     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1148     newsharevernum = sharevernum
1149     while newsharevernum in (1, 2):
1150         newsharevernum = random.randrange(0, 2**32)
1151     newsharevernumbytes = struct.pack(">L", newsharevernum)
1152     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1153
1154 def _corrupt_sharedata_version_number_to_plausible_version(data, debug=False):
1155     """Scramble the file data -- the share data version number will be
1156     changed to 2 if it is 1 or else to 1 if it is 2."""
1157     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1158     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1159     if sharevernum == 1:
1160         newsharevernum = 2
1161     else:
1162         newsharevernum = 1
1163     newsharevernumbytes = struct.pack(">L", newsharevernum)
1164     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1165
1166 def _corrupt_segment_size(data, debug=False):
1167     """Scramble the file data -- the field showing the size of the segment
1168     will have one bit flipped or else be changed to a random value."""
1169     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1170     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1171     if sharevernum == 1:
1172         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1173     else:
1174         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1175
1176 def _corrupt_size_of_sharedata(data, debug=False):
1177     """Scramble the file data -- the field showing the size of the data
1178     within the share data will have one bit flipped or else will be changed
1179     to a random value."""
1180     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1181     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1182     if sharevernum == 1:
1183         return corrupt_field(data, 0x0c+0x08, 4)
1184     else:
1185         return corrupt_field(data, 0x0c+0x0c, 8)
1186
1187 def _corrupt_offset_of_sharedata(data, debug=False):
1188     """Scramble the file data -- the field showing the offset of the data
1189     within the share data will have one bit flipped or else be changed to a
1190     random value."""
1191     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1192     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1193     if sharevernum == 1:
1194         return corrupt_field(data, 0x0c+0x0c, 4)
1195     else:
1196         return corrupt_field(data, 0x0c+0x14, 8)
1197
1198 def _corrupt_offset_of_ciphertext_hash_tree(data, debug=False):
1199     """Scramble the file data -- the field showing the offset of the
1200     ciphertext hash tree within the share data will have one bit flipped or
1201     else be changed to a random value.
1202     """
1203     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1204     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1205     if sharevernum == 1:
1206         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1207     else:
1208         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1209
1210 def _corrupt_offset_of_block_hashes(data, debug=False):
1211     """Scramble the file data -- the field showing the offset of the block
1212     hash tree within the share data will have one bit flipped or else will be
1213     changed to a random value."""
1214     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1215     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1216     if sharevernum == 1:
1217         return corrupt_field(data, 0x0c+0x18, 4)
1218     else:
1219         return corrupt_field(data, 0x0c+0x2c, 8)
1220
1221 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data, debug=False):
1222     """Scramble the file data -- the field showing the offset of the block
1223     hash tree within the share data will have a multiple of hash size
1224     subtracted from it, thus causing the downloader to download an incomplete
1225     crypttext hash tree."""
1226     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1227     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1228     if sharevernum == 1:
1229         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1230         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1231         newvalstr = struct.pack(">L", newval)
1232         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1233     else:
1234         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1235         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1236         newvalstr = struct.pack(">Q", newval)
1237         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1238
1239 def _corrupt_offset_of_share_hashes(data, debug=False):
1240     """Scramble the file data -- the field showing the offset of the share
1241     hash tree within the share data will have one bit flipped or else will be
1242     changed to a random value."""
1243     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1244     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1245     if sharevernum == 1:
1246         return corrupt_field(data, 0x0c+0x1c, 4)
1247     else:
1248         return corrupt_field(data, 0x0c+0x34, 8)
1249
1250 def _corrupt_offset_of_uri_extension(data, debug=False):
1251     """Scramble the file data -- the field showing the offset of the uri
1252     extension will have one bit flipped or else will be changed to a random
1253     value."""
1254     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1255     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1256     if sharevernum == 1:
1257         return corrupt_field(data, 0x0c+0x20, 4)
1258     else:
1259         return corrupt_field(data, 0x0c+0x3c, 8)
1260
1261 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1262     """Scramble the file data -- the field showing the offset of the uri
1263     extension will be set to the size of the file minus 3. This means when
1264     the client tries to read the length field from that location it will get
1265     a short read -- the result string will be only 3 bytes long, not the 4 or
1266     8 bytes necessary to do a successful struct.unpack."""
1267     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1268     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1269     # The "-0x0c" in here is to skip the server-side header in the share
1270     # file, which the client doesn't see when seeking and reading.
1271     if sharevernum == 1:
1272         if debug:
1273             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1274         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1275     else:
1276         if debug:
1277             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1278         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1279
1280 def _corrupt_mutable_share_data(data, debug=False):
1281     prefix = data[:32]
1282     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
1283     data_offset = MutableShareFile.DATA_OFFSET
1284     sharetype = data[data_offset:data_offset+1]
1285     assert sharetype == "\x00", "non-SDMF mutable shares not supported"
1286     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
1287      ig_datalen, offsets) = unpack_header(data[data_offset:])
1288     assert version == 0, "this function only handles v0 SDMF files"
1289     start = data_offset + offsets["share_data"]
1290     length = data_offset + offsets["enc_privkey"] - start
1291     return corrupt_field(data, start, length)
1292
1293 def _corrupt_share_data(data, debug=False):
1294     """Scramble the file data -- the field containing the share data itself
1295     will have one bit flipped or else will be changed to a random value."""
1296     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1297     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1298     if sharevernum == 1:
1299         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1300
1301         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1302     else:
1303         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1304
1305         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1306
1307 def _corrupt_share_data_last_byte(data, debug=False):
1308     """Scramble the file data -- flip all bits of the last byte."""
1309     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1310     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1311     if sharevernum == 1:
1312         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1313         offset = 0x0c+0x24+sharedatasize-1
1314     else:
1315         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1316         offset = 0x0c+0x44+sharedatasize-1
1317
1318     newdata = data[:offset] + chr(ord(data[offset])^0xFF) + data[offset+1:]
1319     if debug:
1320         log.msg("testing: flipping all bits of byte at offset %d: %r, newdata: %r" % (offset, data[offset], newdata[offset]))
1321     return newdata
1322
1323 def _corrupt_crypttext_hash_tree(data, debug=False):
1324     """Scramble the file data -- the field containing the crypttext hash tree
1325     will have one bit flipped or else will be changed to a random value.
1326     """
1327     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1328     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1329     if sharevernum == 1:
1330         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1331         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1332     else:
1333         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1334         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1335
1336     return corrupt_field(data, 0x0c+crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset, debug=debug)
1337
1338 def _corrupt_crypttext_hash_tree_byte_x221(data, debug=False):
1339     """Scramble the file data -- the byte at offset 0x221 will have its 7th
1340     (b1) bit flipped.
1341     """
1342     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1343     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1344     if debug:
1345         log.msg("original data: %r" % (data,))
1346     return data[:0x0c+0x221] + chr(ord(data[0x0c+0x221])^0x02) + data[0x0c+0x2210+1:]
1347
1348 def _corrupt_block_hashes(data, debug=False):
1349     """Scramble the file data -- the field containing the block hash tree
1350     will have one bit flipped or else will be changed to a random value.
1351     """
1352     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1353     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1354     if sharevernum == 1:
1355         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1356         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1357     else:
1358         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1359         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1360
1361     return corrupt_field(data, 0x0c+blockhashesoffset, sharehashesoffset-blockhashesoffset)
1362
1363 def _corrupt_share_hashes(data, debug=False):
1364     """Scramble the file data -- the field containing the share hash chain
1365     will have one bit flipped or else will be changed to a random value.
1366     """
1367     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1368     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1369     if sharevernum == 1:
1370         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1371         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1372     else:
1373         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1374         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1375
1376     return corrupt_field(data, 0x0c+sharehashesoffset, uriextoffset-sharehashesoffset)
1377
1378 def _corrupt_length_of_uri_extension(data, debug=False):
1379     """Scramble the file data -- the field showing the length of the uri
1380     extension will have one bit flipped or else will be changed to a random
1381     value."""
1382     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1383     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1384     if sharevernum == 1:
1385         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1386         return corrupt_field(data, uriextoffset, 4)
1387     else:
1388         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1389         return corrupt_field(data, 0x0c+uriextoffset, 8)
1390
1391 def _corrupt_uri_extension(data, debug=False):
1392     """Scramble the file data -- the field containing the uri extension will
1393     have one bit flipped or else will be changed to a random value."""
1394     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1395     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1396     if sharevernum == 1:
1397         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1398         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1399     else:
1400         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1401         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1402
1403     return corrupt_field(data, 0x0c+uriextoffset, uriextlen)