src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IPullProducer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.api import flushEventualQueue, fireEventually
   9 from allmydata import uri, dirnode, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IMutableFileNode, IImmutableFileNode, \
  12      FileTooLargeError, NotEnoughSharesError, ICheckable
  13 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  14      DeepCheckResults, DeepCheckAndRepairResults
  15 from allmydata.mutable.common import CorruptShareError
  16 from allmydata.mutable.layout import unpack_header
  17 from allmydata.storage.server import storage_index_to_dir
  18 from allmydata.storage.mutable import MutableShareFile
  19 from allmydata.util import hashutil, log, fileutil, pollmixin
  20 from allmydata.util.assertutil import precondition
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.stats import StatsGathererService
  23 from allmydata.key_generator import KeyGeneratorService
  24 import common_util as testutil
  25 from allmydata import immutable
  26
  27
  28 def flush_but_dont_ignore(res):
  29     d = flushEventualQueue()
  30     def _done(ignored):
  31         return res
  32     d.addCallback(_done)
  33     return d
  34
  35 class DummyProducer:
  36     implements(IPullProducer)
  37     def resumeProducing(self):
  38         pass
  39
  40 class FakeCHKFileNode:
  41     """I provide IImmutableFileNode, but all of my data is stored in a
  42     class-level dictionary."""
  43     implements(IImmutableFileNode)
  44     all_contents = {}
  45     bad_shares = {}
  46
  47     def __init__(self, filecap):
  48         precondition(isinstance(filecap, uri.CHKFileURI), filecap)
  49         self.my_uri = filecap
  50         self.storage_index = self.my_uri.storage_index
  51
  52     def get_uri(self):
  53         return self.my_uri.to_string()
  54     def get_readonly_uri(self):
  55         return self.my_uri.to_string()
  56     def get_cap(self):
  57         return self.my_uri
  58     def get_verify_cap(self):
  59         return self.my_uri.get_verify_cap()
  60     def get_repair_cap(self):
  61         return self.my_uri.get_verify_cap()
  62     def get_storage_index(self):
  63         return self.storage_index
  64
  65     def check(self, monitor, verify=False, add_lease=False):
  66         r = CheckResults(self.my_uri, self.storage_index)
  67         is_bad = self.bad_shares.get(self.storage_index, None)
  68         data = {}
  69         data["count-shares-needed"] = 3
  70         data["count-shares-expected"] = 10
  71         data["count-good-share-hosts"] = 10
  72         data["count-wrong-shares"] = 0
  73         nodeid = "\x00"*20
  74         data["list-corrupt-shares"] = []
  75         data["sharemap"] = {1: [nodeid]}
  76         data["servers-responding"] = [nodeid]
  77         data["count-recoverable-versions"] = 1
  78         data["count-unrecoverable-versions"] = 0
  79         if is_bad:
  80              r.set_healthy(False)
  81              r.set_recoverable(True)
  82              data["count-shares-good"] = 9
  83              data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
  84              r.problems = failure.Failure(CorruptShareError(is_bad))
  85         else:
  86              r.set_healthy(True)
  87              r.set_recoverable(True)
  88              data["count-shares-good"] = 10
  89              r.problems = []
  90         r.set_data(data)
  91         r.set_needs_rebalancing(False)
  92         return defer.succeed(r)
  93     def check_and_repair(self, monitor, verify=False, add_lease=False):
  94         d = self.check(verify)
  95         def _got(cr):
  96             r = CheckAndRepairResults(self.storage_index)
  97             r.pre_repair_results = r.post_repair_results = cr
  98             return r
  99         d.addCallback(_got)
 100         return d
 101
 102     def is_mutable(self):
 103         return False
 104     def is_readonly(self):
 105         return True
 106
 107     def get_size(self):
 108         try:
 109             data = self.all_contents[self.my_uri.to_string()]
 110         except KeyError, le:
 111             raise NotEnoughSharesError(le, 0, 3)
 112         return len(data)
 113
 114     def read(self, consumer, offset=0, size=None):
 115         # we don't bother to call registerProducer/unregisterProducer,
 116         # because it's a hassle to write a dummy Producer that does the right
 117         # thing (we have to make sure that DummyProducer.resumeProducing
 118         # writes the data into the consumer immediately, otherwise it will
 119         # loop forever).
 120
 121         d = defer.succeed(None)
 122         d.addCallback(self._read, consumer, offset, size)
 123         return d
 124
 125     def _read(self, ignored, consumer, offset, size):
 126         if self.my_uri.to_string() not in self.all_contents:
 127             raise NotEnoughSharesError(None, 0, 3)
 128         data = self.all_contents[self.my_uri.to_string()]
 129         start = offset
 130         if size is not None:
 131             end = offset + size
 132         else:
 133             end = len(data)
 134         consumer.write(data[start:end])
 135         return consumer
 136
 137 def make_chk_file_cap(size):
 138     return uri.CHKFileURI(key=os.urandom(16),
 139                           uri_extension_hash=os.urandom(32),
 140                           needed_shares=3,
 141                           total_shares=10,
 142                           size=size)
 143 def make_chk_file_uri(size):
 144     return make_chk_file_cap(size).to_string()
 145
 146 def create_chk_filenode(contents):
 147     filecap = make_chk_file_cap(len(contents))
 148     n = FakeCHKFileNode(filecap)
 149     FakeCHKFileNode.all_contents[filecap.to_string()] = contents
 150     return n
 151
 152
 153 class FakeMutableFileNode:
 154     """I provide IMutableFileNode, but all of my data is stored in a
 155     class-level dictionary."""
 156
 157     implements(IMutableFileNode, ICheckable)
 158     MUTABLE_SIZELIMIT = 10000
 159     all_contents = {}
 160     bad_shares = {}
 161
 162     def __init__(self, storage_broker, secret_holder,
 163                  default_encoding_parameters, history):
 164         self.init_from_cap(make_mutable_file_cap())
 165     def create(self, contents, key_generator=None, keysize=None):
 166         initial_contents = self._get_initial_contents(contents)
 167         if len(initial_contents) > self.MUTABLE_SIZELIMIT:
 168             raise FileTooLargeError("SDMF is limited to one segment, and "
 169                                     "%d > %d" % (len(initial_contents),
 170                                                  self.MUTABLE_SIZELIMIT))
 171         self.all_contents[self.storage_index] = initial_contents
 172         return defer.succeed(self)
 173     def _get_initial_contents(self, contents):
 174         if isinstance(contents, str):
 175             return contents
 176         if contents is None:
 177             return ""
 178         assert callable(contents), "%s should be callable, not %s" % \
 179                (contents, type(contents))
 180         return contents(self)
 181     def init_from_cap(self, filecap):
 182         assert isinstance(filecap, (uri.WriteableSSKFileURI,
 183                                     uri.ReadonlySSKFileURI))
 184         self.my_uri = filecap
 185         self.storage_index = self.my_uri.storage_index
 186         return self
 187     def get_cap(self):
 188         return self.my_uri
 189     def get_readcap(self):
 190         return self.my_uri.get_readonly()
 191     def get_uri(self):
 192         return self.my_uri.to_string()
 193     def get_readonly(self):
 194         return self.my_uri.get_readonly()
 195     def get_readonly_uri(self):
 196         return self.my_uri.get_readonly().to_string()
 197     def get_verify_cap(self):
 198         return self.my_uri.get_verify_cap()
 199     def is_readonly(self):
 200         return self.my_uri.is_readonly()
 201     def is_mutable(self):
 202         return self.my_uri.is_mutable()
 203     def get_writekey(self):
 204         return "\x00"*16
 205     def get_size(self):
 206         return len(self.all_contents[self.storage_index])
 207     def get_current_size(self):
 208         return self.get_size_of_best_version()
 209     def get_size_of_best_version(self):
 210         return defer.succeed(len(self.all_contents[self.storage_index]))
 211
 212     def get_storage_index(self):
 213         return self.storage_index
 214
 215     def check(self, monitor, verify=False, add_lease=False):
 216         r = CheckResults(self.my_uri, self.storage_index)
 217         is_bad = self.bad_shares.get(self.storage_index, None)
 218         data = {}
 219         data["count-shares-needed"] = 3
 220         data["count-shares-expected"] = 10
 221         data["count-good-share-hosts"] = 10
 222         data["count-wrong-shares"] = 0
 223         data["list-corrupt-shares"] = []
 224         nodeid = "\x00"*20
 225         data["sharemap"] = {"seq1-abcd-sh0": [nodeid]}
 226         data["servers-responding"] = [nodeid]
 227         data["count-recoverable-versions"] = 1
 228         data["count-unrecoverable-versions"] = 0
 229         if is_bad:
 230              r.set_healthy(False)
 231              r.set_recoverable(True)
 232              data["count-shares-good"] = 9
 233              r.problems = failure.Failure(CorruptShareError("peerid",
 234                                                             0, # shnum
 235                                                             is_bad))
 236         else:
 237              r.set_healthy(True)
 238              r.set_recoverable(True)
 239              data["count-shares-good"] = 10
 240              r.problems = []
 241         r.set_data(data)
 242         r.set_needs_rebalancing(False)
 243         return defer.succeed(r)
 244
 245     def check_and_repair(self, monitor, verify=False, add_lease=False):
 246         d = self.check(verify)
 247         def _got(cr):
 248             r = CheckAndRepairResults(self.storage_index)
 249             r.pre_repair_results = r.post_repair_results = cr
 250             return r
 251         d.addCallback(_got)
 252         return d
 253
 254     def deep_check(self, verify=False, add_lease=False):
 255         d = self.check(verify)
 256         def _done(r):
 257             dr = DeepCheckResults(self.storage_index)
 258             dr.add_check(r, [])
 259             return dr
 260         d.addCallback(_done)
 261         return d
 262
 263     def deep_check_and_repair(self, verify=False, add_lease=False):
 264         d = self.check_and_repair(verify)
 265         def _done(r):
 266             dr = DeepCheckAndRepairResults(self.storage_index)
 267             dr.add_check(r, [])
 268             return dr
 269         d.addCallback(_done)
 270         return d
 271
 272     def download_best_version(self):
 273         return defer.succeed(self.all_contents[self.storage_index])
 274     def overwrite(self, new_contents):
 275         if len(new_contents) > self.MUTABLE_SIZELIMIT:
 276             raise FileTooLargeError("SDMF is limited to one segment, and "
 277                                     "%d > %d" % (len(new_contents),
 278                                                  self.MUTABLE_SIZELIMIT))
 279         assert not self.is_readonly()
 280         self.all_contents[self.storage_index] = new_contents
 281         return defer.succeed(None)
 282     def modify(self, modifier):
 283         # this does not implement FileTooLargeError, but the real one does
 284         return defer.maybeDeferred(self._modify, modifier)
 285     def _modify(self, modifier):
 286         assert not self.is_readonly()
 287         old_contents = self.all_contents[self.storage_index]
 288         self.all_contents[self.storage_index] = modifier(old_contents, None, True)
 289         return None
 290
 291     def download(self, target):
 292         if self.storage_index not in self.all_contents:
 293             f = failure.Failure(NotEnoughSharesError(None, 0, 3))
 294             target.fail(f)
 295             return defer.fail(f)
 296         data = self.all_contents[self.storage_index]
 297         target.open(len(data))
 298         target.write(data)
 299         target.close()
 300         return defer.maybeDeferred(target.finish)
 301     def download_to_data(self):
 302         if self.storage_index not in self.all_contents:
 303             return defer.fail(NotEnoughSharesError(None, 0, 3))
 304         data = self.all_contents[self.storage_index]
 305         return defer.succeed(data)
 306
 307 def make_mutable_file_cap():
 308     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 309                                    fingerprint=os.urandom(32))
 310 def make_mutable_file_uri():
 311     return make_mutable_file_cap().to_string()
 312
 313 def make_verifier_uri():
 314     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 315                               fingerprint=os.urandom(32)).to_string()
 316
 317 class FakeDirectoryNode(dirnode.DirectoryNode):
 318     """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
 319     backing store, so it doesn't go to the grid. The child data is still
 320     encrypted and serialized, so this isn't useful for tests that want to
 321     look inside the dirnodes and check their contents.
 322     """
 323     filenode_class = FakeMutableFileNode
 324
 325 class LoggingServiceParent(service.MultiService):
 326     def log(self, *args, **kwargs):
 327         return log.msg(*args, **kwargs)
 328
 329
 330 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 331
 332     # SystemTestMixin tests tend to be a lot of work, and we have a few
 333     # buildslaves that are pretty slow, and every once in a while these tests
 334     # run up against the default 120 second timeout. So increase the default
 335     # timeout. Individual test cases can override this, of course.
 336     timeout = 300
 337
 338     def setUp(self):
 339         self.sparent = service.MultiService()
 340         self.sparent.startService()
 341
 342         self.stats_gatherer = None
 343         self.stats_gatherer_furl = None
 344         self.key_generator_svc = None
 345         self.key_generator_furl = None
 346
 347     def tearDown(self):
 348         log.msg("shutting down SystemTest services")
 349         d = self.sparent.stopService()
 350         d.addBoth(flush_but_dont_ignore)
 351         return d
 352
 353     def getdir(self, subdir):
 354         return os.path.join(self.basedir, subdir)
 355
 356     def add_service(self, s):
 357         s.setServiceParent(self.sparent)
 358         return s
 359
 360     def set_up_nodes(self, NUMCLIENTS=5,
 361                      use_stats_gatherer=False, use_key_generator=False):
 362         self.numclients = NUMCLIENTS
 363         iv_dir = self.getdir("introducer")
 364         if not os.path.isdir(iv_dir):
 365             fileutil.make_dirs(iv_dir)
 366             f = open(os.path.join(iv_dir, "webport"), "w")
 367             f.write("tcp:0:interface=127.0.0.1\n")
 368             f.close()
 369             if SYSTEM_TEST_CERTS:
 370                 os.mkdir(os.path.join(iv_dir, "private"))
 371                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 372                 f.write(SYSTEM_TEST_CERTS[0])
 373                 f.close()
 374         iv = IntroducerNode(basedir=iv_dir)
 375         self.introducer = self.add_service(iv)
 376         d = self.introducer.when_tub_ready()
 377         d.addCallback(self._get_introducer_web)
 378         if use_stats_gatherer:
 379             d.addCallback(self._set_up_stats_gatherer)
 380         if use_key_generator:
 381             d.addCallback(self._set_up_key_generator)
 382         d.addCallback(self._set_up_nodes_2)
 383         if use_stats_gatherer:
 384             d.addCallback(self._grab_stats)
 385         return d
 386
 387     def _get_introducer_web(self, res):
 388         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 389         self.introweb_url = f.read().strip()
 390         f.close()
 391
 392     def _set_up_stats_gatherer(self, res):
 393         statsdir = self.getdir("stats_gatherer")
 394         fileutil.make_dirs(statsdir)
 395         self.stats_gatherer_svc = StatsGathererService(statsdir)
 396         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 397         self.add_service(self.stats_gatherer_svc)
 398
 399         d = fireEventually()
 400         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 401         def check_for_furl():
 402             return os.path.exists(sgf)
 403         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 404         def get_furl(junk):
 405             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 406         d.addCallback(get_furl)
 407         return d
 408
 409     def _set_up_key_generator(self, res):
 410         kgsdir = self.getdir("key_generator")
 411         fileutil.make_dirs(kgsdir)
 412
 413         self.key_generator_svc = KeyGeneratorService(kgsdir,
 414                                                      display_furl=False,
 415                                                      default_key_size=522)
 416         self.key_generator_svc.key_generator.pool_size = 4
 417         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 418         self.add_service(self.key_generator_svc)
 419
 420         d = fireEventually()
 421         def check_for_furl():
 422             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 423         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 424         def get_furl(junk):
 425             kgf = os.path.join(kgsdir, 'key_generator.furl')
 426             self.key_generator_furl = file(kgf, 'rb').read().strip()
 427         d.addCallback(get_furl)
 428         return d
 429
 430     def _set_up_nodes_2(self, res):
 431         q = self.introducer
 432         self.introducer_furl = q.introducer_url
 433         self.clients = []
 434         basedirs = []
 435         for i in range(self.numclients):
 436             basedir = self.getdir("client%d" % i)
 437             basedirs.append(basedir)
 438             fileutil.make_dirs(os.path.join(basedir, "private"))
 439             if len(SYSTEM_TEST_CERTS) > (i+1):
 440                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 441                 f.write(SYSTEM_TEST_CERTS[i+1])
 442                 f.close()
 443
 444             def write(name, value):
 445                 open(os.path.join(basedir, name), "w").write(value+"\n")
 446             if i == 0:
 447                 # clients[0] runs a webserver and a helper, no key_generator
 448                 write("webport", "tcp:0:interface=127.0.0.1")
 449                 write("run_helper", "yes")
 450                 write("keepalive_timeout", "600")
 451             if i == 3:
 452                 # clients[3] runs a webserver and uses a helper, uses
 453                 # key_generator
 454                 write("webport", "tcp:0:interface=127.0.0.1")
 455                 write("disconnect_timeout", "1800")
 456                 if self.key_generator_furl:
 457                     kgf = "%s\n" % (self.key_generator_furl,)
 458                     write("key_generator.furl", kgf)
 459             write("introducer.furl", self.introducer_furl)
 460             if self.stats_gatherer_furl:
 461                 write("stats_gatherer.furl", self.stats_gatherer_furl)
 462
 463         # give subclasses a chance to append liens to the node's tahoe.cfg
 464         # files before they are launched.
 465         self._set_up_nodes_extra_config()
 466
 467         # start clients[0], wait for it's tub to be ready (at which point it
 468         # will have registered the helper furl).
 469         c = self.add_service(client.Client(basedir=basedirs[0]))
 470         self.clients.append(c)
 471         c.set_default_mutable_keysize(522)
 472         d = c.when_tub_ready()
 473         def _ready(res):
 474             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 475             helper_furl = f.read()
 476             f.close()
 477             self.helper_furl = helper_furl
 478             if self.numclients >= 4:
 479                 f = open(os.path.join(basedirs[3],"helper.furl"), "w")
 480                 f.write(helper_furl)
 481                 f.close()
 482
 483             # this starts the rest of the clients
 484             for i in range(1, self.numclients):
 485                 c = self.add_service(client.Client(basedir=basedirs[i]))
 486                 self.clients.append(c)
 487                 c.set_default_mutable_keysize(522)
 488             log.msg("STARTING")
 489             return self.wait_for_connections()
 490         d.addCallback(_ready)
 491         def _connected(res):
 492             log.msg("CONNECTED")
 493             # now find out where the web port was
 494             l = self.clients[0].getServiceNamed("webish").listener
 495             port = l._port.getHost().port
 496             self.webish_url = "http://localhost:%d/" % port
 497             if self.numclients >=4:
 498                 # and the helper-using webport
 499                 l = self.clients[3].getServiceNamed("webish").listener
 500                 port = l._port.getHost().port
 501                 self.helper_webish_url = "http://localhost:%d/" % port
 502         d.addCallback(_connected)
 503         return d
 504
 505     def _set_up_nodes_extra_config(self):
 506         # for overriding by subclasses
 507         pass
 508
 509     def _grab_stats(self, res):
 510         d = self.stats_gatherer.poll()
 511         return d
 512
 513     def bounce_client(self, num):
 514         c = self.clients[num]
 515         d = c.disownServiceParent()
 516         # I think windows requires a moment to let the connection really stop
 517         # and the port number made available for re-use. TODO: examine the
 518         # behavior, see if this is really the problem, see if we can do
 519         # better than blindly waiting for a second.
 520         d.addCallback(self.stall, 1.0)
 521         def _stopped(res):
 522             new_c = client.Client(basedir=self.getdir("client%d" % num))
 523             self.clients[num] = new_c
 524             new_c.set_default_mutable_keysize(522)
 525             self.add_service(new_c)
 526             return new_c.when_tub_ready()
 527         d.addCallback(_stopped)
 528         d.addCallback(lambda res: self.wait_for_connections())
 529         def _maybe_get_webport(res):
 530             if num == 0:
 531                 # now find out where the web port was
 532                 l = self.clients[0].getServiceNamed("webish").listener
 533                 port = l._port.getHost().port
 534                 self.webish_url = "http://localhost:%d/" % port
 535         d.addCallback(_maybe_get_webport)
 536         return d
 537
 538     def add_extra_node(self, client_num, helper_furl=None,
 539                        add_to_sparent=False):
 540         # usually this node is *not* parented to our self.sparent, so we can
 541         # shut it down separately from the rest, to exercise the
 542         # connection-lost code
 543         basedir = self.getdir("client%d" % client_num)
 544         if not os.path.isdir(basedir):
 545             fileutil.make_dirs(basedir)
 546         open(os.path.join(basedir, "introducer.furl"), "w").write(self.introducer_furl)
 547         if helper_furl:
 548             f = open(os.path.join(basedir, "helper.furl") ,"w")
 549             f.write(helper_furl+"\n")
 550             f.close()
 551
 552         c = client.Client(basedir=basedir)
 553         self.clients.append(c)
 554         c.set_default_mutable_keysize(522)
 555         self.numclients += 1
 556         if add_to_sparent:
 557             c.setServiceParent(self.sparent)
 558         else:
 559             c.startService()
 560         d = self.wait_for_connections()
 561         d.addCallback(lambda res: c)
 562         return d
 563
 564     def _check_connections(self):
 565         for c in self.clients:
 566             if not c.connected_to_introducer():
 567                 return False
 568             sb = c.get_storage_broker()
 569             if len(sb.get_all_servers()) != self.numclients:
 570                 return False
 571         return True
 572
 573     def wait_for_connections(self, ignored=None):
 574         # TODO: replace this with something that takes a list of peerids and
 575         # fires when they've all been heard from, instead of using a count
 576         # and a threshold
 577         return self.poll(self._check_connections, timeout=200)
 578
 579
 580 # our system test uses the same Tub certificates each time, to avoid the
 581 # overhead of key generation
 582 SYSTEM_TEST_CERTS = [
 583 """-----BEGIN CERTIFICATE-----
 584 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 585 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 586 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 587 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 588 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 589 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 590 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 591 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 592 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 593 -----END CERTIFICATE-----
 594 -----BEGIN RSA PRIVATE KEY-----
 595 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 596 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 597 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 598 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 599 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 600 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 601 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 602 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 603 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 604 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 605 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 606 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 607 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 608 -----END RSA PRIVATE KEY-----
 609 """, # 0
 610 """-----BEGIN CERTIFICATE-----
 611 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 612 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 613 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 614 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 615 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 616 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 617 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 618 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 619 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 620 -----END CERTIFICATE-----
 621 -----BEGIN RSA PRIVATE KEY-----
 622 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 623 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 624 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 625 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 626 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 627 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 628 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 629 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 630 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 631 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 632 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 633 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 634 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 635 -----END RSA PRIVATE KEY-----
 636 """, # 1
 637 """-----BEGIN CERTIFICATE-----
 638 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 639 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 640 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 641 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 642 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 643 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 644 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 645 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 646 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 647 -----END CERTIFICATE-----
 648 -----BEGIN RSA PRIVATE KEY-----
 649 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 650 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 651 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 652 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 653 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 654 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 655 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 656 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 657 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 658 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 659 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 660 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 661 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 662 -----END RSA PRIVATE KEY-----
 663 """, # 2
 664 """-----BEGIN CERTIFICATE-----
 665 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 666 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 667 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 668 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 669 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 670 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 671 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 672 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 673 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 674 -----END CERTIFICATE-----
 675 -----BEGIN RSA PRIVATE KEY-----
 676 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 677 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 678 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 679 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 680 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 681 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 682 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 683 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 684 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 685 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 686 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 687 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 688 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 689 -----END RSA PRIVATE KEY-----
 690 """, # 3
 691 """-----BEGIN CERTIFICATE-----
 692 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 693 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 694 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 695 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 696 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 697 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 698 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 699 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 700 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 701 -----END CERTIFICATE-----
 702 -----BEGIN RSA PRIVATE KEY-----
 703 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 704 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 705 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 706 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 707 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 708 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 709 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 710 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 711 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 712 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 713 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 714 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 715 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 716 -----END RSA PRIVATE KEY-----
 717 """, # 4
 718 """-----BEGIN CERTIFICATE-----
 719 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 720 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 721 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 722 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 723 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 724 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 725 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 726 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 727 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 728 -----END CERTIFICATE-----
 729 -----BEGIN RSA PRIVATE KEY-----
 730 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 731 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 732 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 733 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 734 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 735 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 736 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 737 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 738 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 739 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 740 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 741 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 742 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 743 -----END RSA PRIVATE KEY-----
 744 """, # 5
 745 """-----BEGIN CERTIFICATE-----
 746 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 747 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 748 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 749 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 750 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 751 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 752 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 753 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 754 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 755 -----END CERTIFICATE-----
 756 -----BEGIN RSA PRIVATE KEY-----
 757 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 758 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 759 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 760 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 761 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 762 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 763 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 764 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 765 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 766 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 767 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 768 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 769 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 770 -----END RSA PRIVATE KEY-----
 771 """, #6
 772 """-----BEGIN CERTIFICATE-----
 773 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 774 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 775 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 776 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 777 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 778 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 779 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 780 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 781 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 782 -----END CERTIFICATE-----
 783 -----BEGIN RSA PRIVATE KEY-----
 784 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 785 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 786 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 787 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 788 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 789 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 790 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 791 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 792 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 793 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 794 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 795 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 796 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 797 -----END RSA PRIVATE KEY-----
 798 """, #7
 799 """-----BEGIN CERTIFICATE-----
 800 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 801 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 802 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 803 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 804 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 805 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 806 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 807 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 808 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 809 -----END CERTIFICATE-----
 810 -----BEGIN RSA PRIVATE KEY-----
 811 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 812 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 813 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 814 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 815 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 816 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 817 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 818 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 819 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 820 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 821 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 822 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 823 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 824 -----END RSA PRIVATE KEY-----
 825 """, #8
 826 """-----BEGIN CERTIFICATE-----
 827 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 828 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 829 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 830 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 831 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 832 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 833 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 834 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 835 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 836 -----END CERTIFICATE-----
 837 -----BEGIN RSA PRIVATE KEY-----
 838 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 839 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 840 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 841 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 842 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 843 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 844 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 845 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 846 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 847 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 848 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 849 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 850 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 851 -----END RSA PRIVATE KEY-----
 852 """, #9
 853 """-----BEGIN CERTIFICATE-----
 854 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 855 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 856 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 857 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 858 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 859 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 860 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 861 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 862 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 863 -----END CERTIFICATE-----
 864 -----BEGIN RSA PRIVATE KEY-----
 865 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 866 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 867 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 868 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 869 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 870 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 871 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
 872 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
 873 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
 874 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
 875 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
 876 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
 877 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
 878 -----END RSA PRIVATE KEY-----
 879 """, #10
 880 """-----BEGIN CERTIFICATE-----
 881 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 882 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 883 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
 884 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
 885 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
 886 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 887 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
 888 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
 889 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
 890 -----END CERTIFICATE-----
 891 -----BEGIN RSA PRIVATE KEY-----
 892 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
 893 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
 894 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
 895 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
 896 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
 897 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
 898 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
 899 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
 900 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
 901 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
 902 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
 903 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
 904 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
 905 -----END RSA PRIVATE KEY-----
 906 """, #11
 907 ]
 908
 909 # To disable the pre-computed tub certs, uncomment this line.
 910 #SYSTEM_TEST_CERTS = []
 911
 912 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
 913
 914 class ShareManglingMixin(SystemTestMixin):
 915
 916     def setUp(self):
 917         # Set self.basedir to a temp dir which has the name of the current
 918         # test method in its name.
 919         self.basedir = self.mktemp()
 920
 921         d = defer.maybeDeferred(SystemTestMixin.setUp, self)
 922         d.addCallback(lambda x: self.set_up_nodes())
 923
 924         def _upload_a_file(ignored):
 925             cl0 = self.clients[0]
 926             # We need multiple segments to test crypttext hash trees that are
 927             # non-trivial (i.e. they have more than just one hash in them).
 928             cl0.DEFAULT_ENCODING_PARAMETERS['max_segment_size'] = 12
 929             d2 = cl0.upload(immutable.upload.Data(TEST_DATA, convergence=""))
 930             def _after_upload(u):
 931                 filecap = u.uri
 932                 self.n = self.clients[1].create_node_from_uri(filecap)
 933                 self.uri = uri.CHKFileURI.init_from_string(filecap)
 934                 return cl0.create_node_from_uri(filecap)
 935             d2.addCallback(_after_upload)
 936             return d2
 937         d.addCallback(_upload_a_file)
 938
 939         def _stash_it(filenode):
 940             self.filenode = filenode
 941         d.addCallback(_stash_it)
 942         return d
 943
 944     def find_shares(self, unused=None):
 945         """Locate shares on disk. Returns a dict that maps
 946         (clientnum,sharenum) to a string that contains the share container
 947         (copied directly from the disk, containing leases etc). You can
 948         modify this dict and then call replace_shares() to modify the shares.
 949         """
 950         shares = {} # k: (i, sharenum), v: data
 951
 952         for i, c in enumerate(self.clients):
 953             sharedir = c.getServiceNamed("storage").sharedir
 954             for (dirp, dirns, fns) in os.walk(sharedir):
 955                 for fn in fns:
 956                     try:
 957                         sharenum = int(fn)
 958                     except TypeError:
 959                         # Whoops, I guess that's not a share file then.
 960                         pass
 961                     else:
 962                         data = open(os.path.join(sharedir, dirp, fn), "rb").read()
 963                         shares[(i, sharenum)] = data
 964
 965         return shares
 966
 967     def replace_shares(self, newshares, storage_index):
 968         """Replace shares on disk. Takes a dictionary in the same form
 969         as find_shares() returns."""
 970
 971         for i, c in enumerate(self.clients):
 972             sharedir = c.getServiceNamed("storage").sharedir
 973             for (dirp, dirns, fns) in os.walk(sharedir):
 974                 for fn in fns:
 975                     try:
 976                         sharenum = int(fn)
 977                     except TypeError:
 978                         # Whoops, I guess that's not a share file then.
 979                         pass
 980                     else:
 981                         pathtosharefile = os.path.join(sharedir, dirp, fn)
 982                         os.unlink(pathtosharefile)
 983             for ((clientnum, sharenum), newdata) in newshares.iteritems():
 984                 if clientnum == i:
 985                     fullsharedirp=os.path.join(sharedir, storage_index_to_dir(storage_index))
 986                     fileutil.make_dirs(fullsharedirp)
 987                     wf = open(os.path.join(fullsharedirp, str(sharenum)), "wb")
 988                     wf.write(newdata)
 989                     wf.close()
 990
 991     def _delete_a_share(self, unused=None, sharenum=None):
 992         """ Delete one share. """
 993
 994         shares = self.find_shares()
 995         ks = shares.keys()
 996         if sharenum is not None:
 997             k = [ key for key in shares.keys() if key[1] == sharenum ][0]
 998         else:
 999             k = random.choice(ks)
1000         del shares[k]
1001         self.replace_shares(shares, storage_index=self.uri.storage_index)
1002
1003         return unused
1004
1005     def _corrupt_a_share(self, unused, corruptor_func, sharenum):
1006         shares = self.find_shares()
1007         ks = [ key for key in shares.keys() if key[1] == sharenum ]
1008         assert ks, (shares.keys(), sharenum)
1009         k = ks[0]
1010         shares[k] = corruptor_func(shares[k])
1011         self.replace_shares(shares, storage_index=self.uri.storage_index)
1012         return corruptor_func
1013
1014     def _corrupt_all_shares(self, unused, corruptor_func):
1015         """ All shares on disk will be corrupted by corruptor_func. """
1016         shares = self.find_shares()
1017         for k in shares.keys():
1018             self._corrupt_a_share(unused, corruptor_func, k[1])
1019         return corruptor_func
1020
1021     def _corrupt_a_random_share(self, unused, corruptor_func):
1022         """ Exactly one share on disk will be corrupted by corruptor_func. """
1023         shares = self.find_shares()
1024         ks = shares.keys()
1025         k = random.choice(ks)
1026         self._corrupt_a_share(unused, corruptor_func, k[1])
1027         return k[1]
1028
1029     def _count_reads(self):
1030         sum_of_read_counts = 0
1031         for thisclient in self.clients:
1032             counters = thisclient.stats_provider.get_stats()['counters']
1033             sum_of_read_counts += counters.get('storage_server.read', 0)
1034         return sum_of_read_counts
1035
1036     def _count_allocates(self):
1037         sum_of_allocate_counts = 0
1038         for thisclient in self.clients:
1039             counters = thisclient.stats_provider.get_stats()['counters']
1040             sum_of_allocate_counts += counters.get('storage_server.allocate', 0)
1041         return sum_of_allocate_counts
1042
1043     def _count_writes(self):
1044         sum_of_write_counts = 0
1045         for thisclient in self.clients:
1046             counters = thisclient.stats_provider.get_stats()['counters']
1047             sum_of_write_counts += counters.get('storage_server.write', 0)
1048         return sum_of_write_counts
1049
1050     def _download_and_check_plaintext(self, unused=None):
1051         d = download_to_data(self.n)
1052         def _after_download(result):
1053             self.failUnlessEqual(result, TEST_DATA)
1054         d.addCallback(_after_download)
1055         return d
1056
1057 class ShouldFailMixin:
1058     def shouldFail(self, expected_failure, which, substring,
1059                    callable, *args, **kwargs):
1060         """Assert that a function call raises some exception. This is a
1061         Deferred-friendly version of TestCase.assertRaises() .
1062
1063         Suppose you want to verify the following function:
1064
1065          def broken(a, b, c):
1066              if a < 0:
1067                  raise TypeError('a must not be negative')
1068              return defer.succeed(b+c)
1069
1070         You can use:
1071             d = self.shouldFail(TypeError, 'test name',
1072                                 'a must not be negative',
1073                                 broken, -4, 5, c=12)
1074         in your test method. The 'test name' string will be included in the
1075         error message, if any, because Deferred chains frequently make it
1076         difficult to tell which assertion was tripped.
1077
1078         The substring= argument, if not None, must appear inside the
1079         stringified Failure, or the test will fail.
1080         """
1081
1082         assert substring is None or isinstance(substring, str)
1083         d = defer.maybeDeferred(callable, *args, **kwargs)
1084         def done(res):
1085             if isinstance(res, failure.Failure):
1086                 res.trap(expected_failure)
1087                 if substring:
1088                     self.failUnless(substring in str(res),
1089                                     "substring '%s' not in '%s'"
1090                                     % (substring, str(res)))
1091             else:
1092                 self.fail("%s was supposed to raise %s, not get '%s'" %
1093                           (which, expected_failure, res))
1094         d.addBoth(done)
1095         return d
1096
1097 class WebErrorMixin:
1098     def explain_web_error(self, f):
1099         # an error on the server side causes the client-side getPage() to
1100         # return a failure(t.web.error.Error), and its str() doesn't show the
1101         # response body, which is where the useful information lives. Attach
1102         # this method as an errback handler, and it will reveal the hidden
1103         # message.
1104         f.trap(WebError)
1105         print "Web Error:", f.value, ":", f.value.response
1106         return f
1107
1108     def _shouldHTTPError(self, res, which, validator):
1109         if isinstance(res, failure.Failure):
1110             res.trap(WebError)
1111             return validator(res)
1112         else:
1113             self.fail("%s was supposed to Error, not get '%s'" % (which, res))
1114
1115     def shouldHTTPError(self, which,
1116                         code=None, substring=None, response_substring=None,
1117                         callable=None, *args, **kwargs):
1118         # returns a Deferred with the response body
1119         assert substring is None or isinstance(substring, str)
1120         assert callable
1121         def _validate(f):
1122             if code is not None:
1123                 self.failUnlessEqual(f.value.status, str(code))
1124             if substring:
1125                 code_string = str(f)
1126                 self.failUnless(substring in code_string,
1127                                 "substring '%s' not in '%s'"
1128                                 % (substring, code_string))
1129             response_body = f.value.response
1130             if response_substring:
1131                 self.failUnless(response_substring in response_body,
1132                                 "response substring '%s' not in '%s'"
1133                                 % (response_substring, response_body))
1134             return response_body
1135         d = defer.maybeDeferred(callable, *args, **kwargs)
1136         d.addBoth(self._shouldHTTPError, which, _validate)
1137         return d
1138
1139 class ErrorMixin(WebErrorMixin):
1140     def explain_error(self, f):
1141         if f.check(defer.FirstError):
1142             print "First Error:", f.value.subFailure
1143         return f
1144
1145 def corrupt_field(data, offset, size, debug=False):
1146     if random.random() < 0.5:
1147         newdata = testutil.flip_one_bit(data, offset, size)
1148         if debug:
1149             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1150         return newdata
1151     else:
1152         newval = testutil.insecurerandstr(size)
1153         if debug:
1154             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1155         return data[:offset]+newval+data[offset+size:]
1156
1157 def _corrupt_nothing(data):
1158     """Leave the data pristine. """
1159     return data
1160
1161 def _corrupt_file_version_number(data):
1162     """Scramble the file data -- the share file version number have one bit
1163     flipped or else will be changed to a random value."""
1164     return corrupt_field(data, 0x00, 4)
1165
1166 def _corrupt_size_of_file_data(data):
1167     """Scramble the file data -- the field showing the size of the share data
1168     within the file will be set to one smaller."""
1169     return corrupt_field(data, 0x04, 4)
1170
1171 def _corrupt_sharedata_version_number(data):
1172     """Scramble the file data -- the share data version number will have one
1173     bit flipped or else will be changed to a random value, but not 1 or 2."""
1174     return corrupt_field(data, 0x0c, 4)
1175     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1176     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1177     newsharevernum = sharevernum
1178     while newsharevernum in (1, 2):
1179         newsharevernum = random.randrange(0, 2**32)
1180     newsharevernumbytes = struct.pack(">L", newsharevernum)
1181     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1182
1183 def _corrupt_sharedata_version_number_to_plausible_version(data):
1184     """Scramble the file data -- the share data version number will be
1185     changed to 2 if it is 1 or else to 1 if it is 2."""
1186     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1187     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1188     if sharevernum == 1:
1189         newsharevernum = 2
1190     else:
1191         newsharevernum = 1
1192     newsharevernumbytes = struct.pack(">L", newsharevernum)
1193     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1194
1195 def _corrupt_segment_size(data):
1196     """Scramble the file data -- the field showing the size of the segment
1197     will have one bit flipped or else be changed to a random value."""
1198     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1199     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1200     if sharevernum == 1:
1201         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1202     else:
1203         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1204
1205 def _corrupt_size_of_sharedata(data):
1206     """Scramble the file data -- the field showing the size of the data
1207     within the share data will have one bit flipped or else will be changed
1208     to a random value."""
1209     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1210     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1211     if sharevernum == 1:
1212         return corrupt_field(data, 0x0c+0x08, 4)
1213     else:
1214         return corrupt_field(data, 0x0c+0x0c, 8)
1215
1216 def _corrupt_offset_of_sharedata(data):
1217     """Scramble the file data -- the field showing the offset of the data
1218     within the share data will have one bit flipped or else be changed to a
1219     random value."""
1220     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1221     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1222     if sharevernum == 1:
1223         return corrupt_field(data, 0x0c+0x0c, 4)
1224     else:
1225         return corrupt_field(data, 0x0c+0x14, 8)
1226
1227 def _corrupt_offset_of_ciphertext_hash_tree(data):
1228     """Scramble the file data -- the field showing the offset of the
1229     ciphertext hash tree within the share data will have one bit flipped or
1230     else be changed to a random value.
1231     """
1232     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1233     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1234     if sharevernum == 1:
1235         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1236     else:
1237         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1238
1239 def _corrupt_offset_of_block_hashes(data):
1240     """Scramble the file data -- the field showing the offset of the block
1241     hash tree within the share data will have one bit flipped or else will be
1242     changed to a random value."""
1243     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1244     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1245     if sharevernum == 1:
1246         return corrupt_field(data, 0x0c+0x18, 4)
1247     else:
1248         return corrupt_field(data, 0x0c+0x2c, 8)
1249
1250 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data):
1251     """Scramble the file data -- the field showing the offset of the block
1252     hash tree within the share data will have a multiple of hash size
1253     subtracted from it, thus causing the downloader to download an incomplete
1254     crypttext hash tree."""
1255     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1256     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1257     if sharevernum == 1:
1258         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1259         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1260         newvalstr = struct.pack(">L", newval)
1261         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1262     else:
1263         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1264         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1265         newvalstr = struct.pack(">Q", newval)
1266         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1267
1268 def _corrupt_offset_of_share_hashes(data):
1269     """Scramble the file data -- the field showing the offset of the share
1270     hash tree within the share data will have one bit flipped or else will be
1271     changed to a random value."""
1272     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1273     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1274     if sharevernum == 1:
1275         return corrupt_field(data, 0x0c+0x1c, 4)
1276     else:
1277         return corrupt_field(data, 0x0c+0x34, 8)
1278
1279 def _corrupt_offset_of_uri_extension(data):
1280     """Scramble the file data -- the field showing the offset of the uri
1281     extension will have one bit flipped or else will be changed to a random
1282     value."""
1283     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1284     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1285     if sharevernum == 1:
1286         return corrupt_field(data, 0x0c+0x20, 4)
1287     else:
1288         return corrupt_field(data, 0x0c+0x3c, 8)
1289
1290 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1291     """Scramble the file data -- the field showing the offset of the uri
1292     extension will be set to the size of the file minus 3. This means when
1293     the client tries to read the length field from that location it will get
1294     a short read -- the result string will be only 3 bytes long, not the 4 or
1295     8 bytes necessary to do a successful struct.unpack."""
1296     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1297     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1298     # The "-0x0c" in here is to skip the server-side header in the share
1299     # file, which the client doesn't see when seeking and reading.
1300     if sharevernum == 1:
1301         if debug:
1302             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1303         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1304     else:
1305         if debug:
1306             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1307         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1308
1309 def _corrupt_mutable_share_data(data):
1310     prefix = data[:32]
1311     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
1312     data_offset = MutableShareFile.DATA_OFFSET
1313     sharetype = data[data_offset:data_offset+1]
1314     assert sharetype == "\x00", "non-SDMF mutable shares not supported"
1315     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
1316      ig_datalen, offsets) = unpack_header(data[data_offset:])
1317     assert version == 0, "this function only handles v0 SDMF files"
1318     start = data_offset + offsets["share_data"]
1319     length = data_offset + offsets["enc_privkey"] - start
1320     return corrupt_field(data, start, length)
1321
1322 def _corrupt_share_data(data):
1323     """Scramble the file data -- the field containing the share data itself
1324     will have one bit flipped or else will be changed to a random value."""
1325     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1326     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1327     if sharevernum == 1:
1328         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1329
1330         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1331     else:
1332         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1333
1334         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1335
1336 def _corrupt_crypttext_hash_tree(data):
1337     """Scramble the file data -- the field containing the crypttext hash tree
1338     will have one bit flipped or else will be changed to a random value.
1339     """
1340     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1341     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1342     if sharevernum == 1:
1343         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1344         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1345     else:
1346         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1347         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1348
1349     return corrupt_field(data, crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset)
1350
1351 def _corrupt_block_hashes(data):
1352     """Scramble the file data -- the field containing the block hash tree
1353     will have one bit flipped or else will be changed to a random value.
1354     """
1355     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1356     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1357     if sharevernum == 1:
1358         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1359         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1360     else:
1361         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1362         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1363
1364     return corrupt_field(data, blockhashesoffset, sharehashesoffset-blockhashesoffset)
1365
1366 def _corrupt_share_hashes(data):
1367     """Scramble the file data -- the field containing the share hash chain
1368     will have one bit flipped or else will be changed to a random value.
1369     """
1370     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1371     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1372     if sharevernum == 1:
1373         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1374         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1375     else:
1376         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1377         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1378
1379     return corrupt_field(data, sharehashesoffset, uriextoffset-sharehashesoffset)
1380
1381 def _corrupt_length_of_uri_extension(data):
1382     """Scramble the file data -- the field showing the length of the uri
1383     extension will have one bit flipped or else will be changed to a random
1384     value."""
1385     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1386     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1387     if sharevernum == 1:
1388         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1389         return corrupt_field(data, uriextoffset, 4)
1390     else:
1391         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1392         return corrupt_field(data, uriextoffset, 8)
1393
1394 def _corrupt_uri_extension(data):
1395     """Scramble the file data -- the field containing the uri extension will
1396     have one bit flipped or else will be changed to a random value."""
1397     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1398     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1399     if sharevernum == 1:
1400         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1401         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1402     else:
1403         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1404         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1405
1406     return corrupt_field(data, uriextoffset, uriextlen)