src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IConsumer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.api import flushEventualQueue, fireEventually
   9 from allmydata import uri, dirnode, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IMutableFileNode, IFileNode, \
  12      FileTooLargeError, NotEnoughSharesError, ICheckable
  13 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  14      DeepCheckResults, DeepCheckAndRepairResults
  15 from allmydata.mutable.common import CorruptShareError
  16 from allmydata.mutable.layout import unpack_header
  17 from allmydata.storage.server import storage_index_to_dir
  18 from allmydata.storage.mutable import MutableShareFile
  19 from allmydata.util import hashutil, log, fileutil, pollmixin
  20 from allmydata.util.assertutil import precondition
  21 from allmydata.stats import StatsGathererService
  22 from allmydata.key_generator import KeyGeneratorService
  23 import common_util as testutil
  24 from allmydata import immutable
  25
  26
  27 def flush_but_dont_ignore(res):
  28     d = flushEventualQueue()
  29     def _done(ignored):
  30         return res
  31     d.addCallback(_done)
  32     return d
  33
  34 class FakeCHKFileNode:
  35     """I provide IFileNode, but all of my data is stored in a class-level
  36     dictionary."""
  37     implements(IFileNode)
  38     all_contents = {}
  39     bad_shares = {}
  40
  41     def __init__(self, filecap):
  42         precondition(isinstance(filecap, str), filecap)
  43         self.my_uri = uri.CHKFileURI.init_from_string(filecap)
  44         self.storage_index = self.my_uri.storage_index
  45
  46     def get_uri(self):
  47         return self.my_uri.to_string()
  48     def get_readonly_uri(self):
  49         return self.my_uri.to_string()
  50     def get_verify_cap(self):
  51         return self.my_uri.get_verify_cap()
  52     def get_repair_cap(self):
  53         return self.my_uri.get_verify_cap()
  54     def get_storage_index(self):
  55         return self.storage_index
  56
  57     def check(self, monitor, verify=False, add_lease=False):
  58         r = CheckResults(self.my_uri, self.storage_index)
  59         is_bad = self.bad_shares.get(self.storage_index, None)
  60         data = {}
  61         data["count-shares-needed"] = 3
  62         data["count-shares-expected"] = 10
  63         data["count-good-share-hosts"] = 10
  64         data["count-wrong-shares"] = 0
  65         nodeid = "\x00"*20
  66         data["list-corrupt-shares"] = []
  67         data["sharemap"] = {1: [nodeid]}
  68         data["servers-responding"] = [nodeid]
  69         data["count-recoverable-versions"] = 1
  70         data["count-unrecoverable-versions"] = 0
  71         if is_bad:
  72              r.set_healthy(False)
  73              r.set_recoverable(True)
  74              data["count-shares-good"] = 9
  75              data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
  76              r.problems = failure.Failure(CorruptShareError(is_bad))
  77         else:
  78              r.set_healthy(True)
  79              r.set_recoverable(True)
  80              data["count-shares-good"] = 10
  81              r.problems = []
  82         r.set_data(data)
  83         r.set_needs_rebalancing(False)
  84         return defer.succeed(r)
  85     def check_and_repair(self, monitor, verify=False, add_lease=False):
  86         d = self.check(verify)
  87         def _got(cr):
  88             r = CheckAndRepairResults(self.storage_index)
  89             r.pre_repair_results = r.post_repair_results = cr
  90             return r
  91         d.addCallback(_got)
  92         return d
  93
  94     def is_mutable(self):
  95         return False
  96     def is_readonly(self):
  97         return True
  98
  99     def download(self, target):
 100         if self.my_uri.to_string() not in self.all_contents:
 101             f = failure.Failure(NotEnoughSharesError(None, 0, 3))
 102             target.fail(f)
 103             return defer.fail(f)
 104         data = self.all_contents[self.my_uri.to_string()]
 105         target.open(len(data))
 106         target.write(data)
 107         target.close()
 108         return defer.maybeDeferred(target.finish)
 109     def download_to_data(self):
 110         if self.my_uri.to_string() not in self.all_contents:
 111             return defer.fail(NotEnoughSharesError(None, 0, 3))
 112         data = self.all_contents[self.my_uri.to_string()]
 113         return defer.succeed(data)
 114     def get_size(self):
 115         try:
 116             data = self.all_contents[self.my_uri.to_string()]
 117         except KeyError, le:
 118             raise NotEnoughSharesError(le, 0, 3)
 119         return len(data)
 120     def read(self, consumer, offset=0, size=None):
 121         d = self.download_to_data()
 122         def _got(data):
 123             start = offset
 124             if size is not None:
 125                 end = offset + size
 126             else:
 127                 end = len(data)
 128             consumer.write(data[start:end])
 129             return consumer
 130         d.addCallback(_got)
 131         return d
 132
 133 def make_chk_file_uri(size):
 134     u = uri.CHKFileURI(key=os.urandom(16),
 135                        uri_extension_hash=os.urandom(32),
 136                        needed_shares=3,
 137                        total_shares=10,
 138                        size=size)
 139     return u.to_string()
 140
 141 def create_chk_filenode(contents):
 142     filecap = make_chk_file_uri(len(contents))
 143     n = FakeCHKFileNode(filecap)
 144     FakeCHKFileNode.all_contents[filecap] = contents
 145     return n
 146
 147
 148 class FakeMutableFileNode:
 149     """I provide IMutableFileNode, but all of my data is stored in a
 150     class-level dictionary."""
 151
 152     implements(IMutableFileNode, ICheckable)
 153     MUTABLE_SIZELIMIT = 10000
 154     all_contents = {}
 155     bad_shares = {}
 156
 157     def __init__(self, storage_broker, secret_holder,
 158                  default_encoding_parameters, history):
 159         self.init_from_uri(make_mutable_file_uri())
 160     def create(self, initial_contents, key_generator=None, keysize=None):
 161         if len(initial_contents) > self.MUTABLE_SIZELIMIT:
 162             raise FileTooLargeError("SDMF is limited to one segment, and "
 163                                     "%d > %d" % (len(initial_contents),
 164                                                  self.MUTABLE_SIZELIMIT))
 165         self.all_contents[self.storage_index] = initial_contents
 166         return defer.succeed(self)
 167     def init_from_uri(self, filecap):
 168         assert isinstance(filecap, str)
 169         if filecap.startswith("URI:SSK:"):
 170             self.my_uri = uri.WriteableSSKFileURI.init_from_string(filecap)
 171         else:
 172             assert filecap.startswith("URI:SSK-RO:")
 173             self.my_uri = uri.ReadonlySSKFileURI.init_from_string(filecap)
 174         self.storage_index = self.my_uri.storage_index
 175         return self
 176     def get_uri(self):
 177         return self.my_uri.to_string()
 178     def get_readonly(self):
 179         return self.my_uri.get_readonly()
 180     def get_readonly_uri(self):
 181         return self.my_uri.get_readonly().to_string()
 182     def get_verify_cap(self):
 183         return self.my_uri.get_verify_cap()
 184     def is_readonly(self):
 185         return self.my_uri.is_readonly()
 186     def is_mutable(self):
 187         return self.my_uri.is_mutable()
 188     def get_writekey(self):
 189         return "\x00"*16
 190     def get_size(self):
 191         return "?" # TODO: see mutable.MutableFileNode.get_size
 192     def get_size_of_best_version(self):
 193         return defer.succeed(len(self.all_contents[self.storage_index]))
 194
 195     def get_storage_index(self):
 196         return self.storage_index
 197
 198     def check(self, monitor, verify=False, add_lease=False):
 199         r = CheckResults(self.my_uri, self.storage_index)
 200         is_bad = self.bad_shares.get(self.storage_index, None)
 201         data = {}
 202         data["count-shares-needed"] = 3
 203         data["count-shares-expected"] = 10
 204         data["count-good-share-hosts"] = 10
 205         data["count-wrong-shares"] = 0
 206         data["list-corrupt-shares"] = []
 207         nodeid = "\x00"*20
 208         data["sharemap"] = {"seq1-abcd-sh0": [nodeid]}
 209         data["servers-responding"] = [nodeid]
 210         data["count-recoverable-versions"] = 1
 211         data["count-unrecoverable-versions"] = 0
 212         if is_bad:
 213              r.set_healthy(False)
 214              r.set_recoverable(True)
 215              data["count-shares-good"] = 9
 216              r.problems = failure.Failure(CorruptShareError("peerid",
 217                                                             0, # shnum
 218                                                             is_bad))
 219         else:
 220              r.set_healthy(True)
 221              r.set_recoverable(True)
 222              data["count-shares-good"] = 10
 223              r.problems = []
 224         r.set_data(data)
 225         r.set_needs_rebalancing(False)
 226         return defer.succeed(r)
 227
 228     def check_and_repair(self, monitor, verify=False, add_lease=False):
 229         d = self.check(verify)
 230         def _got(cr):
 231             r = CheckAndRepairResults(self.storage_index)
 232             r.pre_repair_results = r.post_repair_results = cr
 233             return r
 234         d.addCallback(_got)
 235         return d
 236
 237     def deep_check(self, verify=False, add_lease=False):
 238         d = self.check(verify)
 239         def _done(r):
 240             dr = DeepCheckResults(self.storage_index)
 241             dr.add_check(r, [])
 242             return dr
 243         d.addCallback(_done)
 244         return d
 245
 246     def deep_check_and_repair(self, verify=False, add_lease=False):
 247         d = self.check_and_repair(verify)
 248         def _done(r):
 249             dr = DeepCheckAndRepairResults(self.storage_index)
 250             dr.add_check(r, [])
 251             return dr
 252         d.addCallback(_done)
 253         return d
 254
 255     def download_best_version(self):
 256         return defer.succeed(self.all_contents[self.storage_index])
 257     def overwrite(self, new_contents):
 258         if len(new_contents) > self.MUTABLE_SIZELIMIT:
 259             raise FileTooLargeError("SDMF is limited to one segment, and "
 260                                     "%d > %d" % (len(new_contents),
 261                                                  self.MUTABLE_SIZELIMIT))
 262         assert not self.is_readonly()
 263         self.all_contents[self.storage_index] = new_contents
 264         return defer.succeed(None)
 265     def modify(self, modifier):
 266         # this does not implement FileTooLargeError, but the real one does
 267         return defer.maybeDeferred(self._modify, modifier)
 268     def _modify(self, modifier):
 269         assert not self.is_readonly()
 270         old_contents = self.all_contents[self.storage_index]
 271         self.all_contents[self.storage_index] = modifier(old_contents, None, True)
 272         return None
 273
 274     def download(self, target):
 275         if self.storage_index not in self.all_contents:
 276             f = failure.Failure(NotEnoughSharesError(None, 0, 3))
 277             target.fail(f)
 278             return defer.fail(f)
 279         data = self.all_contents[self.storage_index]
 280         target.open(len(data))
 281         target.write(data)
 282         target.close()
 283         return defer.maybeDeferred(target.finish)
 284     def download_to_data(self):
 285         if self.storage_index not in self.all_contents:
 286             return defer.fail(NotEnoughSharesError(None, 0, 3))
 287         data = self.all_contents[self.storage_index]
 288         return defer.succeed(data)
 289
 290 def make_mutable_file_uri():
 291     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 292                                    fingerprint=os.urandom(32)).to_string()
 293 def make_verifier_uri():
 294     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 295                               fingerprint=os.urandom(32)).to_string()
 296
 297 class FakeDirectoryNode(dirnode.DirectoryNode):
 298     """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
 299     backing store, so it doesn't go to the grid. The child data is still
 300     encrypted and serialized, so this isn't useful for tests that want to
 301     look inside the dirnodes and check their contents.
 302     """
 303     filenode_class = FakeMutableFileNode
 304
 305 class LoggingServiceParent(service.MultiService):
 306     def log(self, *args, **kwargs):
 307         return log.msg(*args, **kwargs)
 308
 309
 310 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 311
 312     # SystemTestMixin tests tend to be a lot of work, and we have a few
 313     # buildslaves that are pretty slow, and every once in a while these tests
 314     # run up against the default 120 second timeout. So increase the default
 315     # timeout. Individual test cases can override this, of course.
 316     timeout = 300
 317
 318     def setUp(self):
 319         self.sparent = service.MultiService()
 320         self.sparent.startService()
 321
 322         self.stats_gatherer = None
 323         self.stats_gatherer_furl = None
 324         self.key_generator_svc = None
 325         self.key_generator_furl = None
 326
 327     def tearDown(self):
 328         log.msg("shutting down SystemTest services")
 329         d = self.sparent.stopService()
 330         d.addBoth(flush_but_dont_ignore)
 331         return d
 332
 333     def getdir(self, subdir):
 334         return os.path.join(self.basedir, subdir)
 335
 336     def add_service(self, s):
 337         s.setServiceParent(self.sparent)
 338         return s
 339
 340     def set_up_nodes(self, NUMCLIENTS=5,
 341                      use_stats_gatherer=False, use_key_generator=False):
 342         self.numclients = NUMCLIENTS
 343         iv_dir = self.getdir("introducer")
 344         if not os.path.isdir(iv_dir):
 345             fileutil.make_dirs(iv_dir)
 346             f = open(os.path.join(iv_dir, "webport"), "w")
 347             f.write("tcp:0:interface=127.0.0.1\n")
 348             f.close()
 349             if SYSTEM_TEST_CERTS:
 350                 os.mkdir(os.path.join(iv_dir, "private"))
 351                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 352                 f.write(SYSTEM_TEST_CERTS[0])
 353                 f.close()
 354         iv = IntroducerNode(basedir=iv_dir)
 355         self.introducer = self.add_service(iv)
 356         d = self.introducer.when_tub_ready()
 357         d.addCallback(self._get_introducer_web)
 358         if use_stats_gatherer:
 359             d.addCallback(self._set_up_stats_gatherer)
 360         if use_key_generator:
 361             d.addCallback(self._set_up_key_generator)
 362         d.addCallback(self._set_up_nodes_2)
 363         if use_stats_gatherer:
 364             d.addCallback(self._grab_stats)
 365         return d
 366
 367     def _get_introducer_web(self, res):
 368         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 369         self.introweb_url = f.read().strip()
 370         f.close()
 371
 372     def _set_up_stats_gatherer(self, res):
 373         statsdir = self.getdir("stats_gatherer")
 374         fileutil.make_dirs(statsdir)
 375         self.stats_gatherer_svc = StatsGathererService(statsdir)
 376         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 377         self.add_service(self.stats_gatherer_svc)
 378
 379         d = fireEventually()
 380         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 381         def check_for_furl():
 382             return os.path.exists(sgf)
 383         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 384         def get_furl(junk):
 385             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 386         d.addCallback(get_furl)
 387         return d
 388
 389     def _set_up_key_generator(self, res):
 390         kgsdir = self.getdir("key_generator")
 391         fileutil.make_dirs(kgsdir)
 392
 393         self.key_generator_svc = KeyGeneratorService(kgsdir,
 394                                                      display_furl=False,
 395                                                      default_key_size=522)
 396         self.key_generator_svc.key_generator.pool_size = 4
 397         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 398         self.add_service(self.key_generator_svc)
 399
 400         d = fireEventually()
 401         def check_for_furl():
 402             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 403         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 404         def get_furl(junk):
 405             kgf = os.path.join(kgsdir, 'key_generator.furl')
 406             self.key_generator_furl = file(kgf, 'rb').read().strip()
 407         d.addCallback(get_furl)
 408         return d
 409
 410     def _set_up_nodes_2(self, res):
 411         q = self.introducer
 412         self.introducer_furl = q.introducer_url
 413         self.clients = []
 414         basedirs = []
 415         for i in range(self.numclients):
 416             basedir = self.getdir("client%d" % i)
 417             basedirs.append(basedir)
 418             fileutil.make_dirs(os.path.join(basedir, "private"))
 419             if len(SYSTEM_TEST_CERTS) > (i+1):
 420                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 421                 f.write(SYSTEM_TEST_CERTS[i+1])
 422                 f.close()
 423
 424             def write(name, value):
 425                 open(os.path.join(basedir, name), "w").write(value+"\n")
 426             if i == 0:
 427                 # clients[0] runs a webserver and a helper, no key_generator
 428                 write("webport", "tcp:0:interface=127.0.0.1")
 429                 write("run_helper", "yes")
 430                 write("keepalive_timeout", "600")
 431             if i == 3:
 432                 # clients[3] runs a webserver and uses a helper, uses
 433                 # key_generator
 434                 write("webport", "tcp:0:interface=127.0.0.1")
 435                 write("disconnect_timeout", "1800")
 436                 if self.key_generator_furl:
 437                     kgf = "%s\n" % (self.key_generator_furl,)
 438                     write("key_generator.furl", kgf)
 439             write("introducer.furl", self.introducer_furl)
 440             if self.stats_gatherer_furl:
 441                 write("stats_gatherer.furl", self.stats_gatherer_furl)
 442
 443         # give subclasses a chance to append liens to the node's tahoe.cfg
 444         # files before they are launched.
 445         self._set_up_nodes_extra_config()
 446
 447         # start clients[0], wait for it's tub to be ready (at which point it
 448         # will have registered the helper furl).
 449         c = self.add_service(client.Client(basedir=basedirs[0]))
 450         self.clients.append(c)
 451         c.set_default_mutable_keysize(522)
 452         d = c.when_tub_ready()
 453         def _ready(res):
 454             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 455             helper_furl = f.read()
 456             f.close()
 457             self.helper_furl = helper_furl
 458             if self.numclients >= 4:
 459                 f = open(os.path.join(basedirs[3],"helper.furl"), "w")
 460                 f.write(helper_furl)
 461                 f.close()
 462
 463             # this starts the rest of the clients
 464             for i in range(1, self.numclients):
 465                 c = self.add_service(client.Client(basedir=basedirs[i]))
 466                 self.clients.append(c)
 467                 c.set_default_mutable_keysize(522)
 468             log.msg("STARTING")
 469             return self.wait_for_connections()
 470         d.addCallback(_ready)
 471         def _connected(res):
 472             log.msg("CONNECTED")
 473             # now find out where the web port was
 474             l = self.clients[0].getServiceNamed("webish").listener
 475             port = l._port.getHost().port
 476             self.webish_url = "http://localhost:%d/" % port
 477             if self.numclients >=4:
 478                 # and the helper-using webport
 479                 l = self.clients[3].getServiceNamed("webish").listener
 480                 port = l._port.getHost().port
 481                 self.helper_webish_url = "http://localhost:%d/" % port
 482         d.addCallback(_connected)
 483         return d
 484
 485     def _set_up_nodes_extra_config(self):
 486         # for overriding by subclasses
 487         pass
 488
 489     def _grab_stats(self, res):
 490         d = self.stats_gatherer.poll()
 491         return d
 492
 493     def bounce_client(self, num):
 494         c = self.clients[num]
 495         d = c.disownServiceParent()
 496         # I think windows requires a moment to let the connection really stop
 497         # and the port number made available for re-use. TODO: examine the
 498         # behavior, see if this is really the problem, see if we can do
 499         # better than blindly waiting for a second.
 500         d.addCallback(self.stall, 1.0)
 501         def _stopped(res):
 502             new_c = client.Client(basedir=self.getdir("client%d" % num))
 503             self.clients[num] = new_c
 504             new_c.set_default_mutable_keysize(522)
 505             self.add_service(new_c)
 506             return new_c.when_tub_ready()
 507         d.addCallback(_stopped)
 508         d.addCallback(lambda res: self.wait_for_connections())
 509         def _maybe_get_webport(res):
 510             if num == 0:
 511                 # now find out where the web port was
 512                 l = self.clients[0].getServiceNamed("webish").listener
 513                 port = l._port.getHost().port
 514                 self.webish_url = "http://localhost:%d/" % port
 515         d.addCallback(_maybe_get_webport)
 516         return d
 517
 518     def add_extra_node(self, client_num, helper_furl=None,
 519                        add_to_sparent=False):
 520         # usually this node is *not* parented to our self.sparent, so we can
 521         # shut it down separately from the rest, to exercise the
 522         # connection-lost code
 523         basedir = self.getdir("client%d" % client_num)
 524         if not os.path.isdir(basedir):
 525             fileutil.make_dirs(basedir)
 526         open(os.path.join(basedir, "introducer.furl"), "w").write(self.introducer_furl)
 527         if helper_furl:
 528             f = open(os.path.join(basedir, "helper.furl") ,"w")
 529             f.write(helper_furl+"\n")
 530             f.close()
 531
 532         c = client.Client(basedir=basedir)
 533         self.clients.append(c)
 534         c.set_default_mutable_keysize(522)
 535         self.numclients += 1
 536         if add_to_sparent:
 537             c.setServiceParent(self.sparent)
 538         else:
 539             c.startService()
 540         d = self.wait_for_connections()
 541         d.addCallback(lambda res: c)
 542         return d
 543
 544     def _check_connections(self):
 545         for c in self.clients:
 546             if not c.connected_to_introducer():
 547                 return False
 548             sb = c.get_storage_broker()
 549             if len(sb.get_all_servers()) != self.numclients:
 550                 return False
 551         return True
 552
 553     def wait_for_connections(self, ignored=None):
 554         # TODO: replace this with something that takes a list of peerids and
 555         # fires when they've all been heard from, instead of using a count
 556         # and a threshold
 557         return self.poll(self._check_connections, timeout=200)
 558
 559
 560 # our system test uses the same Tub certificates each time, to avoid the
 561 # overhead of key generation
 562 SYSTEM_TEST_CERTS = [
 563 """-----BEGIN CERTIFICATE-----
 564 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 565 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 566 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 567 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 568 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 569 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 570 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 571 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 572 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 573 -----END CERTIFICATE-----
 574 -----BEGIN RSA PRIVATE KEY-----
 575 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 576 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 577 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 578 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 579 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 580 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 581 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 582 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 583 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 584 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 585 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 586 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 587 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 588 -----END RSA PRIVATE KEY-----
 589 """, # 0
 590 """-----BEGIN CERTIFICATE-----
 591 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 592 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 593 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 594 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 595 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 596 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 597 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 598 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 599 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 600 -----END CERTIFICATE-----
 601 -----BEGIN RSA PRIVATE KEY-----
 602 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 603 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 604 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 605 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 606 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 607 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 608 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 609 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 610 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 611 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 612 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 613 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 614 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 615 -----END RSA PRIVATE KEY-----
 616 """, # 1
 617 """-----BEGIN CERTIFICATE-----
 618 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 619 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 620 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 621 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 622 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 623 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 624 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 625 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 626 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 627 -----END CERTIFICATE-----
 628 -----BEGIN RSA PRIVATE KEY-----
 629 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 630 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 631 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 632 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 633 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 634 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 635 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 636 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 637 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 638 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 639 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 640 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 641 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 642 -----END RSA PRIVATE KEY-----
 643 """, # 2
 644 """-----BEGIN CERTIFICATE-----
 645 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 646 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 647 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 648 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 649 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 650 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 651 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 652 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 653 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 654 -----END CERTIFICATE-----
 655 -----BEGIN RSA PRIVATE KEY-----
 656 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 657 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 658 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 659 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 660 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 661 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 662 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 663 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 664 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 665 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 666 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 667 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 668 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 669 -----END RSA PRIVATE KEY-----
 670 """, # 3
 671 """-----BEGIN CERTIFICATE-----
 672 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 673 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 674 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 675 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 676 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 677 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 678 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 679 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 680 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 681 -----END CERTIFICATE-----
 682 -----BEGIN RSA PRIVATE KEY-----
 683 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 684 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 685 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 686 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 687 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 688 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 689 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 690 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 691 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 692 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 693 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 694 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 695 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 696 -----END RSA PRIVATE KEY-----
 697 """, # 4
 698 """-----BEGIN CERTIFICATE-----
 699 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 700 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 701 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 702 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 703 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 704 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 705 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 706 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 707 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 708 -----END CERTIFICATE-----
 709 -----BEGIN RSA PRIVATE KEY-----
 710 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 711 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 712 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 713 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 714 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 715 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 716 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 717 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 718 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 719 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 720 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 721 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 722 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 723 -----END RSA PRIVATE KEY-----
 724 """, # 5
 725 """-----BEGIN CERTIFICATE-----
 726 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 727 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 728 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 729 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 730 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 731 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 732 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 733 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 734 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 735 -----END CERTIFICATE-----
 736 -----BEGIN RSA PRIVATE KEY-----
 737 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 738 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 739 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 740 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 741 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 742 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 743 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 744 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 745 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 746 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 747 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 748 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 749 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 750 -----END RSA PRIVATE KEY-----
 751 """, #6
 752 """-----BEGIN CERTIFICATE-----
 753 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 754 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 755 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 756 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 757 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 758 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 759 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 760 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 761 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 762 -----END CERTIFICATE-----
 763 -----BEGIN RSA PRIVATE KEY-----
 764 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 765 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 766 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 767 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 768 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 769 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 770 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 771 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 772 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 773 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 774 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 775 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 776 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 777 -----END RSA PRIVATE KEY-----
 778 """, #7
 779 """-----BEGIN CERTIFICATE-----
 780 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 781 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 782 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 783 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 784 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 785 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 786 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 787 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 788 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 789 -----END CERTIFICATE-----
 790 -----BEGIN RSA PRIVATE KEY-----
 791 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 792 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 793 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 794 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 795 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 796 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 797 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 798 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 799 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 800 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 801 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 802 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 803 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 804 -----END RSA PRIVATE KEY-----
 805 """, #8
 806 """-----BEGIN CERTIFICATE-----
 807 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 808 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 809 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 810 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 811 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 812 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 813 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 814 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 815 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 816 -----END CERTIFICATE-----
 817 -----BEGIN RSA PRIVATE KEY-----
 818 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 819 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 820 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 821 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 822 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 823 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 824 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 825 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 826 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 827 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 828 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 829 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 830 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 831 -----END RSA PRIVATE KEY-----
 832 """, #9
 833 """-----BEGIN CERTIFICATE-----
 834 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 835 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 836 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 837 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 838 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 839 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 840 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 841 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 842 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 843 -----END CERTIFICATE-----
 844 -----BEGIN RSA PRIVATE KEY-----
 845 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 846 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 847 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 848 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 849 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 850 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 851 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
 852 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
 853 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
 854 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
 855 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
 856 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
 857 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
 858 -----END RSA PRIVATE KEY-----
 859 """, #10
 860 """-----BEGIN CERTIFICATE-----
 861 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 862 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 863 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
 864 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
 865 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
 866 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 867 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
 868 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
 869 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
 870 -----END CERTIFICATE-----
 871 -----BEGIN RSA PRIVATE KEY-----
 872 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
 873 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
 874 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
 875 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
 876 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
 877 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
 878 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
 879 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
 880 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
 881 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
 882 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
 883 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
 884 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
 885 -----END RSA PRIVATE KEY-----
 886 """, #11
 887 ]
 888
 889 # To disable the pre-computed tub certs, uncomment this line.
 890 #SYSTEM_TEST_CERTS = []
 891
 892 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
 893
 894 class ShareManglingMixin(SystemTestMixin):
 895
 896     def setUp(self):
 897         # Set self.basedir to a temp dir which has the name of the current
 898         # test method in its name.
 899         self.basedir = self.mktemp()
 900
 901         d = defer.maybeDeferred(SystemTestMixin.setUp, self)
 902         d.addCallback(lambda x: self.set_up_nodes())
 903
 904         def _upload_a_file(ignored):
 905             cl0 = self.clients[0]
 906             # We need multiple segments to test crypttext hash trees that are
 907             # non-trivial (i.e. they have more than just one hash in them).
 908             cl0.DEFAULT_ENCODING_PARAMETERS['max_segment_size'] = 12
 909             d2 = cl0.upload(immutable.upload.Data(TEST_DATA, convergence=""))
 910             def _after_upload(u):
 911                 filecap = u.uri
 912                 self.uri = uri.CHKFileURI.init_from_string(filecap)
 913                 return cl0.create_node_from_uri(filecap)
 914             d2.addCallback(_after_upload)
 915             return d2
 916         d.addCallback(_upload_a_file)
 917
 918         def _stash_it(filenode):
 919             self.filenode = filenode
 920         d.addCallback(_stash_it)
 921         return d
 922
 923     def find_shares(self, unused=None):
 924         """Locate shares on disk. Returns a dict that maps
 925         (clientnum,sharenum) to a string that contains the share container
 926         (copied directly from the disk, containing leases etc). You can
 927         modify this dict and then call replace_shares() to modify the shares.
 928         """
 929         shares = {} # k: (i, sharenum), v: data
 930
 931         for i, c in enumerate(self.clients):
 932             sharedir = c.getServiceNamed("storage").sharedir
 933             for (dirp, dirns, fns) in os.walk(sharedir):
 934                 for fn in fns:
 935                     try:
 936                         sharenum = int(fn)
 937                     except TypeError:
 938                         # Whoops, I guess that's not a share file then.
 939                         pass
 940                     else:
 941                         data = open(os.path.join(sharedir, dirp, fn), "rb").read()
 942                         shares[(i, sharenum)] = data
 943
 944         return shares
 945
 946     def replace_shares(self, newshares, storage_index):
 947         """Replace shares on disk. Takes a dictionary in the same form
 948         as find_shares() returns."""
 949
 950         for i, c in enumerate(self.clients):
 951             sharedir = c.getServiceNamed("storage").sharedir
 952             for (dirp, dirns, fns) in os.walk(sharedir):
 953                 for fn in fns:
 954                     try:
 955                         sharenum = int(fn)
 956                     except TypeError:
 957                         # Whoops, I guess that's not a share file then.
 958                         pass
 959                     else:
 960                         pathtosharefile = os.path.join(sharedir, dirp, fn)
 961                         os.unlink(pathtosharefile)
 962             for ((clientnum, sharenum), newdata) in newshares.iteritems():
 963                 if clientnum == i:
 964                     fullsharedirp=os.path.join(sharedir, storage_index_to_dir(storage_index))
 965                     fileutil.make_dirs(fullsharedirp)
 966                     wf = open(os.path.join(fullsharedirp, str(sharenum)), "wb")
 967                     wf.write(newdata)
 968                     wf.close()
 969
 970     def _delete_a_share(self, unused=None, sharenum=None):
 971         """ Delete one share. """
 972
 973         shares = self.find_shares()
 974         ks = shares.keys()
 975         if sharenum is not None:
 976             k = [ key for key in shares.keys() if key[1] == sharenum ][0]
 977         else:
 978             k = random.choice(ks)
 979         del shares[k]
 980         self.replace_shares(shares, storage_index=self.uri.storage_index)
 981
 982         return unused
 983
 984     def _corrupt_a_share(self, unused, corruptor_func, sharenum):
 985         shares = self.find_shares()
 986         ks = [ key for key in shares.keys() if key[1] == sharenum ]
 987         assert ks, (shares.keys(), sharenum)
 988         k = ks[0]
 989         shares[k] = corruptor_func(shares[k])
 990         self.replace_shares(shares, storage_index=self.uri.storage_index)
 991         return corruptor_func
 992
 993     def _corrupt_all_shares(self, unused, corruptor_func):
 994         """ All shares on disk will be corrupted by corruptor_func. """
 995         shares = self.find_shares()
 996         for k in shares.keys():
 997             self._corrupt_a_share(unused, corruptor_func, k[1])
 998         return corruptor_func
 999
1000     def _corrupt_a_random_share(self, unused, corruptor_func):
1001         """ Exactly one share on disk will be corrupted by corruptor_func. """
1002         shares = self.find_shares()
1003         ks = shares.keys()
1004         k = random.choice(ks)
1005         self._corrupt_a_share(unused, corruptor_func, k[1])
1006         return k[1]
1007
1008     def _count_reads(self):
1009         sum_of_read_counts = 0
1010         for thisclient in self.clients:
1011             counters = thisclient.stats_provider.get_stats()['counters']
1012             sum_of_read_counts += counters.get('storage_server.read', 0)
1013         return sum_of_read_counts
1014
1015     def _count_allocates(self):
1016         sum_of_allocate_counts = 0
1017         for thisclient in self.clients:
1018             counters = thisclient.stats_provider.get_stats()['counters']
1019             sum_of_allocate_counts += counters.get('storage_server.allocate', 0)
1020         return sum_of_allocate_counts
1021
1022     def _count_writes(self):
1023         sum_of_write_counts = 0
1024         for thisclient in self.clients:
1025             counters = thisclient.stats_provider.get_stats()['counters']
1026             sum_of_write_counts += counters.get('storage_server.write', 0)
1027         return sum_of_write_counts
1028
1029     def _download_and_check_plaintext(self, unused=None):
1030         self.downloader = self.clients[1].getServiceNamed("downloader")
1031         d = self.downloader.download_to_data(self.uri)
1032
1033         def _after_download(result):
1034             self.failUnlessEqual(result, TEST_DATA)
1035         d.addCallback(_after_download)
1036         return d
1037
1038 class ShouldFailMixin:
1039     def shouldFail(self, expected_failure, which, substring,
1040                    callable, *args, **kwargs):
1041         """Assert that a function call raises some exception. This is a
1042         Deferred-friendly version of TestCase.assertRaises() .
1043
1044         Suppose you want to verify the following function:
1045
1046          def broken(a, b, c):
1047              if a < 0:
1048                  raise TypeError('a must not be negative')
1049              return defer.succeed(b+c)
1050
1051         You can use:
1052             d = self.shouldFail(TypeError, 'test name',
1053                                 'a must not be negative',
1054                                 broken, -4, 5, c=12)
1055         in your test method. The 'test name' string will be included in the
1056         error message, if any, because Deferred chains frequently make it
1057         difficult to tell which assertion was tripped.
1058
1059         The substring= argument, if not None, must appear inside the
1060         stringified Failure, or the test will fail.
1061         """
1062
1063         assert substring is None or isinstance(substring, str)
1064         d = defer.maybeDeferred(callable, *args, **kwargs)
1065         def done(res):
1066             if isinstance(res, failure.Failure):
1067                 res.trap(expected_failure)
1068                 if substring:
1069                     self.failUnless(substring in str(res),
1070                                     "substring '%s' not in '%s'"
1071                                     % (substring, str(res)))
1072             else:
1073                 self.fail("%s was supposed to raise %s, not get '%s'" %
1074                           (which, expected_failure, res))
1075         d.addBoth(done)
1076         return d
1077
1078 class WebErrorMixin:
1079     def explain_web_error(self, f):
1080         # an error on the server side causes the client-side getPage() to
1081         # return a failure(t.web.error.Error), and its str() doesn't show the
1082         # response body, which is where the useful information lives. Attach
1083         # this method as an errback handler, and it will reveal the hidden
1084         # message.
1085         f.trap(WebError)
1086         print "Web Error:", f.value, ":", f.value.response
1087         return f
1088
1089     def _shouldHTTPError(self, res, which, validator):
1090         if isinstance(res, failure.Failure):
1091             res.trap(WebError)
1092             return validator(res)
1093         else:
1094             self.fail("%s was supposed to Error, not get '%s'" % (which, res))
1095
1096     def shouldHTTPError(self, which,
1097                         code=None, substring=None, response_substring=None,
1098                         callable=None, *args, **kwargs):
1099         # returns a Deferred with the response body
1100         assert substring is None or isinstance(substring, str)
1101         assert callable
1102         def _validate(f):
1103             if code is not None:
1104                 self.failUnlessEqual(f.value.status, str(code))
1105             if substring:
1106                 code_string = str(f)
1107                 self.failUnless(substring in code_string,
1108                                 "substring '%s' not in '%s'"
1109                                 % (substring, code_string))
1110             response_body = f.value.response
1111             if response_substring:
1112                 self.failUnless(response_substring in response_body,
1113                                 "response substring '%s' not in '%s'"
1114                                 % (response_substring, response_body))
1115             return response_body
1116         d = defer.maybeDeferred(callable, *args, **kwargs)
1117         d.addBoth(self._shouldHTTPError, which, _validate)
1118         return d
1119
1120 class ErrorMixin(WebErrorMixin):
1121     def explain_error(self, f):
1122         if f.check(defer.FirstError):
1123             print "First Error:", f.value.subFailure
1124         return f
1125
1126 class MemoryConsumer:
1127     implements(IConsumer)
1128     def __init__(self):
1129         self.chunks = []
1130         self.done = False
1131     def registerProducer(self, p, streaming):
1132         if streaming:
1133             # call resumeProducing once to start things off
1134             p.resumeProducing()
1135         else:
1136             while not self.done:
1137                 p.resumeProducing()
1138     def write(self, data):
1139         self.chunks.append(data)
1140     def unregisterProducer(self):
1141         self.done = True
1142
1143 def download_to_data(n, offset=0, size=None):
1144     d = n.read(MemoryConsumer(), offset, size)
1145     d.addCallback(lambda mc: "".join(mc.chunks))
1146     return d
1147
1148 def corrupt_field(data, offset, size, debug=False):
1149     if random.random() < 0.5:
1150         newdata = testutil.flip_one_bit(data, offset, size)
1151         if debug:
1152             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1153         return newdata
1154     else:
1155         newval = testutil.insecurerandstr(size)
1156         if debug:
1157             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1158         return data[:offset]+newval+data[offset+size:]
1159
1160 def _corrupt_nothing(data):
1161     """ Leave the data pristine. """
1162     return data
1163
1164 def _corrupt_file_version_number(data):
1165     """ Scramble the file data -- the share file version number have one bit flipped or else
1166     will be changed to a random value."""
1167     return corrupt_field(data, 0x00, 4)
1168
1169 def _corrupt_size_of_file_data(data):
1170     """ Scramble the file data -- the field showing the size of the share data within the file
1171     will be set to one smaller. """
1172     return corrupt_field(data, 0x04, 4)
1173
1174 def _corrupt_sharedata_version_number(data):
1175     """ Scramble the file data -- the share data version number will have one bit flipped or
1176     else will be changed to a random value, but not 1 or 2."""
1177     return corrupt_field(data, 0x0c, 4)
1178     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1179     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1180     newsharevernum = sharevernum
1181     while newsharevernum in (1, 2):
1182         newsharevernum = random.randrange(0, 2**32)
1183     newsharevernumbytes = struct.pack(">L", newsharevernum)
1184     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1185
1186 def _corrupt_sharedata_version_number_to_plausible_version(data):
1187     """ Scramble the file data -- the share data version number will
1188     be changed to 2 if it is 1 or else to 1 if it is 2."""
1189     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1190     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1191     if sharevernum == 1:
1192         newsharevernum = 2
1193     else:
1194         newsharevernum = 1
1195     newsharevernumbytes = struct.pack(">L", newsharevernum)
1196     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1197
1198 def _corrupt_segment_size(data):
1199     """ Scramble the file data -- the field showing the size of the segment will have one
1200     bit flipped or else be changed to a random value. """
1201     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1202     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1203     if sharevernum == 1:
1204         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1205     else:
1206         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1207
1208 def _corrupt_size_of_sharedata(data):
1209     """ Scramble the file data -- the field showing the size of the data within the share
1210     data will have one bit flipped or else will be changed to a random value. """
1211     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1212     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1213     if sharevernum == 1:
1214         return corrupt_field(data, 0x0c+0x08, 4)
1215     else:
1216         return corrupt_field(data, 0x0c+0x0c, 8)
1217
1218 def _corrupt_offset_of_sharedata(data):
1219     """ Scramble the file data -- the field showing the offset of the data within the share
1220     data will have one bit flipped or else be changed to a random value. """
1221     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1222     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1223     if sharevernum == 1:
1224         return corrupt_field(data, 0x0c+0x0c, 4)
1225     else:
1226         return corrupt_field(data, 0x0c+0x14, 8)
1227
1228 def _corrupt_offset_of_ciphertext_hash_tree(data):
1229     """ Scramble the file data -- the field showing the offset of the ciphertext hash tree
1230     within the share data will have one bit flipped or else be changed to a random value.
1231     """
1232     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1233     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1234     if sharevernum == 1:
1235         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1236     else:
1237         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1238
1239 def _corrupt_offset_of_block_hashes(data):
1240     """ Scramble the file data -- the field showing the offset of the block hash tree within
1241     the share data will have one bit flipped or else will be changed to a random value. """
1242     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1243     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1244     if sharevernum == 1:
1245         return corrupt_field(data, 0x0c+0x18, 4)
1246     else:
1247         return corrupt_field(data, 0x0c+0x2c, 8)
1248
1249 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data):
1250     """ Scramble the file data -- the field showing the offset of the block hash tree within the
1251     share data will have a multiple of hash size subtracted from it, thus causing the downloader
1252     to download an incomplete crypttext hash tree."""
1253     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1254     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1255     if sharevernum == 1:
1256         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1257         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1258         newvalstr = struct.pack(">L", newval)
1259         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1260     else:
1261         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1262         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1263         newvalstr = struct.pack(">Q", newval)
1264         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1265
1266 def _corrupt_offset_of_share_hashes(data):
1267     """ Scramble the file data -- the field showing the offset of the share hash tree within
1268     the share data will have one bit flipped or else will be changed to a random value. """
1269     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1270     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1271     if sharevernum == 1:
1272         return corrupt_field(data, 0x0c+0x1c, 4)
1273     else:
1274         return corrupt_field(data, 0x0c+0x34, 8)
1275
1276 def _corrupt_offset_of_uri_extension(data):
1277     """ Scramble the file data -- the field showing the offset of the uri extension will
1278     have one bit flipped or else will be changed to a random value. """
1279     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1280     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1281     if sharevernum == 1:
1282         return corrupt_field(data, 0x0c+0x20, 4)
1283     else:
1284         return corrupt_field(data, 0x0c+0x3c, 8)
1285
1286 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1287     """ Scramble the file data -- the field showing the offset of the uri extension will be set
1288     to the size of the file minus 3.  This means when the client tries to read the length field
1289     from that location it will get a short read -- the result string will be only 3 bytes long,
1290     not the 4 or 8 bytes necessary to do a successful struct.unpack."""
1291     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1292     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1293     # The "-0x0c" in here is to skip the server-side header in the share file, which the client doesn't see when seeking and reading.
1294     if sharevernum == 1:
1295         if debug:
1296             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1297         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1298     else:
1299         if debug:
1300             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1301         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1302
1303 def _corrupt_mutable_share_data(data):
1304     prefix = data[:32]
1305     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
1306     data_offset = MutableShareFile.DATA_OFFSET
1307     sharetype = data[data_offset:data_offset+1]
1308     assert sharetype == "\x00", "non-SDMF mutable shares not supported"
1309     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
1310      ig_datalen, offsets) = unpack_header(data[data_offset:])
1311     assert version == 0, "this function only handles v0 SDMF files"
1312     start = data_offset + offsets["share_data"]
1313     length = data_offset + offsets["enc_privkey"] - start
1314     return corrupt_field(data, start, length)
1315
1316 def _corrupt_share_data(data):
1317     """ Scramble the file data -- the field containing the share data itself will have one
1318     bit flipped or else will be changed to a random value. """
1319     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1320     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1321     if sharevernum == 1:
1322         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1323
1324         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1325     else:
1326         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1327
1328         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1329
1330 def _corrupt_crypttext_hash_tree(data):
1331     """ Scramble the file data -- the field containing the crypttext hash tree will have one
1332     bit flipped or else will be changed to a random value.
1333     """
1334     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1335     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1336     if sharevernum == 1:
1337         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1338         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1339     else:
1340         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1341         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1342
1343     return corrupt_field(data, crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset)
1344
1345 def _corrupt_block_hashes(data):
1346     """ Scramble the file data -- the field containing the block hash tree will have one bit
1347     flipped or else will be changed to a random value.
1348     """
1349     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1350     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1351     if sharevernum == 1:
1352         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1353         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1354     else:
1355         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1356         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1357
1358     return corrupt_field(data, blockhashesoffset, sharehashesoffset-blockhashesoffset)
1359
1360 def _corrupt_share_hashes(data):
1361     """ Scramble the file data -- the field containing the share hash chain will have one
1362     bit flipped or else will be changed to a random value.
1363     """
1364     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1365     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1366     if sharevernum == 1:
1367         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1368         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1369     else:
1370         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1371         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1372
1373     return corrupt_field(data, sharehashesoffset, uriextoffset-sharehashesoffset)
1374
1375 def _corrupt_length_of_uri_extension(data):
1376     """ Scramble the file data -- the field showing the length of the uri extension will
1377     have one bit flipped or else will be changed to a random value. """
1378     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1379     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1380     if sharevernum == 1:
1381         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1382         return corrupt_field(data, uriextoffset, 4)
1383     else:
1384         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1385         return corrupt_field(data, uriextoffset, 8)
1386
1387 def _corrupt_uri_extension(data):
1388     """ Scramble the file data -- the field containing the uri extension will have one bit
1389     flipped or else will be changed to a random value. """
1390     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1391     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1392     if sharevernum == 1:
1393         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1394         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1395     else:
1396         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1397         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1398
1399     return corrupt_field(data, uriextoffset, uriextlen)