src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IPullProducer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.api import flushEventualQueue, fireEventually
   9 from allmydata import uri, dirnode, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IMutableFileNode, IImmutableFileNode, \
  12      FileTooLargeError, NotEnoughSharesError, ICheckable
  13 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  14      DeepCheckResults, DeepCheckAndRepairResults
  15 from allmydata.mutable.common import CorruptShareError
  16 from allmydata.mutable.layout import unpack_header
  17 from allmydata.storage.server import storage_index_to_dir
  18 from allmydata.storage.mutable import MutableShareFile
  19 from allmydata.util import hashutil, log, fileutil, pollmixin
  20 from allmydata.util.assertutil import precondition
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.stats import StatsGathererService
  23 from allmydata.key_generator import KeyGeneratorService
  24 import common_util as testutil
  25 from allmydata import immutable
  26
  27
  28 def flush_but_dont_ignore(res):
  29     d = flushEventualQueue()
  30     def _done(ignored):
  31         return res
  32     d.addCallback(_done)
  33     return d
  34
  35 class DummyProducer:
  36     implements(IPullProducer)
  37     def resumeProducing(self):
  38         pass
  39
  40 class FakeCHKFileNode:
  41     """I provide IImmutableFileNode, but all of my data is stored in a
  42     class-level dictionary."""
  43     implements(IImmutableFileNode)
  44     all_contents = {}
  45     bad_shares = {}
  46
  47     def __init__(self, filecap):
  48         precondition(isinstance(filecap, uri.CHKFileURI), filecap)
  49         self.my_uri = filecap
  50         self.storage_index = self.my_uri.storage_index
  51
  52     def get_uri(self):
  53         return self.my_uri.to_string()
  54     def get_readonly_uri(self):
  55         return self.my_uri.to_string()
  56     def get_cap(self):
  57         return self.my_uri
  58     def get_verify_cap(self):
  59         return self.my_uri.get_verify_cap()
  60     def get_repair_cap(self):
  61         return self.my_uri.get_verify_cap()
  62     def get_storage_index(self):
  63         return self.storage_index
  64
  65     def check(self, monitor, verify=False, add_lease=False):
  66         r = CheckResults(self.my_uri, self.storage_index)
  67         is_bad = self.bad_shares.get(self.storage_index, None)
  68         data = {}
  69         data["count-shares-needed"] = 3
  70         data["count-shares-expected"] = 10
  71         data["count-good-share-hosts"] = 10
  72         data["count-wrong-shares"] = 0
  73         nodeid = "\x00"*20
  74         data["list-corrupt-shares"] = []
  75         data["sharemap"] = {1: [nodeid]}
  76         data["servers-responding"] = [nodeid]
  77         data["count-recoverable-versions"] = 1
  78         data["count-unrecoverable-versions"] = 0
  79         if is_bad:
  80              r.set_healthy(False)
  81              r.set_recoverable(True)
  82              data["count-shares-good"] = 9
  83              data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
  84              r.problems = failure.Failure(CorruptShareError(is_bad))
  85         else:
  86              r.set_healthy(True)
  87              r.set_recoverable(True)
  88              data["count-shares-good"] = 10
  89              r.problems = []
  90         r.set_data(data)
  91         r.set_needs_rebalancing(False)
  92         return defer.succeed(r)
  93     def check_and_repair(self, monitor, verify=False, add_lease=False):
  94         d = self.check(verify)
  95         def _got(cr):
  96             r = CheckAndRepairResults(self.storage_index)
  97             r.pre_repair_results = r.post_repair_results = cr
  98             return r
  99         d.addCallback(_got)
 100         return d
 101
 102     def is_mutable(self):
 103         return False
 104     def is_readonly(self):
 105         return True
 106
 107     def get_size(self):
 108         try:
 109             data = self.all_contents[self.my_uri.to_string()]
 110         except KeyError, le:
 111             raise NotEnoughSharesError(le, 0, 3)
 112         return len(data)
 113
 114     def read(self, consumer, offset=0, size=None):
 115         # we don't bother to call registerProducer/unregisterProducer,
 116         # because it's a hassle to write a dummy Producer that does the right
 117         # thing (we have to make sure that DummyProducer.resumeProducing
 118         # writes the data into the consumer immediately, otherwise it will
 119         # loop forever).
 120
 121         d = defer.succeed(None)
 122         d.addCallback(self._read, consumer, offset, size)
 123         return d
 124
 125     def _read(self, ignored, consumer, offset, size):
 126         if self.my_uri.to_string() not in self.all_contents:
 127             raise NotEnoughSharesError(None, 0, 3)
 128         data = self.all_contents[self.my_uri.to_string()]
 129         start = offset
 130         if size is not None:
 131             end = offset + size
 132         else:
 133             end = len(data)
 134         consumer.write(data[start:end])
 135         return consumer
 136
 137 def make_chk_file_cap(size):
 138     return uri.CHKFileURI(key=os.urandom(16),
 139                           uri_extension_hash=os.urandom(32),
 140                           needed_shares=3,
 141                           total_shares=10,
 142                           size=size)
 143 def make_chk_file_uri(size):
 144     return make_chk_file_cap(size).to_string()
 145
 146 def create_chk_filenode(contents):
 147     filecap = make_chk_file_cap(len(contents))
 148     n = FakeCHKFileNode(filecap)
 149     FakeCHKFileNode.all_contents[filecap.to_string()] = contents
 150     return n
 151
 152
 153 class FakeMutableFileNode:
 154     """I provide IMutableFileNode, but all of my data is stored in a
 155     class-level dictionary."""
 156
 157     implements(IMutableFileNode, ICheckable)
 158     MUTABLE_SIZELIMIT = 10000
 159     all_contents = {}
 160     bad_shares = {}
 161
 162     def __init__(self, storage_broker, secret_holder,
 163                  default_encoding_parameters, history):
 164         self.init_from_cap(make_mutable_file_cap())
 165     def create(self, contents, key_generator=None, keysize=None):
 166         initial_contents = self._get_initial_contents(contents)
 167         if len(initial_contents) > self.MUTABLE_SIZELIMIT:
 168             raise FileTooLargeError("SDMF is limited to one segment, and "
 169                                     "%d > %d" % (len(initial_contents),
 170                                                  self.MUTABLE_SIZELIMIT))
 171         self.all_contents[self.storage_index] = initial_contents
 172         return defer.succeed(self)
 173     def _get_initial_contents(self, contents):
 174         if isinstance(contents, str):
 175             return contents
 176         if contents is None:
 177             return ""
 178         assert callable(contents), "%s should be callable, not %s" % \
 179                (contents, type(contents))
 180         return contents(self)
 181     def init_from_cap(self, filecap):
 182         assert isinstance(filecap, (uri.WriteableSSKFileURI,
 183                                     uri.ReadonlySSKFileURI))
 184         self.my_uri = filecap
 185         self.storage_index = self.my_uri.storage_index
 186         return self
 187     def get_cap(self):
 188         return self.my_uri
 189     def get_readcap(self):
 190         return self.my_uri.get_readonly()
 191     def get_uri(self):
 192         return self.my_uri.to_string()
 193     def get_readonly(self):
 194         return self.my_uri.get_readonly()
 195     def get_readonly_uri(self):
 196         return self.my_uri.get_readonly().to_string()
 197     def get_verify_cap(self):
 198         return self.my_uri.get_verify_cap()
 199     def is_readonly(self):
 200         return self.my_uri.is_readonly()
 201     def is_mutable(self):
 202         return self.my_uri.is_mutable()
 203     def get_writekey(self):
 204         return "\x00"*16
 205     def get_size(self):
 206         return len(self.all_contents[self.storage_index])
 207     def get_current_size(self):
 208         return self.get_size_of_best_version()
 209     def get_size_of_best_version(self):
 210         return defer.succeed(len(self.all_contents[self.storage_index]))
 211
 212     def get_storage_index(self):
 213         return self.storage_index
 214
 215     def check(self, monitor, verify=False, add_lease=False):
 216         r = CheckResults(self.my_uri, self.storage_index)
 217         is_bad = self.bad_shares.get(self.storage_index, None)
 218         data = {}
 219         data["count-shares-needed"] = 3
 220         data["count-shares-expected"] = 10
 221         data["count-good-share-hosts"] = 10
 222         data["count-wrong-shares"] = 0
 223         data["list-corrupt-shares"] = []
 224         nodeid = "\x00"*20
 225         data["sharemap"] = {"seq1-abcd-sh0": [nodeid]}
 226         data["servers-responding"] = [nodeid]
 227         data["count-recoverable-versions"] = 1
 228         data["count-unrecoverable-versions"] = 0
 229         if is_bad:
 230              r.set_healthy(False)
 231              r.set_recoverable(True)
 232              data["count-shares-good"] = 9
 233              r.problems = failure.Failure(CorruptShareError("peerid",
 234                                                             0, # shnum
 235                                                             is_bad))
 236         else:
 237              r.set_healthy(True)
 238              r.set_recoverable(True)
 239              data["count-shares-good"] = 10
 240              r.problems = []
 241         r.set_data(data)
 242         r.set_needs_rebalancing(False)
 243         return defer.succeed(r)
 244
 245     def check_and_repair(self, monitor, verify=False, add_lease=False):
 246         d = self.check(verify)
 247         def _got(cr):
 248             r = CheckAndRepairResults(self.storage_index)
 249             r.pre_repair_results = r.post_repair_results = cr
 250             return r
 251         d.addCallback(_got)
 252         return d
 253
 254     def deep_check(self, verify=False, add_lease=False):
 255         d = self.check(verify)
 256         def _done(r):
 257             dr = DeepCheckResults(self.storage_index)
 258             dr.add_check(r, [])
 259             return dr
 260         d.addCallback(_done)
 261         return d
 262
 263     def deep_check_and_repair(self, verify=False, add_lease=False):
 264         d = self.check_and_repair(verify)
 265         def _done(r):
 266             dr = DeepCheckAndRepairResults(self.storage_index)
 267             dr.add_check(r, [])
 268             return dr
 269         d.addCallback(_done)
 270         return d
 271
 272     def download_best_version(self):
 273         if self.storage_index not in self.all_contents:
 274             return defer.fail(NotEnoughSharesError(None, 0, 3))
 275         return defer.succeed(self.all_contents[self.storage_index])
 276     def overwrite(self, new_contents):
 277         if len(new_contents) > self.MUTABLE_SIZELIMIT:
 278             raise FileTooLargeError("SDMF is limited to one segment, and "
 279                                     "%d > %d" % (len(new_contents),
 280                                                  self.MUTABLE_SIZELIMIT))
 281         assert not self.is_readonly()
 282         self.all_contents[self.storage_index] = new_contents
 283         return defer.succeed(None)
 284     def modify(self, modifier):
 285         # this does not implement FileTooLargeError, but the real one does
 286         return defer.maybeDeferred(self._modify, modifier)
 287     def _modify(self, modifier):
 288         assert not self.is_readonly()
 289         old_contents = self.all_contents[self.storage_index]
 290         self.all_contents[self.storage_index] = modifier(old_contents, None, True)
 291         return None
 292
 293 def make_mutable_file_cap():
 294     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 295                                    fingerprint=os.urandom(32))
 296 def make_mutable_file_uri():
 297     return make_mutable_file_cap().to_string()
 298
 299 def make_verifier_uri():
 300     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 301                               fingerprint=os.urandom(32)).to_string()
 302
 303 class FakeDirectoryNode(dirnode.DirectoryNode):
 304     """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
 305     backing store, so it doesn't go to the grid. The child data is still
 306     encrypted and serialized, so this isn't useful for tests that want to
 307     look inside the dirnodes and check their contents.
 308     """
 309     filenode_class = FakeMutableFileNode
 310
 311 class LoggingServiceParent(service.MultiService):
 312     def log(self, *args, **kwargs):
 313         return log.msg(*args, **kwargs)
 314
 315
 316 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 317
 318     # SystemTestMixin tests tend to be a lot of work, and we have a few
 319     # buildslaves that are pretty slow, and every once in a while these tests
 320     # run up against the default 120 second timeout. So increase the default
 321     # timeout. Individual test cases can override this, of course.
 322     timeout = 300
 323
 324     def setUp(self):
 325         self.sparent = service.MultiService()
 326         self.sparent.startService()
 327
 328         self.stats_gatherer = None
 329         self.stats_gatherer_furl = None
 330         self.key_generator_svc = None
 331         self.key_generator_furl = None
 332
 333     def tearDown(self):
 334         log.msg("shutting down SystemTest services")
 335         d = self.sparent.stopService()
 336         d.addBoth(flush_but_dont_ignore)
 337         return d
 338
 339     def getdir(self, subdir):
 340         return os.path.join(self.basedir, subdir)
 341
 342     def add_service(self, s):
 343         s.setServiceParent(self.sparent)
 344         return s
 345
 346     def set_up_nodes(self, NUMCLIENTS=5,
 347                      use_stats_gatherer=False, use_key_generator=False):
 348         self.numclients = NUMCLIENTS
 349         iv_dir = self.getdir("introducer")
 350         if not os.path.isdir(iv_dir):
 351             fileutil.make_dirs(iv_dir)
 352             f = open(os.path.join(iv_dir, "webport"), "w")
 353             f.write("tcp:0:interface=127.0.0.1\n")
 354             f.close()
 355             if SYSTEM_TEST_CERTS:
 356                 os.mkdir(os.path.join(iv_dir, "private"))
 357                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 358                 f.write(SYSTEM_TEST_CERTS[0])
 359                 f.close()
 360         iv = IntroducerNode(basedir=iv_dir)
 361         self.introducer = self.add_service(iv)
 362         d = self.introducer.when_tub_ready()
 363         d.addCallback(self._get_introducer_web)
 364         if use_stats_gatherer:
 365             d.addCallback(self._set_up_stats_gatherer)
 366         if use_key_generator:
 367             d.addCallback(self._set_up_key_generator)
 368         d.addCallback(self._set_up_nodes_2)
 369         if use_stats_gatherer:
 370             d.addCallback(self._grab_stats)
 371         return d
 372
 373     def _get_introducer_web(self, res):
 374         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 375         self.introweb_url = f.read().strip()
 376         f.close()
 377
 378     def _set_up_stats_gatherer(self, res):
 379         statsdir = self.getdir("stats_gatherer")
 380         fileutil.make_dirs(statsdir)
 381         self.stats_gatherer_svc = StatsGathererService(statsdir)
 382         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 383         self.add_service(self.stats_gatherer_svc)
 384
 385         d = fireEventually()
 386         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 387         def check_for_furl():
 388             return os.path.exists(sgf)
 389         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 390         def get_furl(junk):
 391             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 392         d.addCallback(get_furl)
 393         return d
 394
 395     def _set_up_key_generator(self, res):
 396         kgsdir = self.getdir("key_generator")
 397         fileutil.make_dirs(kgsdir)
 398
 399         self.key_generator_svc = KeyGeneratorService(kgsdir,
 400                                                      display_furl=False,
 401                                                      default_key_size=522)
 402         self.key_generator_svc.key_generator.pool_size = 4
 403         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 404         self.add_service(self.key_generator_svc)
 405
 406         d = fireEventually()
 407         def check_for_furl():
 408             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 409         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 410         def get_furl(junk):
 411             kgf = os.path.join(kgsdir, 'key_generator.furl')
 412             self.key_generator_furl = file(kgf, 'rb').read().strip()
 413         d.addCallback(get_furl)
 414         return d
 415
 416     def _set_up_nodes_2(self, res):
 417         q = self.introducer
 418         self.introducer_furl = q.introducer_url
 419         self.clients = []
 420         basedirs = []
 421         for i in range(self.numclients):
 422             basedir = self.getdir("client%d" % i)
 423             basedirs.append(basedir)
 424             fileutil.make_dirs(os.path.join(basedir, "private"))
 425             if len(SYSTEM_TEST_CERTS) > (i+1):
 426                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 427                 f.write(SYSTEM_TEST_CERTS[i+1])
 428                 f.close()
 429
 430             def write(name, value):
 431                 open(os.path.join(basedir, name), "w").write(value+"\n")
 432             if i == 0:
 433                 # clients[0] runs a webserver and a helper, no key_generator
 434                 write("webport", "tcp:0:interface=127.0.0.1")
 435                 write("run_helper", "yes")
 436                 write("keepalive_timeout", "600")
 437             if i == 3:
 438                 # clients[3] runs a webserver and uses a helper, uses
 439                 # key_generator
 440                 write("webport", "tcp:0:interface=127.0.0.1")
 441                 write("disconnect_timeout", "1800")
 442                 if self.key_generator_furl:
 443                     kgf = "%s\n" % (self.key_generator_furl,)
 444                     write("key_generator.furl", kgf)
 445             write("introducer.furl", self.introducer_furl)
 446             if self.stats_gatherer_furl:
 447                 write("stats_gatherer.furl", self.stats_gatherer_furl)
 448
 449         # give subclasses a chance to append liens to the node's tahoe.cfg
 450         # files before they are launched.
 451         self._set_up_nodes_extra_config()
 452
 453         # start clients[0], wait for it's tub to be ready (at which point it
 454         # will have registered the helper furl).
 455         c = self.add_service(client.Client(basedir=basedirs[0]))
 456         self.clients.append(c)
 457         c.set_default_mutable_keysize(522)
 458         d = c.when_tub_ready()
 459         def _ready(res):
 460             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 461             helper_furl = f.read()
 462             f.close()
 463             self.helper_furl = helper_furl
 464             if self.numclients >= 4:
 465                 f = open(os.path.join(basedirs[3],"helper.furl"), "w")
 466                 f.write(helper_furl)
 467                 f.close()
 468
 469             # this starts the rest of the clients
 470             for i in range(1, self.numclients):
 471                 c = self.add_service(client.Client(basedir=basedirs[i]))
 472                 self.clients.append(c)
 473                 c.set_default_mutable_keysize(522)
 474             log.msg("STARTING")
 475             return self.wait_for_connections()
 476         d.addCallback(_ready)
 477         def _connected(res):
 478             log.msg("CONNECTED")
 479             # now find out where the web port was
 480             l = self.clients[0].getServiceNamed("webish").listener
 481             port = l._port.getHost().port
 482             self.webish_url = "http://localhost:%d/" % port
 483             if self.numclients >=4:
 484                 # and the helper-using webport
 485                 l = self.clients[3].getServiceNamed("webish").listener
 486                 port = l._port.getHost().port
 487                 self.helper_webish_url = "http://localhost:%d/" % port
 488         d.addCallback(_connected)
 489         return d
 490
 491     def _set_up_nodes_extra_config(self):
 492         # for overriding by subclasses
 493         pass
 494
 495     def _grab_stats(self, res):
 496         d = self.stats_gatherer.poll()
 497         return d
 498
 499     def bounce_client(self, num):
 500         c = self.clients[num]
 501         d = c.disownServiceParent()
 502         # I think windows requires a moment to let the connection really stop
 503         # and the port number made available for re-use. TODO: examine the
 504         # behavior, see if this is really the problem, see if we can do
 505         # better than blindly waiting for a second.
 506         d.addCallback(self.stall, 1.0)
 507         def _stopped(res):
 508             new_c = client.Client(basedir=self.getdir("client%d" % num))
 509             self.clients[num] = new_c
 510             new_c.set_default_mutable_keysize(522)
 511             self.add_service(new_c)
 512             return new_c.when_tub_ready()
 513         d.addCallback(_stopped)
 514         d.addCallback(lambda res: self.wait_for_connections())
 515         def _maybe_get_webport(res):
 516             if num == 0:
 517                 # now find out where the web port was
 518                 l = self.clients[0].getServiceNamed("webish").listener
 519                 port = l._port.getHost().port
 520                 self.webish_url = "http://localhost:%d/" % port
 521         d.addCallback(_maybe_get_webport)
 522         return d
 523
 524     def add_extra_node(self, client_num, helper_furl=None,
 525                        add_to_sparent=False):
 526         # usually this node is *not* parented to our self.sparent, so we can
 527         # shut it down separately from the rest, to exercise the
 528         # connection-lost code
 529         basedir = self.getdir("client%d" % client_num)
 530         if not os.path.isdir(basedir):
 531             fileutil.make_dirs(basedir)
 532         open(os.path.join(basedir, "introducer.furl"), "w").write(self.introducer_furl)
 533         if helper_furl:
 534             f = open(os.path.join(basedir, "helper.furl") ,"w")
 535             f.write(helper_furl+"\n")
 536             f.close()
 537
 538         c = client.Client(basedir=basedir)
 539         self.clients.append(c)
 540         c.set_default_mutable_keysize(522)
 541         self.numclients += 1
 542         if add_to_sparent:
 543             c.setServiceParent(self.sparent)
 544         else:
 545             c.startService()
 546         d = self.wait_for_connections()
 547         d.addCallback(lambda res: c)
 548         return d
 549
 550     def _check_connections(self):
 551         for c in self.clients:
 552             if not c.connected_to_introducer():
 553                 return False
 554             sb = c.get_storage_broker()
 555             if len(sb.get_all_servers()) != self.numclients:
 556                 return False
 557         return True
 558
 559     def wait_for_connections(self, ignored=None):
 560         # TODO: replace this with something that takes a list of peerids and
 561         # fires when they've all been heard from, instead of using a count
 562         # and a threshold
 563         return self.poll(self._check_connections, timeout=200)
 564
 565
 566 # our system test uses the same Tub certificates each time, to avoid the
 567 # overhead of key generation
 568 SYSTEM_TEST_CERTS = [
 569 """-----BEGIN CERTIFICATE-----
 570 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 571 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 572 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 573 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 574 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 575 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 576 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 577 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 578 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 579 -----END CERTIFICATE-----
 580 -----BEGIN RSA PRIVATE KEY-----
 581 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 582 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 583 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 584 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 585 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 586 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 587 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 588 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 589 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 590 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 591 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 592 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 593 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 594 -----END RSA PRIVATE KEY-----
 595 """, # 0
 596 """-----BEGIN CERTIFICATE-----
 597 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 598 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 599 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 600 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 601 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 602 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 603 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 604 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 605 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 606 -----END CERTIFICATE-----
 607 -----BEGIN RSA PRIVATE KEY-----
 608 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 609 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 610 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 611 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 612 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 613 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 614 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 615 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 616 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 617 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 618 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 619 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 620 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 621 -----END RSA PRIVATE KEY-----
 622 """, # 1
 623 """-----BEGIN CERTIFICATE-----
 624 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 625 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 626 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 627 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 628 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 629 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 630 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 631 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 632 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 633 -----END CERTIFICATE-----
 634 -----BEGIN RSA PRIVATE KEY-----
 635 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 636 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 637 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 638 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 639 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 640 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 641 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 642 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 643 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 644 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 645 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 646 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 647 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 648 -----END RSA PRIVATE KEY-----
 649 """, # 2
 650 """-----BEGIN CERTIFICATE-----
 651 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 652 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 653 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 654 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 655 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 656 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 657 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 658 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 659 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 660 -----END CERTIFICATE-----
 661 -----BEGIN RSA PRIVATE KEY-----
 662 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 663 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 664 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 665 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 666 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 667 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 668 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 669 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 670 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 671 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 672 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 673 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 674 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 675 -----END RSA PRIVATE KEY-----
 676 """, # 3
 677 """-----BEGIN CERTIFICATE-----
 678 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 679 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 680 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 681 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 682 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 683 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 684 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 685 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 686 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 687 -----END CERTIFICATE-----
 688 -----BEGIN RSA PRIVATE KEY-----
 689 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 690 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 691 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 692 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 693 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 694 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 695 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 696 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 697 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 698 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 699 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 700 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 701 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 702 -----END RSA PRIVATE KEY-----
 703 """, # 4
 704 """-----BEGIN CERTIFICATE-----
 705 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 706 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 707 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 708 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 709 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 710 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 711 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 712 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 713 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 714 -----END CERTIFICATE-----
 715 -----BEGIN RSA PRIVATE KEY-----
 716 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 717 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 718 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 719 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 720 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 721 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 722 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 723 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 724 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 725 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 726 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 727 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 728 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 729 -----END RSA PRIVATE KEY-----
 730 """, # 5
 731 """-----BEGIN CERTIFICATE-----
 732 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 733 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 734 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 735 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 736 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 737 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 738 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 739 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 740 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 741 -----END CERTIFICATE-----
 742 -----BEGIN RSA PRIVATE KEY-----
 743 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 744 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 745 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 746 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 747 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 748 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 749 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 750 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 751 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 752 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 753 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 754 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 755 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 756 -----END RSA PRIVATE KEY-----
 757 """, #6
 758 """-----BEGIN CERTIFICATE-----
 759 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 760 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 761 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 762 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 763 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 764 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 765 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 766 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 767 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 768 -----END CERTIFICATE-----
 769 -----BEGIN RSA PRIVATE KEY-----
 770 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 771 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 772 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 773 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 774 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 775 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 776 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 777 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 778 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 779 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 780 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 781 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 782 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 783 -----END RSA PRIVATE KEY-----
 784 """, #7
 785 """-----BEGIN CERTIFICATE-----
 786 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 787 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 788 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 789 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 790 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 791 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 792 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 793 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 794 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 795 -----END CERTIFICATE-----
 796 -----BEGIN RSA PRIVATE KEY-----
 797 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 798 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 799 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 800 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 801 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 802 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 803 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 804 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 805 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 806 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 807 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 808 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 809 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 810 -----END RSA PRIVATE KEY-----
 811 """, #8
 812 """-----BEGIN CERTIFICATE-----
 813 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 814 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 815 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 816 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 817 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 818 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 819 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 820 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 821 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 822 -----END CERTIFICATE-----
 823 -----BEGIN RSA PRIVATE KEY-----
 824 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 825 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 826 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 827 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 828 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 829 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 830 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 831 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 832 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 833 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 834 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 835 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 836 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 837 -----END RSA PRIVATE KEY-----
 838 """, #9
 839 """-----BEGIN CERTIFICATE-----
 840 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 841 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 842 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 843 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 844 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 845 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 846 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 847 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 848 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 849 -----END CERTIFICATE-----
 850 -----BEGIN RSA PRIVATE KEY-----
 851 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 852 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 853 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 854 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 855 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 856 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 857 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
 858 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
 859 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
 860 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
 861 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
 862 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
 863 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
 864 -----END RSA PRIVATE KEY-----
 865 """, #10
 866 """-----BEGIN CERTIFICATE-----
 867 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 868 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 869 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
 870 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
 871 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
 872 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 873 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
 874 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
 875 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
 876 -----END CERTIFICATE-----
 877 -----BEGIN RSA PRIVATE KEY-----
 878 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
 879 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
 880 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
 881 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
 882 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
 883 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
 884 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
 885 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
 886 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
 887 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
 888 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
 889 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
 890 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
 891 -----END RSA PRIVATE KEY-----
 892 """, #11
 893 ]
 894
 895 # To disable the pre-computed tub certs, uncomment this line.
 896 #SYSTEM_TEST_CERTS = []
 897
 898 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
 899
 900 class ShareManglingMixin(SystemTestMixin):
 901
 902     def setUp(self):
 903         # Set self.basedir to a temp dir which has the name of the current
 904         # test method in its name.
 905         self.basedir = self.mktemp()
 906
 907         d = defer.maybeDeferred(SystemTestMixin.setUp, self)
 908         d.addCallback(lambda x: self.set_up_nodes())
 909
 910         def _upload_a_file(ignored):
 911             cl0 = self.clients[0]
 912             # We need multiple segments to test crypttext hash trees that are
 913             # non-trivial (i.e. they have more than just one hash in them).
 914             cl0.DEFAULT_ENCODING_PARAMETERS['max_segment_size'] = 12
 915             d2 = cl0.upload(immutable.upload.Data(TEST_DATA, convergence=""))
 916             def _after_upload(u):
 917                 filecap = u.uri
 918                 self.n = self.clients[1].create_node_from_uri(filecap)
 919                 self.uri = uri.CHKFileURI.init_from_string(filecap)
 920                 return cl0.create_node_from_uri(filecap)
 921             d2.addCallback(_after_upload)
 922             return d2
 923         d.addCallback(_upload_a_file)
 924
 925         def _stash_it(filenode):
 926             self.filenode = filenode
 927         d.addCallback(_stash_it)
 928         return d
 929
 930     def find_shares(self, unused=None):
 931         """Locate shares on disk. Returns a dict that maps
 932         (clientnum,sharenum) to a string that contains the share container
 933         (copied directly from the disk, containing leases etc). You can
 934         modify this dict and then call replace_shares() to modify the shares.
 935         """
 936         shares = {} # k: (i, sharenum), v: data
 937
 938         for i, c in enumerate(self.clients):
 939             sharedir = c.getServiceNamed("storage").sharedir
 940             for (dirp, dirns, fns) in os.walk(sharedir):
 941                 for fn in fns:
 942                     try:
 943                         sharenum = int(fn)
 944                     except TypeError:
 945                         # Whoops, I guess that's not a share file then.
 946                         pass
 947                     else:
 948                         data = open(os.path.join(sharedir, dirp, fn), "rb").read()
 949                         shares[(i, sharenum)] = data
 950
 951         return shares
 952
 953     def replace_shares(self, newshares, storage_index):
 954         """Replace shares on disk. Takes a dictionary in the same form
 955         as find_shares() returns."""
 956
 957         for i, c in enumerate(self.clients):
 958             sharedir = c.getServiceNamed("storage").sharedir
 959             for (dirp, dirns, fns) in os.walk(sharedir):
 960                 for fn in fns:
 961                     try:
 962                         sharenum = int(fn)
 963                     except TypeError:
 964                         # Whoops, I guess that's not a share file then.
 965                         pass
 966                     else:
 967                         pathtosharefile = os.path.join(sharedir, dirp, fn)
 968                         os.unlink(pathtosharefile)
 969             for ((clientnum, sharenum), newdata) in newshares.iteritems():
 970                 if clientnum == i:
 971                     fullsharedirp=os.path.join(sharedir, storage_index_to_dir(storage_index))
 972                     fileutil.make_dirs(fullsharedirp)
 973                     wf = open(os.path.join(fullsharedirp, str(sharenum)), "wb")
 974                     wf.write(newdata)
 975                     wf.close()
 976
 977     def _delete_a_share(self, unused=None, sharenum=None):
 978         """ Delete one share. """
 979
 980         shares = self.find_shares()
 981         ks = shares.keys()
 982         if sharenum is not None:
 983             k = [ key for key in shares.keys() if key[1] == sharenum ][0]
 984         else:
 985             k = random.choice(ks)
 986         del shares[k]
 987         self.replace_shares(shares, storage_index=self.uri.storage_index)
 988
 989         return unused
 990
 991     def _corrupt_a_share(self, unused, corruptor_func, sharenum):
 992         shares = self.find_shares()
 993         ks = [ key for key in shares.keys() if key[1] == sharenum ]
 994         assert ks, (shares.keys(), sharenum)
 995         k = ks[0]
 996         shares[k] = corruptor_func(shares[k])
 997         self.replace_shares(shares, storage_index=self.uri.storage_index)
 998         return corruptor_func
 999
1000     def _corrupt_all_shares(self, unused, corruptor_func):
1001         """ All shares on disk will be corrupted by corruptor_func. """
1002         shares = self.find_shares()
1003         for k in shares.keys():
1004             self._corrupt_a_share(unused, corruptor_func, k[1])
1005         return corruptor_func
1006
1007     def _corrupt_a_random_share(self, unused, corruptor_func):
1008         """ Exactly one share on disk will be corrupted by corruptor_func. """
1009         shares = self.find_shares()
1010         ks = shares.keys()
1011         k = random.choice(ks)
1012         self._corrupt_a_share(unused, corruptor_func, k[1])
1013         return k[1]
1014
1015     def _count_reads(self):
1016         sum_of_read_counts = 0
1017         for thisclient in self.clients:
1018             counters = thisclient.stats_provider.get_stats()['counters']
1019             sum_of_read_counts += counters.get('storage_server.read', 0)
1020         return sum_of_read_counts
1021
1022     def _count_allocates(self):
1023         sum_of_allocate_counts = 0
1024         for thisclient in self.clients:
1025             counters = thisclient.stats_provider.get_stats()['counters']
1026             sum_of_allocate_counts += counters.get('storage_server.allocate', 0)
1027         return sum_of_allocate_counts
1028
1029     def _count_writes(self):
1030         sum_of_write_counts = 0
1031         for thisclient in self.clients:
1032             counters = thisclient.stats_provider.get_stats()['counters']
1033             sum_of_write_counts += counters.get('storage_server.write', 0)
1034         return sum_of_write_counts
1035
1036     def _download_and_check_plaintext(self, unused=None):
1037         d = download_to_data(self.n)
1038         def _after_download(result):
1039             self.failUnlessEqual(result, TEST_DATA)
1040         d.addCallback(_after_download)
1041         return d
1042
1043 class ShouldFailMixin:
1044     def shouldFail(self, expected_failure, which, substring,
1045                    callable, *args, **kwargs):
1046         """Assert that a function call raises some exception. This is a
1047         Deferred-friendly version of TestCase.assertRaises() .
1048
1049         Suppose you want to verify the following function:
1050
1051          def broken(a, b, c):
1052              if a < 0:
1053                  raise TypeError('a must not be negative')
1054              return defer.succeed(b+c)
1055
1056         You can use:
1057             d = self.shouldFail(TypeError, 'test name',
1058                                 'a must not be negative',
1059                                 broken, -4, 5, c=12)
1060         in your test method. The 'test name' string will be included in the
1061         error message, if any, because Deferred chains frequently make it
1062         difficult to tell which assertion was tripped.
1063
1064         The substring= argument, if not None, must appear inside the
1065         stringified Failure, or the test will fail.
1066         """
1067
1068         assert substring is None or isinstance(substring, str)
1069         d = defer.maybeDeferred(callable, *args, **kwargs)
1070         def done(res):
1071             if isinstance(res, failure.Failure):
1072                 res.trap(expected_failure)
1073                 if substring:
1074                     self.failUnless(substring in str(res),
1075                                     "substring '%s' not in '%s'"
1076                                     % (substring, str(res)))
1077             else:
1078                 self.fail("%s was supposed to raise %s, not get '%s'" %
1079                           (which, expected_failure, res))
1080         d.addBoth(done)
1081         return d
1082
1083 class WebErrorMixin:
1084     def explain_web_error(self, f):
1085         # an error on the server side causes the client-side getPage() to
1086         # return a failure(t.web.error.Error), and its str() doesn't show the
1087         # response body, which is where the useful information lives. Attach
1088         # this method as an errback handler, and it will reveal the hidden
1089         # message.
1090         f.trap(WebError)
1091         print "Web Error:", f.value, ":", f.value.response
1092         return f
1093
1094     def _shouldHTTPError(self, res, which, validator):
1095         if isinstance(res, failure.Failure):
1096             res.trap(WebError)
1097             return validator(res)
1098         else:
1099             self.fail("%s was supposed to Error, not get '%s'" % (which, res))
1100
1101     def shouldHTTPError(self, which,
1102                         code=None, substring=None, response_substring=None,
1103                         callable=None, *args, **kwargs):
1104         # returns a Deferred with the response body
1105         assert substring is None or isinstance(substring, str)
1106         assert callable
1107         def _validate(f):
1108             if code is not None:
1109                 self.failUnlessEqual(f.value.status, str(code))
1110             if substring:
1111                 code_string = str(f)
1112                 self.failUnless(substring in code_string,
1113                                 "substring '%s' not in '%s'"
1114                                 % (substring, code_string))
1115             response_body = f.value.response
1116             if response_substring:
1117                 self.failUnless(response_substring in response_body,
1118                                 "response substring '%s' not in '%s'"
1119                                 % (response_substring, response_body))
1120             return response_body
1121         d = defer.maybeDeferred(callable, *args, **kwargs)
1122         d.addBoth(self._shouldHTTPError, which, _validate)
1123         return d
1124
1125 class ErrorMixin(WebErrorMixin):
1126     def explain_error(self, f):
1127         if f.check(defer.FirstError):
1128             print "First Error:", f.value.subFailure
1129         return f
1130
1131 def corrupt_field(data, offset, size, debug=False):
1132     if random.random() < 0.5:
1133         newdata = testutil.flip_one_bit(data, offset, size)
1134         if debug:
1135             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1136         return newdata
1137     else:
1138         newval = testutil.insecurerandstr(size)
1139         if debug:
1140             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1141         return data[:offset]+newval+data[offset+size:]
1142
1143 def _corrupt_nothing(data, debug=False):
1144     """Leave the data pristine. """
1145     return data
1146
1147 def _corrupt_file_version_number(data, debug=False):
1148     """Scramble the file data -- the share file version number have one bit
1149     flipped or else will be changed to a random value."""
1150     return corrupt_field(data, 0x00, 4)
1151
1152 def _corrupt_size_of_file_data(data, debug=False):
1153     """Scramble the file data -- the field showing the size of the share data
1154     within the file will be set to one smaller."""
1155     return corrupt_field(data, 0x04, 4)
1156
1157 def _corrupt_sharedata_version_number(data, debug=False):
1158     """Scramble the file data -- the share data version number will have one
1159     bit flipped or else will be changed to a random value, but not 1 or 2."""
1160     return corrupt_field(data, 0x0c, 4)
1161     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1162     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1163     newsharevernum = sharevernum
1164     while newsharevernum in (1, 2):
1165         newsharevernum = random.randrange(0, 2**32)
1166     newsharevernumbytes = struct.pack(">L", newsharevernum)
1167     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1168
1169 def _corrupt_sharedata_version_number_to_plausible_version(data, debug=False):
1170     """Scramble the file data -- the share data version number will be
1171     changed to 2 if it is 1 or else to 1 if it is 2."""
1172     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1173     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1174     if sharevernum == 1:
1175         newsharevernum = 2
1176     else:
1177         newsharevernum = 1
1178     newsharevernumbytes = struct.pack(">L", newsharevernum)
1179     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1180
1181 def _corrupt_segment_size(data, debug=False):
1182     """Scramble the file data -- the field showing the size of the segment
1183     will have one bit flipped or else be changed to a random value."""
1184     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1185     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1186     if sharevernum == 1:
1187         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1188     else:
1189         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1190
1191 def _corrupt_size_of_sharedata(data, debug=False):
1192     """Scramble the file data -- the field showing the size of the data
1193     within the share data will have one bit flipped or else will be changed
1194     to a random value."""
1195     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1196     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1197     if sharevernum == 1:
1198         return corrupt_field(data, 0x0c+0x08, 4)
1199     else:
1200         return corrupt_field(data, 0x0c+0x0c, 8)
1201
1202 def _corrupt_offset_of_sharedata(data, debug=False):
1203     """Scramble the file data -- the field showing the offset of the data
1204     within the share data will have one bit flipped or else be changed to a
1205     random value."""
1206     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1207     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1208     if sharevernum == 1:
1209         return corrupt_field(data, 0x0c+0x0c, 4)
1210     else:
1211         return corrupt_field(data, 0x0c+0x14, 8)
1212
1213 def _corrupt_offset_of_ciphertext_hash_tree(data, debug=False):
1214     """Scramble the file data -- the field showing the offset of the
1215     ciphertext hash tree within the share data will have one bit flipped or
1216     else be changed to a random value.
1217     """
1218     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1219     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1220     if sharevernum == 1:
1221         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1222     else:
1223         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1224
1225 def _corrupt_offset_of_block_hashes(data, debug=False):
1226     """Scramble the file data -- the field showing the offset of the block
1227     hash tree within the share data will have one bit flipped or else will be
1228     changed to a random value."""
1229     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1230     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1231     if sharevernum == 1:
1232         return corrupt_field(data, 0x0c+0x18, 4)
1233     else:
1234         return corrupt_field(data, 0x0c+0x2c, 8)
1235
1236 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data, debug=False):
1237     """Scramble the file data -- the field showing the offset of the block
1238     hash tree within the share data will have a multiple of hash size
1239     subtracted from it, thus causing the downloader to download an incomplete
1240     crypttext hash tree."""
1241     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1242     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1243     if sharevernum == 1:
1244         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1245         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1246         newvalstr = struct.pack(">L", newval)
1247         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1248     else:
1249         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1250         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1251         newvalstr = struct.pack(">Q", newval)
1252         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1253
1254 def _corrupt_offset_of_share_hashes(data, debug=False):
1255     """Scramble the file data -- the field showing the offset of the share
1256     hash tree within the share data will have one bit flipped or else will be
1257     changed to a random value."""
1258     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1259     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1260     if sharevernum == 1:
1261         return corrupt_field(data, 0x0c+0x1c, 4)
1262     else:
1263         return corrupt_field(data, 0x0c+0x34, 8)
1264
1265 def _corrupt_offset_of_uri_extension(data, debug=False):
1266     """Scramble the file data -- the field showing the offset of the uri
1267     extension will have one bit flipped or else will be changed to a random
1268     value."""
1269     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1270     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1271     if sharevernum == 1:
1272         return corrupt_field(data, 0x0c+0x20, 4)
1273     else:
1274         return corrupt_field(data, 0x0c+0x3c, 8)
1275
1276 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1277     """Scramble the file data -- the field showing the offset of the uri
1278     extension will be set to the size of the file minus 3. This means when
1279     the client tries to read the length field from that location it will get
1280     a short read -- the result string will be only 3 bytes long, not the 4 or
1281     8 bytes necessary to do a successful struct.unpack."""
1282     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1283     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1284     # The "-0x0c" in here is to skip the server-side header in the share
1285     # file, which the client doesn't see when seeking and reading.
1286     if sharevernum == 1:
1287         if debug:
1288             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1289         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1290     else:
1291         if debug:
1292             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1293         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1294
1295 def _corrupt_mutable_share_data(data, debug=False):
1296     prefix = data[:32]
1297     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
1298     data_offset = MutableShareFile.DATA_OFFSET
1299     sharetype = data[data_offset:data_offset+1]
1300     assert sharetype == "\x00", "non-SDMF mutable shares not supported"
1301     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
1302      ig_datalen, offsets) = unpack_header(data[data_offset:])
1303     assert version == 0, "this function only handles v0 SDMF files"
1304     start = data_offset + offsets["share_data"]
1305     length = data_offset + offsets["enc_privkey"] - start
1306     return corrupt_field(data, start, length)
1307
1308 def _corrupt_share_data(data, debug=False):
1309     """Scramble the file data -- the field containing the share data itself
1310     will have one bit flipped or else will be changed to a random value."""
1311     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1312     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1313     if sharevernum == 1:
1314         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1315
1316         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1317     else:
1318         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1319
1320         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1321
1322 def _corrupt_crypttext_hash_tree(data, debug=False):
1323     """Scramble the file data -- the field containing the crypttext hash tree
1324     will have one bit flipped or else will be changed to a random value.
1325     """
1326     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1327     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1328     if sharevernum == 1:
1329         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1330         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1331     else:
1332         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1333         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1334
1335     return corrupt_field(data, 0x0c+crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset, debug=debug)
1336
1337 def _corrupt_crypttext_hash_tree_byte_x221(data, debug=False):
1338     """Scramble the file data -- the field containing the crypttext hash tree
1339     will have the 7th bit of the 9th byte flipped.
1340     """
1341     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1342     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1343     if sharevernum == 1:
1344         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1345         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1346     else:
1347         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1348         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1349
1350     if debug:
1351         log.msg("original data: %r" % (data,))
1352     return data[:0x0c+0x221] + chr(ord(data[0x0c+0x221])^0x02) + data[0x0c+0x2210+1:]
1353
1354 def _corrupt_block_hashes(data, debug=False):
1355     """Scramble the file data -- the field containing the block hash tree
1356     will have one bit flipped or else will be changed to a random value.
1357     """
1358     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1359     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1360     if sharevernum == 1:
1361         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1362         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1363     else:
1364         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1365         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1366
1367     return corrupt_field(data, 0x0c+blockhashesoffset, sharehashesoffset-blockhashesoffset)
1368
1369 def _corrupt_share_hashes(data, debug=False):
1370     """Scramble the file data -- the field containing the share hash chain
1371     will have one bit flipped or else will be changed to a random value.
1372     """
1373     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1374     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1375     if sharevernum == 1:
1376         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1377         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1378     else:
1379         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1380         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1381
1382     return corrupt_field(data, 0x0c+sharehashesoffset, uriextoffset-sharehashesoffset)
1383
1384 def _corrupt_length_of_uri_extension(data, debug=False):
1385     """Scramble the file data -- the field showing the length of the uri
1386     extension will have one bit flipped or else will be changed to a random
1387     value."""
1388     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1389     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1390     if sharevernum == 1:
1391         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1392         return corrupt_field(data, uriextoffset, 4)
1393     else:
1394         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1395         return corrupt_field(data, 0x0c+uriextoffset, 8)
1396
1397 def _corrupt_uri_extension(data, debug=False):
1398     """Scramble the file data -- the field containing the uri extension will
1399     have one bit flipped or else will be changed to a random value."""
1400     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1401     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1402     if sharevernum == 1:
1403         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1404         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1405     else:
1406         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1407         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1408
1409     return corrupt_field(data, 0x0c+uriextoffset, uriextlen)