src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IPullProducer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.api import flushEventualQueue, fireEventually
   9 from allmydata import uri, dirnode, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IMutableFileNode, IImmutableFileNode, \
  12      FileTooLargeError, NotEnoughSharesError, ICheckable
  13 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  14      DeepCheckResults, DeepCheckAndRepairResults
  15 from allmydata.mutable.common import CorruptShareError
  16 from allmydata.mutable.layout import unpack_header
  17 from allmydata.storage.server import storage_index_to_dir
  18 from allmydata.storage.mutable import MutableShareFile
  19 from allmydata.util import hashutil, log, fileutil, pollmixin
  20 from allmydata.util.assertutil import precondition
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.stats import StatsGathererService
  23 from allmydata.key_generator import KeyGeneratorService
  24 import common_util as testutil
  25 from allmydata import immutable
  26
  27
  28 def flush_but_dont_ignore(res):
  29     d = flushEventualQueue()
  30     def _done(ignored):
  31         return res
  32     d.addCallback(_done)
  33     return d
  34
  35 class DummyProducer:
  36     implements(IPullProducer)
  37     def resumeProducing(self):
  38         pass
  39
  40 class FakeCHKFileNode:
  41     """I provide IImmutableFileNode, but all of my data is stored in a
  42     class-level dictionary."""
  43     implements(IImmutableFileNode)
  44     all_contents = {}
  45     bad_shares = {}
  46
  47     def __init__(self, filecap):
  48         precondition(isinstance(filecap, uri.CHKFileURI), filecap)
  49         self.my_uri = filecap
  50         self.storage_index = self.my_uri.storage_index
  51
  52     def get_uri(self):
  53         return self.my_uri.to_string()
  54     def get_write_uri(self):
  55         return None
  56     def get_readonly_uri(self):
  57         return self.my_uri.to_string()
  58     def get_cap(self):
  59         return self.my_uri
  60     def get_verify_cap(self):
  61         return self.my_uri.get_verify_cap()
  62     def get_repair_cap(self):
  63         return self.my_uri.get_verify_cap()
  64     def get_storage_index(self):
  65         return self.storage_index
  66
  67     def check(self, monitor, verify=False, add_lease=False):
  68         r = CheckResults(self.my_uri, self.storage_index)
  69         is_bad = self.bad_shares.get(self.storage_index, None)
  70         data = {}
  71         data["count-shares-needed"] = 3
  72         data["count-shares-expected"] = 10
  73         data["count-good-share-hosts"] = 10
  74         data["count-wrong-shares"] = 0
  75         nodeid = "\x00"*20
  76         data["list-corrupt-shares"] = []
  77         data["sharemap"] = {1: [nodeid]}
  78         data["servers-responding"] = [nodeid]
  79         data["count-recoverable-versions"] = 1
  80         data["count-unrecoverable-versions"] = 0
  81         if is_bad:
  82              r.set_healthy(False)
  83              r.set_recoverable(True)
  84              data["count-shares-good"] = 9
  85              data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
  86              r.problems = failure.Failure(CorruptShareError(is_bad))
  87         else:
  88              r.set_healthy(True)
  89              r.set_recoverable(True)
  90              data["count-shares-good"] = 10
  91              r.problems = []
  92         r.set_data(data)
  93         r.set_needs_rebalancing(False)
  94         return defer.succeed(r)
  95     def check_and_repair(self, monitor, verify=False, add_lease=False):
  96         d = self.check(verify)
  97         def _got(cr):
  98             r = CheckAndRepairResults(self.storage_index)
  99             r.pre_repair_results = r.post_repair_results = cr
 100             return r
 101         d.addCallback(_got)
 102         return d
 103
 104     def is_mutable(self):
 105         return False
 106     def is_readonly(self):
 107         return True
 108     def is_unknown(self):
 109         return False
 110     def is_allowed_in_immutable_directory(self):
 111         return True
 112     def raise_error(self):
 113         pass
 114
 115     def get_size(self):
 116         try:
 117             data = self.all_contents[self.my_uri.to_string()]
 118         except KeyError, le:
 119             raise NotEnoughSharesError(le, 0, 3)
 120         return len(data)
 121
 122     def read(self, consumer, offset=0, size=None):
 123         # we don't bother to call registerProducer/unregisterProducer,
 124         # because it's a hassle to write a dummy Producer that does the right
 125         # thing (we have to make sure that DummyProducer.resumeProducing
 126         # writes the data into the consumer immediately, otherwise it will
 127         # loop forever).
 128
 129         d = defer.succeed(None)
 130         d.addCallback(self._read, consumer, offset, size)
 131         return d
 132
 133     def _read(self, ignored, consumer, offset, size):
 134         if self.my_uri.to_string() not in self.all_contents:
 135             raise NotEnoughSharesError(None, 0, 3)
 136         data = self.all_contents[self.my_uri.to_string()]
 137         start = offset
 138         if size is not None:
 139             end = offset + size
 140         else:
 141             end = len(data)
 142         consumer.write(data[start:end])
 143         return consumer
 144
 145 def make_chk_file_cap(size):
 146     return uri.CHKFileURI(key=os.urandom(16),
 147                           uri_extension_hash=os.urandom(32),
 148                           needed_shares=3,
 149                           total_shares=10,
 150                           size=size)
 151 def make_chk_file_uri(size):
 152     return make_chk_file_cap(size).to_string()
 153
 154 def create_chk_filenode(contents):
 155     filecap = make_chk_file_cap(len(contents))
 156     n = FakeCHKFileNode(filecap)
 157     FakeCHKFileNode.all_contents[filecap.to_string()] = contents
 158     return n
 159
 160
 161 class FakeMutableFileNode:
 162     """I provide IMutableFileNode, but all of my data is stored in a
 163     class-level dictionary."""
 164
 165     implements(IMutableFileNode, ICheckable)
 166     MUTABLE_SIZELIMIT = 10000
 167     all_contents = {}
 168     bad_shares = {}
 169
 170     def __init__(self, storage_broker, secret_holder,
 171                  default_encoding_parameters, history):
 172         self.init_from_cap(make_mutable_file_cap())
 173     def create(self, contents, key_generator=None, keysize=None):
 174         initial_contents = self._get_initial_contents(contents)
 175         if len(initial_contents) > self.MUTABLE_SIZELIMIT:
 176             raise FileTooLargeError("SDMF is limited to one segment, and "
 177                                     "%d > %d" % (len(initial_contents),
 178                                                  self.MUTABLE_SIZELIMIT))
 179         self.all_contents[self.storage_index] = initial_contents
 180         return defer.succeed(self)
 181     def _get_initial_contents(self, contents):
 182         if isinstance(contents, str):
 183             return contents
 184         if contents is None:
 185             return ""
 186         assert callable(contents), "%s should be callable, not %s" % \
 187                (contents, type(contents))
 188         return contents(self)
 189     def init_from_cap(self, filecap):
 190         assert isinstance(filecap, (uri.WriteableSSKFileURI,
 191                                     uri.ReadonlySSKFileURI))
 192         self.my_uri = filecap
 193         self.storage_index = self.my_uri.storage_index
 194         return self
 195     def get_cap(self):
 196         return self.my_uri
 197     def get_readcap(self):
 198         return self.my_uri.get_readonly()
 199     def get_uri(self):
 200         return self.my_uri.to_string()
 201     def get_write_uri(self):
 202         if self.is_readonly():
 203             return None
 204         return self.my_uri.to_string()
 205     def get_readonly(self):
 206         return self.my_uri.get_readonly()
 207     def get_readonly_uri(self):
 208         return self.my_uri.get_readonly().to_string()
 209     def get_verify_cap(self):
 210         return self.my_uri.get_verify_cap()
 211     def is_readonly(self):
 212         return self.my_uri.is_readonly()
 213     def is_mutable(self):
 214         return self.my_uri.is_mutable()
 215     def is_unknown(self):
 216         return False
 217     def is_allowed_in_immutable_directory(self):
 218         return not self.my_uri.is_mutable()
 219     def raise_error(self):
 220         pass
 221     def get_writekey(self):
 222         return "\x00"*16
 223     def get_size(self):
 224         return len(self.all_contents[self.storage_index])
 225     def get_current_size(self):
 226         return self.get_size_of_best_version()
 227     def get_size_of_best_version(self):
 228         return defer.succeed(len(self.all_contents[self.storage_index]))
 229
 230     def get_storage_index(self):
 231         return self.storage_index
 232
 233     def check(self, monitor, verify=False, add_lease=False):
 234         r = CheckResults(self.my_uri, self.storage_index)
 235         is_bad = self.bad_shares.get(self.storage_index, None)
 236         data = {}
 237         data["count-shares-needed"] = 3
 238         data["count-shares-expected"] = 10
 239         data["count-good-share-hosts"] = 10
 240         data["count-wrong-shares"] = 0
 241         data["list-corrupt-shares"] = []
 242         nodeid = "\x00"*20
 243         data["sharemap"] = {"seq1-abcd-sh0": [nodeid]}
 244         data["servers-responding"] = [nodeid]
 245         data["count-recoverable-versions"] = 1
 246         data["count-unrecoverable-versions"] = 0
 247         if is_bad:
 248              r.set_healthy(False)
 249              r.set_recoverable(True)
 250              data["count-shares-good"] = 9
 251              r.problems = failure.Failure(CorruptShareError("peerid",
 252                                                             0, # shnum
 253                                                             is_bad))
 254         else:
 255              r.set_healthy(True)
 256              r.set_recoverable(True)
 257              data["count-shares-good"] = 10
 258              r.problems = []
 259         r.set_data(data)
 260         r.set_needs_rebalancing(False)
 261         return defer.succeed(r)
 262
 263     def check_and_repair(self, monitor, verify=False, add_lease=False):
 264         d = self.check(verify)
 265         def _got(cr):
 266             r = CheckAndRepairResults(self.storage_index)
 267             r.pre_repair_results = r.post_repair_results = cr
 268             return r
 269         d.addCallback(_got)
 270         return d
 271
 272     def deep_check(self, verify=False, add_lease=False):
 273         d = self.check(verify)
 274         def _done(r):
 275             dr = DeepCheckResults(self.storage_index)
 276             dr.add_check(r, [])
 277             return dr
 278         d.addCallback(_done)
 279         return d
 280
 281     def deep_check_and_repair(self, verify=False, add_lease=False):
 282         d = self.check_and_repair(verify)
 283         def _done(r):
 284             dr = DeepCheckAndRepairResults(self.storage_index)
 285             dr.add_check(r, [])
 286             return dr
 287         d.addCallback(_done)
 288         return d
 289
 290     def download_best_version(self):
 291         if self.storage_index not in self.all_contents:
 292             return defer.fail(NotEnoughSharesError(None, 0, 3))
 293         return defer.succeed(self.all_contents[self.storage_index])
 294     def overwrite(self, new_contents):
 295         if len(new_contents) > self.MUTABLE_SIZELIMIT:
 296             raise FileTooLargeError("SDMF is limited to one segment, and "
 297                                     "%d > %d" % (len(new_contents),
 298                                                  self.MUTABLE_SIZELIMIT))
 299         assert not self.is_readonly()
 300         self.all_contents[self.storage_index] = new_contents
 301         return defer.succeed(None)
 302     def modify(self, modifier):
 303         # this does not implement FileTooLargeError, but the real one does
 304         return defer.maybeDeferred(self._modify, modifier)
 305     def _modify(self, modifier):
 306         assert not self.is_readonly()
 307         old_contents = self.all_contents[self.storage_index]
 308         self.all_contents[self.storage_index] = modifier(old_contents, None, True)
 309         return None
 310
 311 def make_mutable_file_cap():
 312     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 313                                    fingerprint=os.urandom(32))
 314 def make_mutable_file_uri():
 315     return make_mutable_file_cap().to_string()
 316
 317 def make_verifier_uri():
 318     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 319                               fingerprint=os.urandom(32)).to_string()
 320
 321 class FakeDirectoryNode(dirnode.DirectoryNode):
 322     """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
 323     backing store, so it doesn't go to the grid. The child data is still
 324     encrypted and serialized, so this isn't useful for tests that want to
 325     look inside the dirnodes and check their contents.
 326     """
 327     filenode_class = FakeMutableFileNode
 328
 329 class LoggingServiceParent(service.MultiService):
 330     def log(self, *args, **kwargs):
 331         return log.msg(*args, **kwargs)
 332
 333
 334 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 335
 336     # SystemTestMixin tests tend to be a lot of work, and we have a few
 337     # buildslaves that are pretty slow, and every once in a while these tests
 338     # run up against the default 120 second timeout. So increase the default
 339     # timeout. Individual test cases can override this, of course.
 340     timeout = 300
 341
 342     def setUp(self):
 343         self.sparent = service.MultiService()
 344         self.sparent.startService()
 345
 346         self.stats_gatherer = None
 347         self.stats_gatherer_furl = None
 348         self.key_generator_svc = None
 349         self.key_generator_furl = None
 350
 351     def tearDown(self):
 352         log.msg("shutting down SystemTest services")
 353         d = self.sparent.stopService()
 354         d.addBoth(flush_but_dont_ignore)
 355         return d
 356
 357     def getdir(self, subdir):
 358         return os.path.join(self.basedir, subdir)
 359
 360     def add_service(self, s):
 361         s.setServiceParent(self.sparent)
 362         return s
 363
 364     def set_up_nodes(self, NUMCLIENTS=5,
 365                      use_stats_gatherer=False, use_key_generator=False):
 366         self.numclients = NUMCLIENTS
 367         iv_dir = self.getdir("introducer")
 368         if not os.path.isdir(iv_dir):
 369             fileutil.make_dirs(iv_dir)
 370             f = open(os.path.join(iv_dir, "webport"), "w")
 371             f.write("tcp:0:interface=127.0.0.1\n")
 372             f.close()
 373             if SYSTEM_TEST_CERTS:
 374                 os.mkdir(os.path.join(iv_dir, "private"))
 375                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 376                 f.write(SYSTEM_TEST_CERTS[0])
 377                 f.close()
 378         iv = IntroducerNode(basedir=iv_dir)
 379         self.introducer = self.add_service(iv)
 380         d = self.introducer.when_tub_ready()
 381         d.addCallback(self._get_introducer_web)
 382         if use_stats_gatherer:
 383             d.addCallback(self._set_up_stats_gatherer)
 384         if use_key_generator:
 385             d.addCallback(self._set_up_key_generator)
 386         d.addCallback(self._set_up_nodes_2)
 387         if use_stats_gatherer:
 388             d.addCallback(self._grab_stats)
 389         return d
 390
 391     def _get_introducer_web(self, res):
 392         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 393         self.introweb_url = f.read().strip()
 394         f.close()
 395
 396     def _set_up_stats_gatherer(self, res):
 397         statsdir = self.getdir("stats_gatherer")
 398         fileutil.make_dirs(statsdir)
 399         self.stats_gatherer_svc = StatsGathererService(statsdir)
 400         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 401         self.add_service(self.stats_gatherer_svc)
 402
 403         d = fireEventually()
 404         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 405         def check_for_furl():
 406             return os.path.exists(sgf)
 407         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 408         def get_furl(junk):
 409             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 410         d.addCallback(get_furl)
 411         return d
 412
 413     def _set_up_key_generator(self, res):
 414         kgsdir = self.getdir("key_generator")
 415         fileutil.make_dirs(kgsdir)
 416
 417         self.key_generator_svc = KeyGeneratorService(kgsdir,
 418                                                      display_furl=False,
 419                                                      default_key_size=522)
 420         self.key_generator_svc.key_generator.pool_size = 4
 421         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 422         self.add_service(self.key_generator_svc)
 423
 424         d = fireEventually()
 425         def check_for_furl():
 426             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 427         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 428         def get_furl(junk):
 429             kgf = os.path.join(kgsdir, 'key_generator.furl')
 430             self.key_generator_furl = file(kgf, 'rb').read().strip()
 431         d.addCallback(get_furl)
 432         return d
 433
 434     def _set_up_nodes_2(self, res):
 435         q = self.introducer
 436         self.introducer_furl = q.introducer_url
 437         self.clients = []
 438         basedirs = []
 439         for i in range(self.numclients):
 440             basedir = self.getdir("client%d" % i)
 441             basedirs.append(basedir)
 442             fileutil.make_dirs(os.path.join(basedir, "private"))
 443             if len(SYSTEM_TEST_CERTS) > (i+1):
 444                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 445                 f.write(SYSTEM_TEST_CERTS[i+1])
 446                 f.close()
 447
 448             def write(name, value):
 449                 open(os.path.join(basedir, name), "w").write(value+"\n")
 450             if i == 0:
 451                 # clients[0] runs a webserver and a helper, no key_generator
 452                 write("webport", "tcp:0:interface=127.0.0.1")
 453                 write("run_helper", "yes")
 454                 write("keepalive_timeout", "600")
 455             if i == 3:
 456                 # clients[3] runs a webserver and uses a helper, uses
 457                 # key_generator
 458                 write("webport", "tcp:0:interface=127.0.0.1")
 459                 write("disconnect_timeout", "1800")
 460                 if self.key_generator_furl:
 461                     kgf = "%s\n" % (self.key_generator_furl,)
 462                     write("key_generator.furl", kgf)
 463             write("introducer.furl", self.introducer_furl)
 464             if self.stats_gatherer_furl:
 465                 write("stats_gatherer.furl", self.stats_gatherer_furl)
 466
 467         # give subclasses a chance to append liens to the node's tahoe.cfg
 468         # files before they are launched.
 469         self._set_up_nodes_extra_config()
 470
 471         # start clients[0], wait for it's tub to be ready (at which point it
 472         # will have registered the helper furl).
 473         c = self.add_service(client.Client(basedir=basedirs[0]))
 474         self.clients.append(c)
 475         c.set_default_mutable_keysize(522)
 476         d = c.when_tub_ready()
 477         def _ready(res):
 478             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 479             helper_furl = f.read()
 480             f.close()
 481             self.helper_furl = helper_furl
 482             if self.numclients >= 4:
 483                 f = open(os.path.join(basedirs[3],"helper.furl"), "w")
 484                 f.write(helper_furl)
 485                 f.close()
 486
 487             # this starts the rest of the clients
 488             for i in range(1, self.numclients):
 489                 c = self.add_service(client.Client(basedir=basedirs[i]))
 490                 self.clients.append(c)
 491                 c.set_default_mutable_keysize(522)
 492             log.msg("STARTING")
 493             return self.wait_for_connections()
 494         d.addCallback(_ready)
 495         def _connected(res):
 496             log.msg("CONNECTED")
 497             # now find out where the web port was
 498             l = self.clients[0].getServiceNamed("webish").listener
 499             port = l._port.getHost().port
 500             self.webish_url = "http://localhost:%d/" % port
 501             if self.numclients >=4:
 502                 # and the helper-using webport
 503                 l = self.clients[3].getServiceNamed("webish").listener
 504                 port = l._port.getHost().port
 505                 self.helper_webish_url = "http://localhost:%d/" % port
 506         d.addCallback(_connected)
 507         return d
 508
 509     def _set_up_nodes_extra_config(self):
 510         # for overriding by subclasses
 511         pass
 512
 513     def _grab_stats(self, res):
 514         d = self.stats_gatherer.poll()
 515         return d
 516
 517     def bounce_client(self, num):
 518         c = self.clients[num]
 519         d = c.disownServiceParent()
 520         # I think windows requires a moment to let the connection really stop
 521         # and the port number made available for re-use. TODO: examine the
 522         # behavior, see if this is really the problem, see if we can do
 523         # better than blindly waiting for a second.
 524         d.addCallback(self.stall, 1.0)
 525         def _stopped(res):
 526             new_c = client.Client(basedir=self.getdir("client%d" % num))
 527             self.clients[num] = new_c
 528             new_c.set_default_mutable_keysize(522)
 529             self.add_service(new_c)
 530             return new_c.when_tub_ready()
 531         d.addCallback(_stopped)
 532         d.addCallback(lambda res: self.wait_for_connections())
 533         def _maybe_get_webport(res):
 534             if num == 0:
 535                 # now find out where the web port was
 536                 l = self.clients[0].getServiceNamed("webish").listener
 537                 port = l._port.getHost().port
 538                 self.webish_url = "http://localhost:%d/" % port
 539         d.addCallback(_maybe_get_webport)
 540         return d
 541
 542     def add_extra_node(self, client_num, helper_furl=None,
 543                        add_to_sparent=False):
 544         # usually this node is *not* parented to our self.sparent, so we can
 545         # shut it down separately from the rest, to exercise the
 546         # connection-lost code
 547         basedir = self.getdir("client%d" % client_num)
 548         if not os.path.isdir(basedir):
 549             fileutil.make_dirs(basedir)
 550         open(os.path.join(basedir, "introducer.furl"), "w").write(self.introducer_furl)
 551         if helper_furl:
 552             f = open(os.path.join(basedir, "helper.furl") ,"w")
 553             f.write(helper_furl+"\n")
 554             f.close()
 555
 556         c = client.Client(basedir=basedir)
 557         self.clients.append(c)
 558         c.set_default_mutable_keysize(522)
 559         self.numclients += 1
 560         if add_to_sparent:
 561             c.setServiceParent(self.sparent)
 562         else:
 563             c.startService()
 564         d = self.wait_for_connections()
 565         d.addCallback(lambda res: c)
 566         return d
 567
 568     def _check_connections(self):
 569         for c in self.clients:
 570             if not c.connected_to_introducer():
 571                 return False
 572             sb = c.get_storage_broker()
 573             if len(sb.get_all_servers()) != self.numclients:
 574                 return False
 575         return True
 576
 577     def wait_for_connections(self, ignored=None):
 578         # TODO: replace this with something that takes a list of peerids and
 579         # fires when they've all been heard from, instead of using a count
 580         # and a threshold
 581         return self.poll(self._check_connections, timeout=200)
 582
 583
 584 # our system test uses the same Tub certificates each time, to avoid the
 585 # overhead of key generation
 586 SYSTEM_TEST_CERTS = [
 587 """-----BEGIN CERTIFICATE-----
 588 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 589 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 590 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 591 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 592 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 593 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 594 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 595 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 596 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 597 -----END CERTIFICATE-----
 598 -----BEGIN RSA PRIVATE KEY-----
 599 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 600 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 601 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 602 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 603 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 604 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 605 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 606 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 607 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 608 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 609 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 610 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 611 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 612 -----END RSA PRIVATE KEY-----
 613 """, # 0
 614 """-----BEGIN CERTIFICATE-----
 615 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 616 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 617 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 618 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 619 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 620 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 621 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 622 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 623 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 624 -----END CERTIFICATE-----
 625 -----BEGIN RSA PRIVATE KEY-----
 626 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 627 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 628 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 629 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 630 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 631 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 632 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 633 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 634 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 635 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 636 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 637 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 638 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 639 -----END RSA PRIVATE KEY-----
 640 """, # 1
 641 """-----BEGIN CERTIFICATE-----
 642 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 643 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 644 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 645 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 646 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 647 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 648 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 649 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 650 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 651 -----END CERTIFICATE-----
 652 -----BEGIN RSA PRIVATE KEY-----
 653 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 654 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 655 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 656 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 657 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 658 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 659 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 660 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 661 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 662 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 663 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 664 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 665 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 666 -----END RSA PRIVATE KEY-----
 667 """, # 2
 668 """-----BEGIN CERTIFICATE-----
 669 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 670 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 671 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 672 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 673 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 674 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 675 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 676 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 677 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 678 -----END CERTIFICATE-----
 679 -----BEGIN RSA PRIVATE KEY-----
 680 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 681 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 682 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 683 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 684 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 685 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 686 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 687 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 688 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 689 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 690 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 691 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 692 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 693 -----END RSA PRIVATE KEY-----
 694 """, # 3
 695 """-----BEGIN CERTIFICATE-----
 696 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 697 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 698 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 699 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 700 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 701 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 702 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 703 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 704 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 705 -----END CERTIFICATE-----
 706 -----BEGIN RSA PRIVATE KEY-----
 707 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 708 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 709 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 710 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 711 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 712 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 713 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 714 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 715 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 716 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 717 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 718 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 719 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 720 -----END RSA PRIVATE KEY-----
 721 """, # 4
 722 """-----BEGIN CERTIFICATE-----
 723 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 724 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 725 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 726 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 727 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 728 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 729 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 730 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 731 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 732 -----END CERTIFICATE-----
 733 -----BEGIN RSA PRIVATE KEY-----
 734 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 735 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 736 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 737 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 738 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 739 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 740 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 741 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 742 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 743 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 744 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 745 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 746 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 747 -----END RSA PRIVATE KEY-----
 748 """, # 5
 749 """-----BEGIN CERTIFICATE-----
 750 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 751 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 752 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 753 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 754 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 755 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 756 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 757 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 758 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 759 -----END CERTIFICATE-----
 760 -----BEGIN RSA PRIVATE KEY-----
 761 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 762 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 763 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 764 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 765 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 766 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 767 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 768 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 769 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 770 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 771 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 772 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 773 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 774 -----END RSA PRIVATE KEY-----
 775 """, #6
 776 """-----BEGIN CERTIFICATE-----
 777 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 778 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 779 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 780 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 781 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 782 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 783 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 784 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 785 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 786 -----END CERTIFICATE-----
 787 -----BEGIN RSA PRIVATE KEY-----
 788 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 789 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 790 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 791 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 792 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 793 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 794 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 795 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 796 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 797 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 798 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 799 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 800 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 801 -----END RSA PRIVATE KEY-----
 802 """, #7
 803 """-----BEGIN CERTIFICATE-----
 804 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 805 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 806 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 807 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 808 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 809 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 810 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 811 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 812 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 813 -----END CERTIFICATE-----
 814 -----BEGIN RSA PRIVATE KEY-----
 815 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 816 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 817 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 818 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 819 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 820 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 821 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 822 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 823 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 824 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 825 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 826 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 827 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 828 -----END RSA PRIVATE KEY-----
 829 """, #8
 830 """-----BEGIN CERTIFICATE-----
 831 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 832 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 833 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 834 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 835 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 836 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 837 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 838 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 839 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 840 -----END CERTIFICATE-----
 841 -----BEGIN RSA PRIVATE KEY-----
 842 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 843 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 844 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 845 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 846 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 847 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 848 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 849 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 850 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 851 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 852 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 853 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 854 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 855 -----END RSA PRIVATE KEY-----
 856 """, #9
 857 """-----BEGIN CERTIFICATE-----
 858 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 859 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 860 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 861 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 862 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 863 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 864 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 865 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 866 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 867 -----END CERTIFICATE-----
 868 -----BEGIN RSA PRIVATE KEY-----
 869 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 870 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 871 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 872 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 873 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 874 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 875 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
 876 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
 877 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
 878 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
 879 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
 880 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
 881 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
 882 -----END RSA PRIVATE KEY-----
 883 """, #10
 884 """-----BEGIN CERTIFICATE-----
 885 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 886 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 887 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
 888 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
 889 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
 890 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 891 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
 892 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
 893 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
 894 -----END CERTIFICATE-----
 895 -----BEGIN RSA PRIVATE KEY-----
 896 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
 897 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
 898 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
 899 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
 900 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
 901 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
 902 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
 903 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
 904 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
 905 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
 906 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
 907 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
 908 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
 909 -----END RSA PRIVATE KEY-----
 910 """, #11
 911 ]
 912
 913 # To disable the pre-computed tub certs, uncomment this line.
 914 #SYSTEM_TEST_CERTS = []
 915
 916 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
 917
 918 class ShareManglingMixin(SystemTestMixin):
 919
 920     def setUp(self):
 921         # Set self.basedir to a temp dir which has the name of the current
 922         # test method in its name.
 923         self.basedir = self.mktemp()
 924
 925         d = defer.maybeDeferred(SystemTestMixin.setUp, self)
 926         d.addCallback(lambda x: self.set_up_nodes())
 927
 928         def _upload_a_file(ignored):
 929             cl0 = self.clients[0]
 930             # We need multiple segments to test crypttext hash trees that are
 931             # non-trivial (i.e. they have more than just one hash in them).
 932             cl0.DEFAULT_ENCODING_PARAMETERS['max_segment_size'] = 12
 933             d2 = cl0.upload(immutable.upload.Data(TEST_DATA, convergence=""))
 934             def _after_upload(u):
 935                 filecap = u.uri
 936                 self.n = self.clients[1].create_node_from_uri(filecap)
 937                 self.uri = uri.CHKFileURI.init_from_string(filecap)
 938                 return cl0.create_node_from_uri(filecap)
 939             d2.addCallback(_after_upload)
 940             return d2
 941         d.addCallback(_upload_a_file)
 942
 943         def _stash_it(filenode):
 944             self.filenode = filenode
 945         d.addCallback(_stash_it)
 946         return d
 947
 948     def find_shares(self, unused=None):
 949         """Locate shares on disk. Returns a dict that maps
 950         (clientnum,sharenum) to a string that contains the share container
 951         (copied directly from the disk, containing leases etc). You can
 952         modify this dict and then call replace_shares() to modify the shares.
 953         """
 954         shares = {} # k: (i, sharenum), v: data
 955
 956         for i, c in enumerate(self.clients):
 957             sharedir = c.getServiceNamed("storage").sharedir
 958             for (dirp, dirns, fns) in os.walk(sharedir):
 959                 for fn in fns:
 960                     try:
 961                         sharenum = int(fn)
 962                     except TypeError:
 963                         # Whoops, I guess that's not a share file then.
 964                         pass
 965                     else:
 966                         data = open(os.path.join(sharedir, dirp, fn), "rb").read()
 967                         shares[(i, sharenum)] = data
 968
 969         return shares
 970
 971     def replace_shares(self, newshares, storage_index):
 972         """Replace shares on disk. Takes a dictionary in the same form
 973         as find_shares() returns."""
 974
 975         for i, c in enumerate(self.clients):
 976             sharedir = c.getServiceNamed("storage").sharedir
 977             for (dirp, dirns, fns) in os.walk(sharedir):
 978                 for fn in fns:
 979                     try:
 980                         sharenum = int(fn)
 981                     except TypeError:
 982                         # Whoops, I guess that's not a share file then.
 983                         pass
 984                     else:
 985                         pathtosharefile = os.path.join(sharedir, dirp, fn)
 986                         os.unlink(pathtosharefile)
 987             for ((clientnum, sharenum), newdata) in newshares.iteritems():
 988                 if clientnum == i:
 989                     fullsharedirp=os.path.join(sharedir, storage_index_to_dir(storage_index))
 990                     fileutil.make_dirs(fullsharedirp)
 991                     wf = open(os.path.join(fullsharedirp, str(sharenum)), "wb")
 992                     wf.write(newdata)
 993                     wf.close()
 994
 995     def _delete_a_share(self, unused=None, sharenum=None):
 996         """ Delete one share. """
 997
 998         shares = self.find_shares()
 999         ks = shares.keys()
1000         if sharenum is not None:
1001             k = [ key for key in shares.keys() if key[1] == sharenum ][0]
1002         else:
1003             k = random.choice(ks)
1004         del shares[k]
1005         self.replace_shares(shares, storage_index=self.uri.storage_index)
1006
1007         return unused
1008
1009     def _corrupt_a_share(self, unused, corruptor_func, sharenum):
1010         shares = self.find_shares()
1011         ks = [ key for key in shares.keys() if key[1] == sharenum ]
1012         assert ks, (shares.keys(), sharenum)
1013         k = ks[0]
1014         shares[k] = corruptor_func(shares[k])
1015         self.replace_shares(shares, storage_index=self.uri.storage_index)
1016         return corruptor_func
1017
1018     def _corrupt_all_shares(self, unused, corruptor_func):
1019         """ All shares on disk will be corrupted by corruptor_func. """
1020         shares = self.find_shares()
1021         for k in shares.keys():
1022             self._corrupt_a_share(unused, corruptor_func, k[1])
1023         return corruptor_func
1024
1025     def _corrupt_a_random_share(self, unused, corruptor_func):
1026         """ Exactly one share on disk will be corrupted by corruptor_func. """
1027         shares = self.find_shares()
1028         ks = shares.keys()
1029         k = random.choice(ks)
1030         self._corrupt_a_share(unused, corruptor_func, k[1])
1031         return k[1]
1032
1033     def _count_reads(self):
1034         sum_of_read_counts = 0
1035         for thisclient in self.clients:
1036             counters = thisclient.stats_provider.get_stats()['counters']
1037             sum_of_read_counts += counters.get('storage_server.read', 0)
1038         return sum_of_read_counts
1039
1040     def _count_allocates(self):
1041         sum_of_allocate_counts = 0
1042         for thisclient in self.clients:
1043             counters = thisclient.stats_provider.get_stats()['counters']
1044             sum_of_allocate_counts += counters.get('storage_server.allocate', 0)
1045         return sum_of_allocate_counts
1046
1047     def _count_writes(self):
1048         sum_of_write_counts = 0
1049         for thisclient in self.clients:
1050             counters = thisclient.stats_provider.get_stats()['counters']
1051             sum_of_write_counts += counters.get('storage_server.write', 0)
1052         return sum_of_write_counts
1053
1054     def _download_and_check_plaintext(self, unused=None):
1055         d = download_to_data(self.n)
1056         def _after_download(result):
1057             self.failUnlessEqual(result, TEST_DATA)
1058         d.addCallback(_after_download)
1059         return d
1060
1061 class ShouldFailMixin:
1062     def shouldFail(self, expected_failure, which, substring,
1063                    callable, *args, **kwargs):
1064         """Assert that a function call raises some exception. This is a
1065         Deferred-friendly version of TestCase.assertRaises() .
1066
1067         Suppose you want to verify the following function:
1068
1069          def broken(a, b, c):
1070              if a < 0:
1071                  raise TypeError('a must not be negative')
1072              return defer.succeed(b+c)
1073
1074         You can use:
1075             d = self.shouldFail(TypeError, 'test name',
1076                                 'a must not be negative',
1077                                 broken, -4, 5, c=12)
1078         in your test method. The 'test name' string will be included in the
1079         error message, if any, because Deferred chains frequently make it
1080         difficult to tell which assertion was tripped.
1081
1082         The substring= argument, if not None, must appear inside the
1083         stringified Failure, or the test will fail.
1084         """
1085
1086         assert substring is None or isinstance(substring, str)
1087         d = defer.maybeDeferred(callable, *args, **kwargs)
1088         def done(res):
1089             if isinstance(res, failure.Failure):
1090                 res.trap(expected_failure)
1091                 if substring:
1092                     self.failUnless(substring in str(res),
1093                                     "substring '%s' not in '%s'"
1094                                     % (substring, str(res)))
1095             else:
1096                 self.fail("%s was supposed to raise %s, not get '%s'" %
1097                           (which, expected_failure, res))
1098         d.addBoth(done)
1099         return d
1100
1101 class WebErrorMixin:
1102     def explain_web_error(self, f):
1103         # an error on the server side causes the client-side getPage() to
1104         # return a failure(t.web.error.Error), and its str() doesn't show the
1105         # response body, which is where the useful information lives. Attach
1106         # this method as an errback handler, and it will reveal the hidden
1107         # message.
1108         f.trap(WebError)
1109         print "Web Error:", f.value, ":", f.value.response
1110         return f
1111
1112     def _shouldHTTPError(self, res, which, validator):
1113         if isinstance(res, failure.Failure):
1114             res.trap(WebError)
1115             return validator(res)
1116         else:
1117             self.fail("%s was supposed to Error, not get '%s'" % (which, res))
1118
1119     def shouldHTTPError(self, which,
1120                         code=None, substring=None, response_substring=None,
1121                         callable=None, *args, **kwargs):
1122         # returns a Deferred with the response body
1123         assert substring is None or isinstance(substring, str)
1124         assert callable
1125         def _validate(f):
1126             if code is not None:
1127                 self.failUnlessEqual(f.value.status, str(code))
1128             if substring:
1129                 code_string = str(f)
1130                 self.failUnless(substring in code_string,
1131                                 "substring '%s' not in '%s'"
1132                                 % (substring, code_string))
1133             response_body = f.value.response
1134             if response_substring:
1135                 self.failUnless(response_substring in response_body,
1136                                 "response substring '%s' not in '%s'"
1137                                 % (response_substring, response_body))
1138             return response_body
1139         d = defer.maybeDeferred(callable, *args, **kwargs)
1140         d.addBoth(self._shouldHTTPError, which, _validate)
1141         return d
1142
1143 class ErrorMixin(WebErrorMixin):
1144     def explain_error(self, f):
1145         if f.check(defer.FirstError):
1146             print "First Error:", f.value.subFailure
1147         return f
1148
1149 def corrupt_field(data, offset, size, debug=False):
1150     if random.random() < 0.5:
1151         newdata = testutil.flip_one_bit(data, offset, size)
1152         if debug:
1153             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1154         return newdata
1155     else:
1156         newval = testutil.insecurerandstr(size)
1157         if debug:
1158             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1159         return data[:offset]+newval+data[offset+size:]
1160
1161 def _corrupt_nothing(data, debug=False):
1162     """Leave the data pristine. """
1163     return data
1164
1165 def _corrupt_file_version_number(data, debug=False):
1166     """Scramble the file data -- the share file version number have one bit
1167     flipped or else will be changed to a random value."""
1168     return corrupt_field(data, 0x00, 4)
1169
1170 def _corrupt_size_of_file_data(data, debug=False):
1171     """Scramble the file data -- the field showing the size of the share data
1172     within the file will be set to one smaller."""
1173     return corrupt_field(data, 0x04, 4)
1174
1175 def _corrupt_sharedata_version_number(data, debug=False):
1176     """Scramble the file data -- the share data version number will have one
1177     bit flipped or else will be changed to a random value, but not 1 or 2."""
1178     return corrupt_field(data, 0x0c, 4)
1179     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1180     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1181     newsharevernum = sharevernum
1182     while newsharevernum in (1, 2):
1183         newsharevernum = random.randrange(0, 2**32)
1184     newsharevernumbytes = struct.pack(">L", newsharevernum)
1185     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1186
1187 def _corrupt_sharedata_version_number_to_plausible_version(data, debug=False):
1188     """Scramble the file data -- the share data version number will be
1189     changed to 2 if it is 1 or else to 1 if it is 2."""
1190     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1191     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1192     if sharevernum == 1:
1193         newsharevernum = 2
1194     else:
1195         newsharevernum = 1
1196     newsharevernumbytes = struct.pack(">L", newsharevernum)
1197     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1198
1199 def _corrupt_segment_size(data, debug=False):
1200     """Scramble the file data -- the field showing the size of the segment
1201     will have one bit flipped or else be changed to a random value."""
1202     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1203     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1204     if sharevernum == 1:
1205         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1206     else:
1207         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1208
1209 def _corrupt_size_of_sharedata(data, debug=False):
1210     """Scramble the file data -- the field showing the size of the data
1211     within the share data will have one bit flipped or else will be changed
1212     to a random value."""
1213     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1214     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1215     if sharevernum == 1:
1216         return corrupt_field(data, 0x0c+0x08, 4)
1217     else:
1218         return corrupt_field(data, 0x0c+0x0c, 8)
1219
1220 def _corrupt_offset_of_sharedata(data, debug=False):
1221     """Scramble the file data -- the field showing the offset of the data
1222     within the share data will have one bit flipped or else be changed to a
1223     random value."""
1224     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1225     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1226     if sharevernum == 1:
1227         return corrupt_field(data, 0x0c+0x0c, 4)
1228     else:
1229         return corrupt_field(data, 0x0c+0x14, 8)
1230
1231 def _corrupt_offset_of_ciphertext_hash_tree(data, debug=False):
1232     """Scramble the file data -- the field showing the offset of the
1233     ciphertext hash tree within the share data will have one bit flipped or
1234     else be changed to a random value.
1235     """
1236     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1237     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1238     if sharevernum == 1:
1239         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1240     else:
1241         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1242
1243 def _corrupt_offset_of_block_hashes(data, debug=False):
1244     """Scramble the file data -- the field showing the offset of the block
1245     hash tree within the share data will have one bit flipped or else will be
1246     changed to a random value."""
1247     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1248     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1249     if sharevernum == 1:
1250         return corrupt_field(data, 0x0c+0x18, 4)
1251     else:
1252         return corrupt_field(data, 0x0c+0x2c, 8)
1253
1254 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data, debug=False):
1255     """Scramble the file data -- the field showing the offset of the block
1256     hash tree within the share data will have a multiple of hash size
1257     subtracted from it, thus causing the downloader to download an incomplete
1258     crypttext hash tree."""
1259     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1260     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1261     if sharevernum == 1:
1262         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1263         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1264         newvalstr = struct.pack(">L", newval)
1265         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1266     else:
1267         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1268         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1269         newvalstr = struct.pack(">Q", newval)
1270         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1271
1272 def _corrupt_offset_of_share_hashes(data, debug=False):
1273     """Scramble the file data -- the field showing the offset of the share
1274     hash tree within the share data will have one bit flipped or else will be
1275     changed to a random value."""
1276     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1277     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1278     if sharevernum == 1:
1279         return corrupt_field(data, 0x0c+0x1c, 4)
1280     else:
1281         return corrupt_field(data, 0x0c+0x34, 8)
1282
1283 def _corrupt_offset_of_uri_extension(data, debug=False):
1284     """Scramble the file data -- the field showing the offset of the uri
1285     extension will have one bit flipped or else will be changed to a random
1286     value."""
1287     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1288     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1289     if sharevernum == 1:
1290         return corrupt_field(data, 0x0c+0x20, 4)
1291     else:
1292         return corrupt_field(data, 0x0c+0x3c, 8)
1293
1294 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1295     """Scramble the file data -- the field showing the offset of the uri
1296     extension will be set to the size of the file minus 3. This means when
1297     the client tries to read the length field from that location it will get
1298     a short read -- the result string will be only 3 bytes long, not the 4 or
1299     8 bytes necessary to do a successful struct.unpack."""
1300     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1301     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1302     # The "-0x0c" in here is to skip the server-side header in the share
1303     # file, which the client doesn't see when seeking and reading.
1304     if sharevernum == 1:
1305         if debug:
1306             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1307         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1308     else:
1309         if debug:
1310             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1311         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1312
1313 def _corrupt_mutable_share_data(data, debug=False):
1314     prefix = data[:32]
1315     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
1316     data_offset = MutableShareFile.DATA_OFFSET
1317     sharetype = data[data_offset:data_offset+1]
1318     assert sharetype == "\x00", "non-SDMF mutable shares not supported"
1319     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
1320      ig_datalen, offsets) = unpack_header(data[data_offset:])
1321     assert version == 0, "this function only handles v0 SDMF files"
1322     start = data_offset + offsets["share_data"]
1323     length = data_offset + offsets["enc_privkey"] - start
1324     return corrupt_field(data, start, length)
1325
1326 def _corrupt_share_data(data, debug=False):
1327     """Scramble the file data -- the field containing the share data itself
1328     will have one bit flipped or else will be changed to a random value."""
1329     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1330     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1331     if sharevernum == 1:
1332         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1333
1334         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1335     else:
1336         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1337
1338         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1339
1340 def _corrupt_crypttext_hash_tree(data, debug=False):
1341     """Scramble the file data -- the field containing the crypttext hash tree
1342     will have one bit flipped or else will be changed to a random value.
1343     """
1344     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1345     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1346     if sharevernum == 1:
1347         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1348         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1349     else:
1350         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1351         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1352
1353     return corrupt_field(data, 0x0c+crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset, debug=debug)
1354
1355 def _corrupt_crypttext_hash_tree_byte_x221(data, debug=False):
1356     """Scramble the file data -- the byte at offset 0x221 will have its 7th
1357     (b1) bit flipped.
1358     """
1359     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1360     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1361     if debug:
1362         log.msg("original data: %r" % (data,))
1363     return data[:0x0c+0x221] + chr(ord(data[0x0c+0x221])^0x02) + data[0x0c+0x2210+1:]
1364
1365 def _corrupt_block_hashes(data, debug=False):
1366     """Scramble the file data -- the field containing the block hash tree
1367     will have one bit flipped or else will be changed to a random value.
1368     """
1369     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1370     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1371     if sharevernum == 1:
1372         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1373         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1374     else:
1375         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1376         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1377
1378     return corrupt_field(data, 0x0c+blockhashesoffset, sharehashesoffset-blockhashesoffset)
1379
1380 def _corrupt_share_hashes(data, debug=False):
1381     """Scramble the file data -- the field containing the share hash chain
1382     will have one bit flipped or else will be changed to a random value.
1383     """
1384     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1385     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1386     if sharevernum == 1:
1387         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1388         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1389     else:
1390         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1391         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1392
1393     return corrupt_field(data, 0x0c+sharehashesoffset, uriextoffset-sharehashesoffset)
1394
1395 def _corrupt_length_of_uri_extension(data, debug=False):
1396     """Scramble the file data -- the field showing the length of the uri
1397     extension will have one bit flipped or else will be changed to a random
1398     value."""
1399     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1400     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1401     if sharevernum == 1:
1402         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1403         return corrupt_field(data, uriextoffset, 4)
1404     else:
1405         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1406         return corrupt_field(data, 0x0c+uriextoffset, 8)
1407
1408 def _corrupt_uri_extension(data, debug=False):
1409     """Scramble the file data -- the field containing the uri extension will
1410     have one bit flipped or else will be changed to a random value."""
1411     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1412     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1413     if sharevernum == 1:
1414         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1415         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1416     else:
1417         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1418         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1419
1420     return corrupt_field(data, 0x0c+uriextoffset, uriextlen)