src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IPullProducer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.api import flushEventualQueue, fireEventually
   9 from allmydata import uri, dirnode, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IMutableFileNode, IImmutableFileNode, \
  12      FileTooLargeError, NotEnoughSharesError, ICheckable
  13 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  14      DeepCheckResults, DeepCheckAndRepairResults
  15 from allmydata.mutable.common import CorruptShareError
  16 from allmydata.mutable.layout import unpack_header
  17 from allmydata.storage.server import storage_index_to_dir
  18 from allmydata.storage.mutable import MutableShareFile
  19 from allmydata.util import hashutil, log, fileutil, pollmixin
  20 from allmydata.util.assertutil import precondition
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.stats import StatsGathererService
  23 from allmydata.key_generator import KeyGeneratorService
  24 import common_util as testutil
  25 from allmydata import immutable
  26
  27
  28 def flush_but_dont_ignore(res):
  29     d = flushEventualQueue()
  30     def _done(ignored):
  31         return res
  32     d.addCallback(_done)
  33     return d
  34
  35 class DummyProducer:
  36     implements(IPullProducer)
  37     def resumeProducing(self):
  38         pass
  39
  40 class FakeCHKFileNode:
  41     """I provide IImmutableFileNode, but all of my data is stored in a
  42     class-level dictionary."""
  43     implements(IImmutableFileNode)
  44     all_contents = {}
  45     bad_shares = {}
  46
  47     def __init__(self, filecap):
  48         precondition(isinstance(filecap, (uri.CHKFileURI, uri.LiteralFileURI)), filecap)
  49         self.my_uri = filecap
  50         self.storage_index = self.my_uri.get_storage_index()
  51
  52     def get_uri(self):
  53         return self.my_uri.to_string()
  54     def get_write_uri(self):
  55         return None
  56     def get_readonly_uri(self):
  57         return self.my_uri.to_string()
  58     def get_cap(self):
  59         return self.my_uri
  60     def get_verify_cap(self):
  61         return self.my_uri.get_verify_cap()
  62     def get_repair_cap(self):
  63         return self.my_uri.get_verify_cap()
  64     def get_storage_index(self):
  65         return self.storage_index
  66
  67     def check(self, monitor, verify=False, add_lease=False):
  68         r = CheckResults(self.my_uri, self.storage_index)
  69         is_bad = self.bad_shares.get(self.storage_index, None)
  70         data = {}
  71         data["count-shares-needed"] = 3
  72         data["count-shares-expected"] = 10
  73         data["count-good-share-hosts"] = 10
  74         data["count-wrong-shares"] = 0
  75         nodeid = "\x00"*20
  76         data["list-corrupt-shares"] = []
  77         data["sharemap"] = {1: [nodeid]}
  78         data["servers-responding"] = [nodeid]
  79         data["count-recoverable-versions"] = 1
  80         data["count-unrecoverable-versions"] = 0
  81         if is_bad:
  82              r.set_healthy(False)
  83              r.set_recoverable(True)
  84              data["count-shares-good"] = 9
  85              data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
  86              r.problems = failure.Failure(CorruptShareError(is_bad))
  87         else:
  88              r.set_healthy(True)
  89              r.set_recoverable(True)
  90              data["count-shares-good"] = 10
  91              r.problems = []
  92         r.set_data(data)
  93         r.set_needs_rebalancing(False)
  94         return defer.succeed(r)
  95     def check_and_repair(self, monitor, verify=False, add_lease=False):
  96         d = self.check(verify)
  97         def _got(cr):
  98             r = CheckAndRepairResults(self.storage_index)
  99             r.pre_repair_results = r.post_repair_results = cr
 100             return r
 101         d.addCallback(_got)
 102         return d
 103
 104     def is_mutable(self):
 105         return False
 106     def is_readonly(self):
 107         return True
 108     def is_unknown(self):
 109         return False
 110     def is_allowed_in_immutable_directory(self):
 111         return True
 112     def raise_error(self):
 113         pass
 114
 115     def get_size(self):
 116         if isinstance(self.my_uri, uri.LiteralFileURI):
 117             return self.my_uri.get_size()
 118         try:
 119             data = self.all_contents[self.my_uri.to_string()]
 120         except KeyError, le:
 121             raise NotEnoughSharesError(le, 0, 3)
 122         return len(data)
 123
 124     def read(self, consumer, offset=0, size=None):
 125         # we don't bother to call registerProducer/unregisterProducer,
 126         # because it's a hassle to write a dummy Producer that does the right
 127         # thing (we have to make sure that DummyProducer.resumeProducing
 128         # writes the data into the consumer immediately, otherwise it will
 129         # loop forever).
 130
 131         d = defer.succeed(None)
 132         d.addCallback(self._read, consumer, offset, size)
 133         return d
 134
 135     def _read(self, ignored, consumer, offset, size):
 136         if isinstance(self.my_uri, uri.LiteralFileURI):
 137             data = self.my_uri.data
 138         else:
 139             if self.my_uri.to_string() not in self.all_contents:
 140                 raise NotEnoughSharesError(None, 0, 3)
 141             data = self.all_contents[self.my_uri.to_string()]
 142         start = offset
 143         if size is not None:
 144             end = offset + size
 145         else:
 146             end = len(data)
 147         consumer.write(data[start:end])
 148         return consumer
 149
 150 def make_chk_file_cap(size):
 151     return uri.CHKFileURI(key=os.urandom(16),
 152                           uri_extension_hash=os.urandom(32),
 153                           needed_shares=3,
 154                           total_shares=10,
 155                           size=size)
 156 def make_chk_file_uri(size):
 157     return make_chk_file_cap(size).to_string()
 158
 159 def create_chk_filenode(contents):
 160     filecap = make_chk_file_cap(len(contents))
 161     n = FakeCHKFileNode(filecap)
 162     FakeCHKFileNode.all_contents[filecap.to_string()] = contents
 163     return n
 164
 165
 166 class FakeMutableFileNode:
 167     """I provide IMutableFileNode, but all of my data is stored in a
 168     class-level dictionary."""
 169
 170     implements(IMutableFileNode, ICheckable)
 171     MUTABLE_SIZELIMIT = 10000
 172     all_contents = {}
 173     bad_shares = {}
 174
 175     def __init__(self, storage_broker, secret_holder,
 176                  default_encoding_parameters, history):
 177         self.init_from_cap(make_mutable_file_cap())
 178     def create(self, contents, key_generator=None, keysize=None):
 179         initial_contents = self._get_initial_contents(contents)
 180         if len(initial_contents) > self.MUTABLE_SIZELIMIT:
 181             raise FileTooLargeError("SDMF is limited to one segment, and "
 182                                     "%d > %d" % (len(initial_contents),
 183                                                  self.MUTABLE_SIZELIMIT))
 184         self.all_contents[self.storage_index] = initial_contents
 185         return defer.succeed(self)
 186     def _get_initial_contents(self, contents):
 187         if isinstance(contents, str):
 188             return contents
 189         if contents is None:
 190             return ""
 191         assert callable(contents), "%s should be callable, not %s" % \
 192                (contents, type(contents))
 193         return contents(self)
 194     def init_from_cap(self, filecap):
 195         assert isinstance(filecap, (uri.WriteableSSKFileURI,
 196                                     uri.ReadonlySSKFileURI))
 197         self.my_uri = filecap
 198         self.storage_index = self.my_uri.get_storage_index()
 199         return self
 200     def get_cap(self):
 201         return self.my_uri
 202     def get_readcap(self):
 203         return self.my_uri.get_readonly()
 204     def get_uri(self):
 205         return self.my_uri.to_string()
 206     def get_write_uri(self):
 207         if self.is_readonly():
 208             return None
 209         return self.my_uri.to_string()
 210     def get_readonly(self):
 211         return self.my_uri.get_readonly()
 212     def get_readonly_uri(self):
 213         return self.my_uri.get_readonly().to_string()
 214     def get_verify_cap(self):
 215         return self.my_uri.get_verify_cap()
 216     def is_readonly(self):
 217         return self.my_uri.is_readonly()
 218     def is_mutable(self):
 219         return self.my_uri.is_mutable()
 220     def is_unknown(self):
 221         return False
 222     def is_allowed_in_immutable_directory(self):
 223         return not self.my_uri.is_mutable()
 224     def raise_error(self):
 225         pass
 226     def get_writekey(self):
 227         return "\x00"*16
 228     def get_size(self):
 229         return len(self.all_contents[self.storage_index])
 230     def get_current_size(self):
 231         return self.get_size_of_best_version()
 232     def get_size_of_best_version(self):
 233         return defer.succeed(len(self.all_contents[self.storage_index]))
 234
 235     def get_storage_index(self):
 236         return self.storage_index
 237
 238     def check(self, monitor, verify=False, add_lease=False):
 239         r = CheckResults(self.my_uri, self.storage_index)
 240         is_bad = self.bad_shares.get(self.storage_index, None)
 241         data = {}
 242         data["count-shares-needed"] = 3
 243         data["count-shares-expected"] = 10
 244         data["count-good-share-hosts"] = 10
 245         data["count-wrong-shares"] = 0
 246         data["list-corrupt-shares"] = []
 247         nodeid = "\x00"*20
 248         data["sharemap"] = {"seq1-abcd-sh0": [nodeid]}
 249         data["servers-responding"] = [nodeid]
 250         data["count-recoverable-versions"] = 1
 251         data["count-unrecoverable-versions"] = 0
 252         if is_bad:
 253              r.set_healthy(False)
 254              r.set_recoverable(True)
 255              data["count-shares-good"] = 9
 256              r.problems = failure.Failure(CorruptShareError("peerid",
 257                                                             0, # shnum
 258                                                             is_bad))
 259         else:
 260              r.set_healthy(True)
 261              r.set_recoverable(True)
 262              data["count-shares-good"] = 10
 263              r.problems = []
 264         r.set_data(data)
 265         r.set_needs_rebalancing(False)
 266         return defer.succeed(r)
 267
 268     def check_and_repair(self, monitor, verify=False, add_lease=False):
 269         d = self.check(verify)
 270         def _got(cr):
 271             r = CheckAndRepairResults(self.storage_index)
 272             r.pre_repair_results = r.post_repair_results = cr
 273             return r
 274         d.addCallback(_got)
 275         return d
 276
 277     def deep_check(self, verify=False, add_lease=False):
 278         d = self.check(verify)
 279         def _done(r):
 280             dr = DeepCheckResults(self.storage_index)
 281             dr.add_check(r, [])
 282             return dr
 283         d.addCallback(_done)
 284         return d
 285
 286     def deep_check_and_repair(self, verify=False, add_lease=False):
 287         d = self.check_and_repair(verify)
 288         def _done(r):
 289             dr = DeepCheckAndRepairResults(self.storage_index)
 290             dr.add_check(r, [])
 291             return dr
 292         d.addCallback(_done)
 293         return d
 294
 295     def download_best_version(self):
 296         if isinstance(self.my_uri, uri.LiteralFileURI):
 297             return defer.succeed(self.my_uri.data)
 298         if self.storage_index not in self.all_contents:
 299             return defer.fail(NotEnoughSharesError(None, 0, 3))
 300         return defer.succeed(self.all_contents[self.storage_index])
 301
 302     def overwrite(self, new_contents):
 303         if len(new_contents) > self.MUTABLE_SIZELIMIT:
 304             raise FileTooLargeError("SDMF is limited to one segment, and "
 305                                     "%d > %d" % (len(new_contents),
 306                                                  self.MUTABLE_SIZELIMIT))
 307         assert not self.is_readonly()
 308         self.all_contents[self.storage_index] = new_contents
 309         return defer.succeed(None)
 310     def modify(self, modifier):
 311         # this does not implement FileTooLargeError, but the real one does
 312         return defer.maybeDeferred(self._modify, modifier)
 313     def _modify(self, modifier):
 314         assert not self.is_readonly()
 315         old_contents = self.all_contents[self.storage_index]
 316         self.all_contents[self.storage_index] = modifier(old_contents, None, True)
 317         return None
 318
 319 def make_mutable_file_cap():
 320     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 321                                    fingerprint=os.urandom(32))
 322 def make_mutable_file_uri():
 323     return make_mutable_file_cap().to_string()
 324
 325 def make_verifier_uri():
 326     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 327                               fingerprint=os.urandom(32)).to_string()
 328
 329 class FakeDirectoryNode(dirnode.DirectoryNode):
 330     """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
 331     backing store, so it doesn't go to the grid. The child data is still
 332     encrypted and serialized, so this isn't useful for tests that want to
 333     look inside the dirnodes and check their contents.
 334     """
 335     filenode_class = FakeMutableFileNode
 336
 337 class LoggingServiceParent(service.MultiService):
 338     def log(self, *args, **kwargs):
 339         return log.msg(*args, **kwargs)
 340
 341
 342 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 343
 344     # SystemTestMixin tests tend to be a lot of work, and we have a few
 345     # buildslaves that are pretty slow, and every once in a while these tests
 346     # run up against the default 120 second timeout. So increase the default
 347     # timeout. Individual test cases can override this, of course.
 348     timeout = 300
 349
 350     def setUp(self):
 351         self.sparent = service.MultiService()
 352         self.sparent.startService()
 353
 354         self.stats_gatherer = None
 355         self.stats_gatherer_furl = None
 356         self.key_generator_svc = None
 357         self.key_generator_furl = None
 358
 359     def tearDown(self):
 360         log.msg("shutting down SystemTest services")
 361         d = self.sparent.stopService()
 362         d.addBoth(flush_but_dont_ignore)
 363         return d
 364
 365     def getdir(self, subdir):
 366         return os.path.join(self.basedir, subdir)
 367
 368     def add_service(self, s):
 369         s.setServiceParent(self.sparent)
 370         return s
 371
 372     def set_up_nodes(self, NUMCLIENTS=5,
 373                      use_stats_gatherer=False, use_key_generator=False):
 374         self.numclients = NUMCLIENTS
 375         iv_dir = self.getdir("introducer")
 376         if not os.path.isdir(iv_dir):
 377             fileutil.make_dirs(iv_dir)
 378             f = open(os.path.join(iv_dir, "webport"), "w")
 379             f.write("tcp:0:interface=127.0.0.1\n")
 380             f.close()
 381             if SYSTEM_TEST_CERTS:
 382                 os.mkdir(os.path.join(iv_dir, "private"))
 383                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 384                 f.write(SYSTEM_TEST_CERTS[0])
 385                 f.close()
 386         iv = IntroducerNode(basedir=iv_dir)
 387         self.introducer = self.add_service(iv)
 388         d = self.introducer.when_tub_ready()
 389         d.addCallback(self._get_introducer_web)
 390         if use_stats_gatherer:
 391             d.addCallback(self._set_up_stats_gatherer)
 392         if use_key_generator:
 393             d.addCallback(self._set_up_key_generator)
 394         d.addCallback(self._set_up_nodes_2)
 395         if use_stats_gatherer:
 396             d.addCallback(self._grab_stats)
 397         return d
 398
 399     def _get_introducer_web(self, res):
 400         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 401         self.introweb_url = f.read().strip()
 402         f.close()
 403
 404     def _set_up_stats_gatherer(self, res):
 405         statsdir = self.getdir("stats_gatherer")
 406         fileutil.make_dirs(statsdir)
 407         self.stats_gatherer_svc = StatsGathererService(statsdir)
 408         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 409         self.add_service(self.stats_gatherer_svc)
 410
 411         d = fireEventually()
 412         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 413         def check_for_furl():
 414             return os.path.exists(sgf)
 415         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 416         def get_furl(junk):
 417             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 418         d.addCallback(get_furl)
 419         return d
 420
 421     def _set_up_key_generator(self, res):
 422         kgsdir = self.getdir("key_generator")
 423         fileutil.make_dirs(kgsdir)
 424
 425         self.key_generator_svc = KeyGeneratorService(kgsdir,
 426                                                      display_furl=False,
 427                                                      default_key_size=522)
 428         self.key_generator_svc.key_generator.pool_size = 4
 429         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 430         self.add_service(self.key_generator_svc)
 431
 432         d = fireEventually()
 433         def check_for_furl():
 434             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 435         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 436         def get_furl(junk):
 437             kgf = os.path.join(kgsdir, 'key_generator.furl')
 438             self.key_generator_furl = file(kgf, 'rb').read().strip()
 439         d.addCallback(get_furl)
 440         return d
 441
 442     def _set_up_nodes_2(self, res):
 443         q = self.introducer
 444         self.introducer_furl = q.introducer_url
 445         self.clients = []
 446         basedirs = []
 447         for i in range(self.numclients):
 448             basedir = self.getdir("client%d" % i)
 449             basedirs.append(basedir)
 450             fileutil.make_dirs(os.path.join(basedir, "private"))
 451             if len(SYSTEM_TEST_CERTS) > (i+1):
 452                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 453                 f.write(SYSTEM_TEST_CERTS[i+1])
 454                 f.close()
 455
 456             def write(name, value):
 457                 open(os.path.join(basedir, name), "w").write(value+"\n")
 458             if i == 0:
 459                 # clients[0] runs a webserver and a helper, no key_generator
 460                 write("webport", "tcp:0:interface=127.0.0.1")
 461                 write("run_helper", "yes")
 462                 write("keepalive_timeout", "600")
 463             if i == 3:
 464                 # clients[3] runs a webserver and uses a helper, uses
 465                 # key_generator
 466                 write("webport", "tcp:0:interface=127.0.0.1")
 467                 write("disconnect_timeout", "1800")
 468                 if self.key_generator_furl:
 469                     kgf = "%s\n" % (self.key_generator_furl,)
 470                     write("key_generator.furl", kgf)
 471             write("introducer.furl", self.introducer_furl)
 472             if self.stats_gatherer_furl:
 473                 write("stats_gatherer.furl", self.stats_gatherer_furl)
 474
 475         # give subclasses a chance to append lines to the node's tahoe.cfg
 476         # files before they are launched.
 477         self._set_up_nodes_extra_config()
 478
 479         # start clients[0], wait for it's tub to be ready (at which point it
 480         # will have registered the helper furl).
 481         c = self.add_service(client.Client(basedir=basedirs[0]))
 482         self.clients.append(c)
 483         c.set_default_mutable_keysize(522)
 484         d = c.when_tub_ready()
 485         def _ready(res):
 486             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 487             helper_furl = f.read()
 488             f.close()
 489             self.helper_furl = helper_furl
 490             if self.numclients >= 4:
 491                 f = open(os.path.join(basedirs[3],"helper.furl"), "w")
 492                 f.write(helper_furl)
 493                 f.close()
 494
 495             # this starts the rest of the clients
 496             for i in range(1, self.numclients):
 497                 c = self.add_service(client.Client(basedir=basedirs[i]))
 498                 self.clients.append(c)
 499                 c.set_default_mutable_keysize(522)
 500             log.msg("STARTING")
 501             return self.wait_for_connections()
 502         d.addCallback(_ready)
 503         def _connected(res):
 504             log.msg("CONNECTED")
 505             # now find out where the web port was
 506             l = self.clients[0].getServiceNamed("webish").listener
 507             port = l._port.getHost().port
 508             self.webish_url = "http://localhost:%d/" % port
 509             if self.numclients >=4:
 510                 # and the helper-using webport
 511                 l = self.clients[3].getServiceNamed("webish").listener
 512                 port = l._port.getHost().port
 513                 self.helper_webish_url = "http://localhost:%d/" % port
 514         d.addCallback(_connected)
 515         return d
 516
 517     def _set_up_nodes_extra_config(self):
 518         # for overriding by subclasses
 519         pass
 520
 521     def _grab_stats(self, res):
 522         d = self.stats_gatherer.poll()
 523         return d
 524
 525     def bounce_client(self, num):
 526         c = self.clients[num]
 527         d = c.disownServiceParent()
 528         # I think windows requires a moment to let the connection really stop
 529         # and the port number made available for re-use. TODO: examine the
 530         # behavior, see if this is really the problem, see if we can do
 531         # better than blindly waiting for a second.
 532         d.addCallback(self.stall, 1.0)
 533         def _stopped(res):
 534             new_c = client.Client(basedir=self.getdir("client%d" % num))
 535             self.clients[num] = new_c
 536             new_c.set_default_mutable_keysize(522)
 537             self.add_service(new_c)
 538             return new_c.when_tub_ready()
 539         d.addCallback(_stopped)
 540         d.addCallback(lambda res: self.wait_for_connections())
 541         def _maybe_get_webport(res):
 542             if num == 0:
 543                 # now find out where the web port was
 544                 l = self.clients[0].getServiceNamed("webish").listener
 545                 port = l._port.getHost().port
 546                 self.webish_url = "http://localhost:%d/" % port
 547         d.addCallback(_maybe_get_webport)
 548         return d
 549
 550     def add_extra_node(self, client_num, helper_furl=None,
 551                        add_to_sparent=False):
 552         # usually this node is *not* parented to our self.sparent, so we can
 553         # shut it down separately from the rest, to exercise the
 554         # connection-lost code
 555         basedir = self.getdir("client%d" % client_num)
 556         if not os.path.isdir(basedir):
 557             fileutil.make_dirs(basedir)
 558         open(os.path.join(basedir, "introducer.furl"), "w").write(self.introducer_furl)
 559         if helper_furl:
 560             f = open(os.path.join(basedir, "helper.furl") ,"w")
 561             f.write(helper_furl+"\n")
 562             f.close()
 563
 564         c = client.Client(basedir=basedir)
 565         self.clients.append(c)
 566         c.set_default_mutable_keysize(522)
 567         self.numclients += 1
 568         if add_to_sparent:
 569             c.setServiceParent(self.sparent)
 570         else:
 571             c.startService()
 572         d = self.wait_for_connections()
 573         d.addCallback(lambda res: c)
 574         return d
 575
 576     def _check_connections(self):
 577         for c in self.clients:
 578             if not c.connected_to_introducer():
 579                 return False
 580             sb = c.get_storage_broker()
 581             if len(sb.get_all_servers()) != self.numclients:
 582                 return False
 583         return True
 584
 585     def wait_for_connections(self, ignored=None):
 586         # TODO: replace this with something that takes a list of peerids and
 587         # fires when they've all been heard from, instead of using a count
 588         # and a threshold
 589         return self.poll(self._check_connections, timeout=200)
 590
 591
 592 # our system test uses the same Tub certificates each time, to avoid the
 593 # overhead of key generation
 594 SYSTEM_TEST_CERTS = [
 595 """-----BEGIN CERTIFICATE-----
 596 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 597 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 598 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 599 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 600 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 601 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 602 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 603 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 604 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 605 -----END CERTIFICATE-----
 606 -----BEGIN RSA PRIVATE KEY-----
 607 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 608 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 609 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 610 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 611 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 612 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 613 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 614 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 615 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 616 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 617 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 618 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 619 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 620 -----END RSA PRIVATE KEY-----
 621 """, # 0
 622 """-----BEGIN CERTIFICATE-----
 623 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 624 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 625 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 626 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 627 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 628 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 629 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 630 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 631 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 632 -----END CERTIFICATE-----
 633 -----BEGIN RSA PRIVATE KEY-----
 634 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 635 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 636 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 637 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 638 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 639 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 640 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 641 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 642 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 643 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 644 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 645 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 646 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 647 -----END RSA PRIVATE KEY-----
 648 """, # 1
 649 """-----BEGIN CERTIFICATE-----
 650 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 651 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 652 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 653 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 654 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 655 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 656 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 657 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 658 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 659 -----END CERTIFICATE-----
 660 -----BEGIN RSA PRIVATE KEY-----
 661 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 662 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 663 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 664 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 665 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 666 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 667 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 668 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 669 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 670 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 671 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 672 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 673 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 674 -----END RSA PRIVATE KEY-----
 675 """, # 2
 676 """-----BEGIN CERTIFICATE-----
 677 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 678 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 679 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 680 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 681 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 682 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 683 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 684 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 685 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 686 -----END CERTIFICATE-----
 687 -----BEGIN RSA PRIVATE KEY-----
 688 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 689 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 690 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 691 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 692 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 693 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 694 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 695 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 696 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 697 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 698 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 699 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 700 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 701 -----END RSA PRIVATE KEY-----
 702 """, # 3
 703 """-----BEGIN CERTIFICATE-----
 704 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 705 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 706 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 707 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 708 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 709 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 710 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 711 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 712 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 713 -----END CERTIFICATE-----
 714 -----BEGIN RSA PRIVATE KEY-----
 715 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 716 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 717 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 718 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 719 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 720 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 721 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 722 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 723 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 724 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 725 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 726 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 727 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 728 -----END RSA PRIVATE KEY-----
 729 """, # 4
 730 """-----BEGIN CERTIFICATE-----
 731 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 732 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 733 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 734 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 735 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 736 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 737 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 738 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 739 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 740 -----END CERTIFICATE-----
 741 -----BEGIN RSA PRIVATE KEY-----
 742 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 743 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 744 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 745 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 746 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 747 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 748 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 749 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 750 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 751 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 752 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 753 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 754 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 755 -----END RSA PRIVATE KEY-----
 756 """, # 5
 757 """-----BEGIN CERTIFICATE-----
 758 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 759 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 760 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 761 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 762 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 763 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 764 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 765 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 766 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 767 -----END CERTIFICATE-----
 768 -----BEGIN RSA PRIVATE KEY-----
 769 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 770 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 771 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 772 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 773 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 774 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 775 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 776 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 777 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 778 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 779 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 780 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 781 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 782 -----END RSA PRIVATE KEY-----
 783 """, #6
 784 """-----BEGIN CERTIFICATE-----
 785 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 786 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 787 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 788 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 789 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 790 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 791 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 792 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 793 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 794 -----END CERTIFICATE-----
 795 -----BEGIN RSA PRIVATE KEY-----
 796 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 797 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 798 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 799 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 800 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 801 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 802 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 803 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 804 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 805 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 806 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 807 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 808 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 809 -----END RSA PRIVATE KEY-----
 810 """, #7
 811 """-----BEGIN CERTIFICATE-----
 812 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 813 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 814 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 815 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 816 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 817 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 818 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 819 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 820 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 821 -----END CERTIFICATE-----
 822 -----BEGIN RSA PRIVATE KEY-----
 823 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 824 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 825 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 826 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 827 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 828 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 829 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 830 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 831 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 832 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 833 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 834 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 835 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 836 -----END RSA PRIVATE KEY-----
 837 """, #8
 838 """-----BEGIN CERTIFICATE-----
 839 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 840 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 841 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 842 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 843 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 844 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 845 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 846 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 847 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 848 -----END CERTIFICATE-----
 849 -----BEGIN RSA PRIVATE KEY-----
 850 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 851 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 852 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 853 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 854 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 855 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 856 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 857 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 858 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 859 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 860 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 861 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 862 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 863 -----END RSA PRIVATE KEY-----
 864 """, #9
 865 """-----BEGIN CERTIFICATE-----
 866 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 867 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 868 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 869 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 870 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 871 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 872 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 873 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 874 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 875 -----END CERTIFICATE-----
 876 -----BEGIN RSA PRIVATE KEY-----
 877 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 878 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 879 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 880 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 881 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 882 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 883 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
 884 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
 885 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
 886 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
 887 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
 888 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
 889 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
 890 -----END RSA PRIVATE KEY-----
 891 """, #10
 892 """-----BEGIN CERTIFICATE-----
 893 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 894 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 895 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
 896 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
 897 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
 898 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 899 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
 900 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
 901 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
 902 -----END CERTIFICATE-----
 903 -----BEGIN RSA PRIVATE KEY-----
 904 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
 905 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
 906 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
 907 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
 908 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
 909 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
 910 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
 911 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
 912 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
 913 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
 914 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
 915 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
 916 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
 917 -----END RSA PRIVATE KEY-----
 918 """, #11
 919 ]
 920
 921 # To disable the pre-computed tub certs, uncomment this line.
 922 #SYSTEM_TEST_CERTS = []
 923
 924 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
 925
 926 class ShareManglingMixin(SystemTestMixin):
 927
 928     def setUp(self):
 929         # Set self.basedir to a temp dir which has the name of the current
 930         # test method in its name.
 931         self.basedir = self.mktemp()
 932
 933         d = defer.maybeDeferred(SystemTestMixin.setUp, self)
 934         d.addCallback(lambda x: self.set_up_nodes())
 935
 936         def _upload_a_file(ignored):
 937             cl0 = self.clients[0]
 938             # We need multiple segments to test crypttext hash trees that are
 939             # non-trivial (i.e. they have more than just one hash in them).
 940             cl0.DEFAULT_ENCODING_PARAMETERS['max_segment_size'] = 12
 941             # Tests that need to test servers of happiness using this should
 942             # set their own value for happy -- the default (7) breaks stuff.
 943             cl0.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
 944             d2 = cl0.upload(immutable.upload.Data(TEST_DATA, convergence=""))
 945             def _after_upload(u):
 946                 filecap = u.uri
 947                 self.n = self.clients[1].create_node_from_uri(filecap)
 948                 self.uri = uri.CHKFileURI.init_from_string(filecap)
 949                 return cl0.create_node_from_uri(filecap)
 950             d2.addCallback(_after_upload)
 951             return d2
 952         d.addCallback(_upload_a_file)
 953
 954         def _stash_it(filenode):
 955             self.filenode = filenode
 956         d.addCallback(_stash_it)
 957         return d
 958
 959     def find_shares(self, unused=None):
 960         """Locate shares on disk. Returns a dict that maps
 961         (clientnum,sharenum) to a string that contains the share container
 962         (copied directly from the disk, containing leases etc). You can
 963         modify this dict and then call replace_shares() to modify the shares.
 964         """
 965         shares = {} # k: (i, sharenum), v: data
 966
 967         for i, c in enumerate(self.clients):
 968             sharedir = c.getServiceNamed("storage").sharedir
 969             for (dirp, dirns, fns) in os.walk(sharedir):
 970                 for fn in fns:
 971                     try:
 972                         sharenum = int(fn)
 973                     except TypeError:
 974                         # Whoops, I guess that's not a share file then.
 975                         pass
 976                     else:
 977                         data = open(os.path.join(sharedir, dirp, fn), "rb").read()
 978                         shares[(i, sharenum)] = data
 979
 980         return shares
 981
 982     def replace_shares(self, newshares, storage_index):
 983         """Replace shares on disk. Takes a dictionary in the same form
 984         as find_shares() returns."""
 985
 986         for i, c in enumerate(self.clients):
 987             sharedir = c.getServiceNamed("storage").sharedir
 988             for (dirp, dirns, fns) in os.walk(sharedir):
 989                 for fn in fns:
 990                     try:
 991                         sharenum = int(fn)
 992                     except TypeError:
 993                         # Whoops, I guess that's not a share file then.
 994                         pass
 995                     else:
 996                         pathtosharefile = os.path.join(sharedir, dirp, fn)
 997                         os.unlink(pathtosharefile)
 998             for ((clientnum, sharenum), newdata) in newshares.iteritems():
 999                 if clientnum == i:
1000                     fullsharedirp=os.path.join(sharedir, storage_index_to_dir(storage_index))
1001                     fileutil.make_dirs(fullsharedirp)
1002                     wf = open(os.path.join(fullsharedirp, str(sharenum)), "wb")
1003                     wf.write(newdata)
1004                     wf.close()
1005
1006     def _delete_a_share(self, unused=None, sharenum=None):
1007         """ Delete one share. """
1008
1009         shares = self.find_shares()
1010         ks = shares.keys()
1011         if sharenum is not None:
1012             k = [ key for key in shares.keys() if key[1] == sharenum ][0]
1013         else:
1014             k = random.choice(ks)
1015         del shares[k]
1016         self.replace_shares(shares, storage_index=self.uri.get_storage_index())
1017
1018         return unused
1019
1020     def _corrupt_a_share(self, unused, corruptor_func, sharenum):
1021         shares = self.find_shares()
1022         ks = [ key for key in shares.keys() if key[1] == sharenum ]
1023         assert ks, (shares.keys(), sharenum)
1024         k = ks[0]
1025         shares[k] = corruptor_func(shares[k])
1026         self.replace_shares(shares, storage_index=self.uri.get_storage_index())
1027         return corruptor_func
1028
1029     def _corrupt_all_shares(self, unused, corruptor_func):
1030         """ All shares on disk will be corrupted by corruptor_func. """
1031         shares = self.find_shares()
1032         for k in shares.keys():
1033             self._corrupt_a_share(unused, corruptor_func, k[1])
1034         return corruptor_func
1035
1036     def _corrupt_a_random_share(self, unused, corruptor_func):
1037         """ Exactly one share on disk will be corrupted by corruptor_func. """
1038         shares = self.find_shares()
1039         ks = shares.keys()
1040         k = random.choice(ks)
1041         self._corrupt_a_share(unused, corruptor_func, k[1])
1042         return k[1]
1043
1044     def _count_reads(self):
1045         sum_of_read_counts = 0
1046         for thisclient in self.clients:
1047             counters = thisclient.stats_provider.get_stats()['counters']
1048             sum_of_read_counts += counters.get('storage_server.read', 0)
1049         return sum_of_read_counts
1050
1051     def _count_allocates(self):
1052         sum_of_allocate_counts = 0
1053         for thisclient in self.clients:
1054             counters = thisclient.stats_provider.get_stats()['counters']
1055             sum_of_allocate_counts += counters.get('storage_server.allocate', 0)
1056         return sum_of_allocate_counts
1057
1058     def _count_writes(self):
1059         sum_of_write_counts = 0
1060         for thisclient in self.clients:
1061             counters = thisclient.stats_provider.get_stats()['counters']
1062             sum_of_write_counts += counters.get('storage_server.write', 0)
1063         return sum_of_write_counts
1064
1065     def _download_and_check_plaintext(self, unused=None):
1066         d = download_to_data(self.n)
1067         def _after_download(result):
1068             self.failUnlessEqual(result, TEST_DATA)
1069         d.addCallback(_after_download)
1070         return d
1071
1072 class ShouldFailMixin:
1073     def shouldFail(self, expected_failure, which, substring,
1074                    callable, *args, **kwargs):
1075         """Assert that a function call raises some exception. This is a
1076         Deferred-friendly version of TestCase.assertRaises() .
1077
1078         Suppose you want to verify the following function:
1079
1080          def broken(a, b, c):
1081              if a < 0:
1082                  raise TypeError('a must not be negative')
1083              return defer.succeed(b+c)
1084
1085         You can use:
1086             d = self.shouldFail(TypeError, 'test name',
1087                                 'a must not be negative',
1088                                 broken, -4, 5, c=12)
1089         in your test method. The 'test name' string will be included in the
1090         error message, if any, because Deferred chains frequently make it
1091         difficult to tell which assertion was tripped.
1092
1093         The substring= argument, if not None, must appear inside the
1094         stringified Failure, or the test will fail.
1095         """
1096
1097         assert substring is None or isinstance(substring, str)
1098         d = defer.maybeDeferred(callable, *args, **kwargs)
1099         def done(res):
1100             if isinstance(res, failure.Failure):
1101                 res.trap(expected_failure)
1102                 if substring:
1103                     self.failUnless(substring in str(res),
1104                                     "substring '%s' not in '%s'"
1105                                     % (substring, str(res)))
1106             else:
1107                 self.fail("%s was supposed to raise %s, not get '%s'" %
1108                           (which, expected_failure, res))
1109         d.addBoth(done)
1110         return d
1111
1112 class WebErrorMixin:
1113     def explain_web_error(self, f):
1114         # an error on the server side causes the client-side getPage() to
1115         # return a failure(t.web.error.Error), and its str() doesn't show the
1116         # response body, which is where the useful information lives. Attach
1117         # this method as an errback handler, and it will reveal the hidden
1118         # message.
1119         f.trap(WebError)
1120         print "Web Error:", f.value, ":", f.value.response
1121         return f
1122
1123     def _shouldHTTPError(self, res, which, validator):
1124         if isinstance(res, failure.Failure):
1125             res.trap(WebError)
1126             return validator(res)
1127         else:
1128             self.fail("%s was supposed to Error, not get '%s'" % (which, res))
1129
1130     def shouldHTTPError(self, which,
1131                         code=None, substring=None, response_substring=None,
1132                         callable=None, *args, **kwargs):
1133         # returns a Deferred with the response body
1134         assert substring is None or isinstance(substring, str)
1135         assert callable
1136         def _validate(f):
1137             if code is not None:
1138                 self.failUnlessEqual(f.value.status, str(code))
1139             if substring:
1140                 code_string = str(f)
1141                 self.failUnless(substring in code_string,
1142                                 "substring '%s' not in '%s'"
1143                                 % (substring, code_string))
1144             response_body = f.value.response
1145             if response_substring:
1146                 self.failUnless(response_substring in response_body,
1147                                 "response substring '%s' not in '%s'"
1148                                 % (response_substring, response_body))
1149             return response_body
1150         d = defer.maybeDeferred(callable, *args, **kwargs)
1151         d.addBoth(self._shouldHTTPError, which, _validate)
1152         return d
1153
1154 class ErrorMixin(WebErrorMixin):
1155     def explain_error(self, f):
1156         if f.check(defer.FirstError):
1157             print "First Error:", f.value.subFailure
1158         return f
1159
1160 def corrupt_field(data, offset, size, debug=False):
1161     if random.random() < 0.5:
1162         newdata = testutil.flip_one_bit(data, offset, size)
1163         if debug:
1164             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1165         return newdata
1166     else:
1167         newval = testutil.insecurerandstr(size)
1168         if debug:
1169             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1170         return data[:offset]+newval+data[offset+size:]
1171
1172 def _corrupt_nothing(data, debug=False):
1173     """Leave the data pristine. """
1174     return data
1175
1176 def _corrupt_file_version_number(data, debug=False):
1177     """Scramble the file data -- the share file version number have one bit
1178     flipped or else will be changed to a random value."""
1179     return corrupt_field(data, 0x00, 4)
1180
1181 def _corrupt_size_of_file_data(data, debug=False):
1182     """Scramble the file data -- the field showing the size of the share data
1183     within the file will be set to one smaller."""
1184     return corrupt_field(data, 0x04, 4)
1185
1186 def _corrupt_sharedata_version_number(data, debug=False):
1187     """Scramble the file data -- the share data version number will have one
1188     bit flipped or else will be changed to a random value, but not 1 or 2."""
1189     return corrupt_field(data, 0x0c, 4)
1190     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1191     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1192     newsharevernum = sharevernum
1193     while newsharevernum in (1, 2):
1194         newsharevernum = random.randrange(0, 2**32)
1195     newsharevernumbytes = struct.pack(">L", newsharevernum)
1196     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1197
1198 def _corrupt_sharedata_version_number_to_plausible_version(data, debug=False):
1199     """Scramble the file data -- the share data version number will be
1200     changed to 2 if it is 1 or else to 1 if it is 2."""
1201     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1202     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1203     if sharevernum == 1:
1204         newsharevernum = 2
1205     else:
1206         newsharevernum = 1
1207     newsharevernumbytes = struct.pack(">L", newsharevernum)
1208     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1209
1210 def _corrupt_segment_size(data, debug=False):
1211     """Scramble the file data -- the field showing the size of the segment
1212     will have one bit flipped or else be changed to a random value."""
1213     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1214     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1215     if sharevernum == 1:
1216         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1217     else:
1218         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1219
1220 def _corrupt_size_of_sharedata(data, debug=False):
1221     """Scramble the file data -- the field showing the size of the data
1222     within the share data will have one bit flipped or else will be changed
1223     to a random value."""
1224     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1225     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1226     if sharevernum == 1:
1227         return corrupt_field(data, 0x0c+0x08, 4)
1228     else:
1229         return corrupt_field(data, 0x0c+0x0c, 8)
1230
1231 def _corrupt_offset_of_sharedata(data, debug=False):
1232     """Scramble the file data -- the field showing the offset of the data
1233     within the share data will have one bit flipped or else be changed to a
1234     random value."""
1235     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1236     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1237     if sharevernum == 1:
1238         return corrupt_field(data, 0x0c+0x0c, 4)
1239     else:
1240         return corrupt_field(data, 0x0c+0x14, 8)
1241
1242 def _corrupt_offset_of_ciphertext_hash_tree(data, debug=False):
1243     """Scramble the file data -- the field showing the offset of the
1244     ciphertext hash tree within the share data will have one bit flipped or
1245     else be changed to a random value.
1246     """
1247     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1248     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1249     if sharevernum == 1:
1250         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1251     else:
1252         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1253
1254 def _corrupt_offset_of_block_hashes(data, debug=False):
1255     """Scramble the file data -- the field showing the offset of the block
1256     hash tree within the share data will have one bit flipped or else will be
1257     changed to a random value."""
1258     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1259     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1260     if sharevernum == 1:
1261         return corrupt_field(data, 0x0c+0x18, 4)
1262     else:
1263         return corrupt_field(data, 0x0c+0x2c, 8)
1264
1265 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data, debug=False):
1266     """Scramble the file data -- the field showing the offset of the block
1267     hash tree within the share data will have a multiple of hash size
1268     subtracted from it, thus causing the downloader to download an incomplete
1269     crypttext hash tree."""
1270     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1271     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1272     if sharevernum == 1:
1273         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1274         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1275         newvalstr = struct.pack(">L", newval)
1276         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1277     else:
1278         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1279         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1280         newvalstr = struct.pack(">Q", newval)
1281         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1282
1283 def _corrupt_offset_of_share_hashes(data, debug=False):
1284     """Scramble the file data -- the field showing the offset of the share
1285     hash tree within the share data will have one bit flipped or else will be
1286     changed to a random value."""
1287     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1288     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1289     if sharevernum == 1:
1290         return corrupt_field(data, 0x0c+0x1c, 4)
1291     else:
1292         return corrupt_field(data, 0x0c+0x34, 8)
1293
1294 def _corrupt_offset_of_uri_extension(data, debug=False):
1295     """Scramble the file data -- the field showing the offset of the uri
1296     extension will have one bit flipped or else will be changed to a random
1297     value."""
1298     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1299     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1300     if sharevernum == 1:
1301         return corrupt_field(data, 0x0c+0x20, 4)
1302     else:
1303         return corrupt_field(data, 0x0c+0x3c, 8)
1304
1305 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1306     """Scramble the file data -- the field showing the offset of the uri
1307     extension will be set to the size of the file minus 3. This means when
1308     the client tries to read the length field from that location it will get
1309     a short read -- the result string will be only 3 bytes long, not the 4 or
1310     8 bytes necessary to do a successful struct.unpack."""
1311     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1312     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1313     # The "-0x0c" in here is to skip the server-side header in the share
1314     # file, which the client doesn't see when seeking and reading.
1315     if sharevernum == 1:
1316         if debug:
1317             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1318         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1319     else:
1320         if debug:
1321             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1322         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1323
1324 def _corrupt_mutable_share_data(data, debug=False):
1325     prefix = data[:32]
1326     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
1327     data_offset = MutableShareFile.DATA_OFFSET
1328     sharetype = data[data_offset:data_offset+1]
1329     assert sharetype == "\x00", "non-SDMF mutable shares not supported"
1330     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
1331      ig_datalen, offsets) = unpack_header(data[data_offset:])
1332     assert version == 0, "this function only handles v0 SDMF files"
1333     start = data_offset + offsets["share_data"]
1334     length = data_offset + offsets["enc_privkey"] - start
1335     return corrupt_field(data, start, length)
1336
1337 def _corrupt_share_data(data, debug=False):
1338     """Scramble the file data -- the field containing the share data itself
1339     will have one bit flipped or else will be changed to a random value."""
1340     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1341     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1342     if sharevernum == 1:
1343         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1344
1345         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1346     else:
1347         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1348
1349         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1350
1351 def _corrupt_crypttext_hash_tree(data, debug=False):
1352     """Scramble the file data -- the field containing the crypttext hash tree
1353     will have one bit flipped or else will be changed to a random value.
1354     """
1355     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1356     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1357     if sharevernum == 1:
1358         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1359         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1360     else:
1361         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1362         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1363
1364     return corrupt_field(data, 0x0c+crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset, debug=debug)
1365
1366 def _corrupt_crypttext_hash_tree_byte_x221(data, debug=False):
1367     """Scramble the file data -- the byte at offset 0x221 will have its 7th
1368     (b1) bit flipped.
1369     """
1370     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1371     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1372     if debug:
1373         log.msg("original data: %r" % (data,))
1374     return data[:0x0c+0x221] + chr(ord(data[0x0c+0x221])^0x02) + data[0x0c+0x2210+1:]
1375
1376 def _corrupt_block_hashes(data, debug=False):
1377     """Scramble the file data -- the field containing the block hash tree
1378     will have one bit flipped or else will be changed to a random value.
1379     """
1380     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1381     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1382     if sharevernum == 1:
1383         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1384         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1385     else:
1386         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1387         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1388
1389     return corrupt_field(data, 0x0c+blockhashesoffset, sharehashesoffset-blockhashesoffset)
1390
1391 def _corrupt_share_hashes(data, debug=False):
1392     """Scramble the file data -- the field containing the share hash chain
1393     will have one bit flipped or else will be changed to a random value.
1394     """
1395     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1396     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1397     if sharevernum == 1:
1398         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1399         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1400     else:
1401         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1402         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1403
1404     return corrupt_field(data, 0x0c+sharehashesoffset, uriextoffset-sharehashesoffset)
1405
1406 def _corrupt_length_of_uri_extension(data, debug=False):
1407     """Scramble the file data -- the field showing the length of the uri
1408     extension will have one bit flipped or else will be changed to a random
1409     value."""
1410     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1411     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1412     if sharevernum == 1:
1413         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1414         return corrupt_field(data, uriextoffset, 4)
1415     else:
1416         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1417         return corrupt_field(data, 0x0c+uriextoffset, 8)
1418
1419 def _corrupt_uri_extension(data, debug=False):
1420     """Scramble the file data -- the field containing the uri extension will
1421     have one bit flipped or else will be changed to a random value."""
1422     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1423     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1424     if sharevernum == 1:
1425         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1426         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1427     else:
1428         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1429         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1430
1431     return corrupt_field(data, 0x0c+uriextoffset, uriextlen)