src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IConsumer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.eventual import flushEventualQueue, fireEventually
   9 from allmydata import uri, dirnode, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IURI, IMutableFileNode, IFileNode, \
  12      FileTooLargeError, NotEnoughSharesError, ICheckable
  13 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  14      DeepCheckResults, DeepCheckAndRepairResults
  15 from allmydata.mutable.common import CorruptShareError
  16 from allmydata.storage import storage_index_to_dir
  17 from allmydata.util import hashutil, log, fileutil, pollmixin
  18 from allmydata.util.assertutil import precondition
  19 from allmydata.stats import StatsGathererService
  20 from allmydata.key_generator import KeyGeneratorService
  21 import common_util as testutil
  22 from allmydata import immutable
  23
  24
  25 def flush_but_dont_ignore(res):
  26     d = flushEventualQueue()
  27     def _done(ignored):
  28         return res
  29     d.addCallback(_done)
  30     return d
  31
  32 class FakeCHKFileNode:
  33     """I provide IFileNode, but all of my data is stored in a class-level
  34     dictionary."""
  35     implements(IFileNode)
  36     all_contents = {}
  37     bad_shares = {}
  38
  39     def __init__(self, u, thisclient):
  40         precondition(IURI.providedBy(u), u)
  41         self.client = thisclient
  42         self.my_uri = u
  43         self.storage_index = u.storage_index
  44
  45     def get_uri(self):
  46         return self.my_uri.to_string()
  47     def get_readonly_uri(self):
  48         return self.my_uri.to_string()
  49     def get_verify_cap(self):
  50         return self.my_uri.get_verify_cap()
  51     def get_repair_cap(self):
  52         return self.my_uri.get_verify_cap()
  53     def get_storage_index(self):
  54         return self.storage_index
  55
  56     def check(self, monitor, verify=False, add_lease=False):
  57         r = CheckResults(self.my_uri, self.storage_index)
  58         is_bad = self.bad_shares.get(self.storage_index, None)
  59         data = {}
  60         data["count-shares-needed"] = 3
  61         data["count-shares-expected"] = 10
  62         data["count-good-share-hosts"] = 10
  63         data["count-wrong-shares"] = 0
  64         nodeid = "\x00"*20
  65         data["list-corrupt-shares"] = []
  66         data["sharemap"] = {1: [nodeid]}
  67         data["servers-responding"] = [nodeid]
  68         data["count-recoverable-versions"] = 1
  69         data["count-unrecoverable-versions"] = 0
  70         if is_bad:
  71              r.set_healthy(False)
  72              r.set_recoverable(True)
  73              data["count-shares-good"] = 9
  74              data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
  75              r.problems = failure.Failure(CorruptShareError(is_bad))
  76         else:
  77              r.set_healthy(True)
  78              r.set_recoverable(True)
  79              data["count-shares-good"] = 10
  80              r.problems = []
  81         r.set_data(data)
  82         r.set_needs_rebalancing(False)
  83         return defer.succeed(r)
  84     def check_and_repair(self, monitor, verify=False, add_lease=False):
  85         d = self.check(verify)
  86         def _got(cr):
  87             r = CheckAndRepairResults(self.storage_index)
  88             r.pre_repair_results = r.post_repair_results = cr
  89             return r
  90         d.addCallback(_got)
  91         return d
  92
  93     def is_mutable(self):
  94         return False
  95     def is_readonly(self):
  96         return True
  97
  98     def download(self, target):
  99         if self.my_uri.to_string() not in self.all_contents:
 100             f = failure.Failure(NotEnoughSharesError())
 101             target.fail(f)
 102             return defer.fail(f)
 103         data = self.all_contents[self.my_uri.to_string()]
 104         target.open(len(data))
 105         target.write(data)
 106         target.close()
 107         return defer.maybeDeferred(target.finish)
 108     def download_to_data(self):
 109         if self.my_uri.to_string() not in self.all_contents:
 110             return defer.fail(NotEnoughSharesError())
 111         data = self.all_contents[self.my_uri.to_string()]
 112         return defer.succeed(data)
 113     def get_size(self):
 114         try:
 115             data = self.all_contents[self.my_uri.to_string()]
 116         except KeyError, le:
 117             raise NotEnoughSharesError(le)
 118         return len(data)
 119     def read(self, consumer, offset=0, size=None):
 120         d = self.download_to_data()
 121         def _got(data):
 122             start = offset
 123             if size is not None:
 124                 end = offset + size
 125             else:
 126                 end = len(data)
 127             consumer.write(data[start:end])
 128             return consumer
 129         d.addCallback(_got)
 130         return d
 131
 132 def make_chk_file_uri(size):
 133     return uri.CHKFileURI(key=os.urandom(16),
 134                           uri_extension_hash=os.urandom(32),
 135                           needed_shares=3,
 136                           total_shares=10,
 137                           size=size)
 138
 139 def create_chk_filenode(thisclient, contents):
 140     u = make_chk_file_uri(len(contents))
 141     n = FakeCHKFileNode(u, thisclient)
 142     FakeCHKFileNode.all_contents[u.to_string()] = contents
 143     return n
 144
 145
 146 class FakeMutableFileNode:
 147     """I provide IMutableFileNode, but all of my data is stored in a
 148     class-level dictionary."""
 149
 150     implements(IMutableFileNode, ICheckable)
 151     MUTABLE_SIZELIMIT = 10000
 152     all_contents = {}
 153     bad_shares = {}
 154
 155     def __init__(self, thisclient):
 156         self.client = thisclient
 157         self.my_uri = make_mutable_file_uri()
 158         self.storage_index = self.my_uri.storage_index
 159     def create(self, initial_contents, key_generator=None):
 160         if len(initial_contents) > self.MUTABLE_SIZELIMIT:
 161             raise FileTooLargeError("SDMF is limited to one segment, and "
 162                                     "%d > %d" % (len(initial_contents),
 163                                                  self.MUTABLE_SIZELIMIT))
 164         self.all_contents[self.storage_index] = initial_contents
 165         return defer.succeed(self)
 166     def init_from_uri(self, myuri):
 167         self.my_uri = IURI(myuri)
 168         self.storage_index = self.my_uri.storage_index
 169         return self
 170     def get_uri(self):
 171         return self.my_uri.to_string()
 172     def get_readonly(self):
 173         return self.my_uri.get_readonly()
 174     def get_readonly_uri(self):
 175         return self.my_uri.get_readonly().to_string()
 176     def get_verify_cap(self):
 177         return self.my_uri.get_verify_cap()
 178     def is_readonly(self):
 179         return self.my_uri.is_readonly()
 180     def is_mutable(self):
 181         return self.my_uri.is_mutable()
 182     def get_writekey(self):
 183         return "\x00"*16
 184     def get_size(self):
 185         return "?" # TODO: see mutable.MutableFileNode.get_size
 186     def get_size_of_best_version(self):
 187         return defer.succeed(len(self.all_contents[self.storage_index]))
 188
 189     def get_storage_index(self):
 190         return self.storage_index
 191
 192     def check(self, monitor, verify=False, add_lease=False):
 193         r = CheckResults(self.my_uri, self.storage_index)
 194         is_bad = self.bad_shares.get(self.storage_index, None)
 195         data = {}
 196         data["count-shares-needed"] = 3
 197         data["count-shares-expected"] = 10
 198         data["count-good-share-hosts"] = 10
 199         data["count-wrong-shares"] = 0
 200         data["list-corrupt-shares"] = []
 201         nodeid = "\x00"*20
 202         data["sharemap"] = {"seq1-abcd-sh0": [nodeid]}
 203         data["servers-responding"] = [nodeid]
 204         data["count-recoverable-versions"] = 1
 205         data["count-unrecoverable-versions"] = 0
 206         if is_bad:
 207              r.set_healthy(False)
 208              r.set_recoverable(True)
 209              data["count-shares-good"] = 9
 210              r.problems = failure.Failure(CorruptShareError("peerid",
 211                                                             0, # shnum
 212                                                             is_bad))
 213         else:
 214              r.set_healthy(True)
 215              r.set_recoverable(True)
 216              data["count-shares-good"] = 10
 217              r.problems = []
 218         r.set_data(data)
 219         r.set_needs_rebalancing(False)
 220         return defer.succeed(r)
 221
 222     def check_and_repair(self, monitor, verify=False, add_lease=False):
 223         d = self.check(verify)
 224         def _got(cr):
 225             r = CheckAndRepairResults(self.storage_index)
 226             r.pre_repair_results = r.post_repair_results = cr
 227             return r
 228         d.addCallback(_got)
 229         return d
 230
 231     def deep_check(self, verify=False, add_lease=False):
 232         d = self.check(verify)
 233         def _done(r):
 234             dr = DeepCheckResults(self.storage_index)
 235             dr.add_check(r, [])
 236             return dr
 237         d.addCallback(_done)
 238         return d
 239
 240     def deep_check_and_repair(self, verify=False, add_lease=False):
 241         d = self.check_and_repair(verify)
 242         def _done(r):
 243             dr = DeepCheckAndRepairResults(self.storage_index)
 244             dr.add_check(r, [])
 245             return dr
 246         d.addCallback(_done)
 247         return d
 248
 249     def download_best_version(self):
 250         return defer.succeed(self.all_contents[self.storage_index])
 251     def overwrite(self, new_contents):
 252         if len(new_contents) > self.MUTABLE_SIZELIMIT:
 253             raise FileTooLargeError("SDMF is limited to one segment, and "
 254                                     "%d > %d" % (len(new_contents),
 255                                                  self.MUTABLE_SIZELIMIT))
 256         assert not self.is_readonly()
 257         self.all_contents[self.storage_index] = new_contents
 258         return defer.succeed(None)
 259     def modify(self, modifier):
 260         # this does not implement FileTooLargeError, but the real one does
 261         return defer.maybeDeferred(self._modify, modifier)
 262     def _modify(self, modifier):
 263         assert not self.is_readonly()
 264         old_contents = self.all_contents[self.storage_index]
 265         self.all_contents[self.storage_index] = modifier(old_contents, None, True)
 266         return None
 267
 268     def download(self, target):
 269         if self.storage_index not in self.all_contents:
 270             f = failure.Failure(NotEnoughSharesError())
 271             target.fail(f)
 272             return defer.fail(f)
 273         data = self.all_contents[self.storage_index]
 274         target.open(len(data))
 275         target.write(data)
 276         target.close()
 277         return defer.maybeDeferred(target.finish)
 278     def download_to_data(self):
 279         if self.storage_index not in self.all_contents:
 280             return defer.fail(NotEnoughSharesError())
 281         data = self.all_contents[self.storage_index]
 282         return defer.succeed(data)
 283
 284 def make_mutable_file_uri():
 285     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 286                                    fingerprint=os.urandom(32))
 287 def make_verifier_uri():
 288     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 289                               fingerprint=os.urandom(32))
 290
 291 class FakeDirectoryNode(dirnode.NewDirectoryNode):
 292     """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
 293     backing store, so it doesn't go to the grid. The child data is still
 294     encrypted and serialized, so this isn't useful for tests that want to
 295     look inside the dirnodes and check their contents.
 296     """
 297     filenode_class = FakeMutableFileNode
 298
 299 class LoggingServiceParent(service.MultiService):
 300     def log(self, *args, **kwargs):
 301         return log.msg(*args, **kwargs)
 302
 303
 304 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 305
 306     # SystemTestMixin tests tend to be a lot of work, and we have a few
 307     # buildslaves that are pretty slow, and every once in a while these tests
 308     # run up against the default 120 second timeout. So increase the default
 309     # timeout. Individual test cases can override this, of course.
 310     timeout = 300
 311
 312     def setUp(self):
 313         self.sparent = service.MultiService()
 314         self.sparent.startService()
 315
 316         self.stats_gatherer = None
 317         self.stats_gatherer_furl = None
 318         self.key_generator_svc = None
 319         self.key_generator_furl = None
 320
 321     def tearDown(self):
 322         log.msg("shutting down SystemTest services")
 323         d = self.sparent.stopService()
 324         d.addBoth(flush_but_dont_ignore)
 325         return d
 326
 327     def getdir(self, subdir):
 328         return os.path.join(self.basedir, subdir)
 329
 330     def add_service(self, s):
 331         s.setServiceParent(self.sparent)
 332         return s
 333
 334     def set_up_nodes(self, NUMCLIENTS=5,
 335                      use_stats_gatherer=False, use_key_generator=False):
 336         self.numclients = NUMCLIENTS
 337         iv_dir = self.getdir("introducer")
 338         if not os.path.isdir(iv_dir):
 339             fileutil.make_dirs(iv_dir)
 340             f = open(os.path.join(iv_dir, "webport"), "w")
 341             f.write("tcp:0:interface=127.0.0.1\n")
 342             f.close()
 343             if SYSTEM_TEST_CERTS:
 344                 os.mkdir(os.path.join(iv_dir, "private"))
 345                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 346                 f.write(SYSTEM_TEST_CERTS[0])
 347                 f.close()
 348         iv = IntroducerNode(basedir=iv_dir)
 349         self.introducer = self.add_service(iv)
 350         d = self.introducer.when_tub_ready()
 351         d.addCallback(self._get_introducer_web)
 352         if use_stats_gatherer:
 353             d.addCallback(self._set_up_stats_gatherer)
 354         if use_key_generator:
 355             d.addCallback(self._set_up_key_generator)
 356         d.addCallback(self._set_up_nodes_2)
 357         if use_stats_gatherer:
 358             d.addCallback(self._grab_stats)
 359         return d
 360
 361     def _get_introducer_web(self, res):
 362         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 363         self.introweb_url = f.read().strip()
 364         f.close()
 365
 366     def _set_up_stats_gatherer(self, res):
 367         statsdir = self.getdir("stats_gatherer")
 368         fileutil.make_dirs(statsdir)
 369         self.stats_gatherer_svc = StatsGathererService(statsdir)
 370         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 371         self.add_service(self.stats_gatherer_svc)
 372
 373         d = fireEventually()
 374         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 375         def check_for_furl():
 376             return os.path.exists(sgf)
 377         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 378         def get_furl(junk):
 379             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 380         d.addCallback(get_furl)
 381         return d
 382
 383     def _set_up_key_generator(self, res):
 384         kgsdir = self.getdir("key_generator")
 385         fileutil.make_dirs(kgsdir)
 386
 387         self.key_generator_svc = KeyGeneratorService(kgsdir, display_furl=False)
 388         self.key_generator_svc.key_generator.pool_size = 4
 389         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 390         self.add_service(self.key_generator_svc)
 391
 392         d = fireEventually()
 393         def check_for_furl():
 394             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 395         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 396         def get_furl(junk):
 397             kgf = os.path.join(kgsdir, 'key_generator.furl')
 398             self.key_generator_furl = file(kgf, 'rb').read().strip()
 399         d.addCallback(get_furl)
 400         return d
 401
 402     def _set_up_nodes_2(self, res):
 403         q = self.introducer
 404         self.introducer_furl = q.introducer_url
 405         self.clients = []
 406         basedirs = []
 407         for i in range(self.numclients):
 408             basedir = self.getdir("client%d" % i)
 409             basedirs.append(basedir)
 410             fileutil.make_dirs(os.path.join(basedir, "private"))
 411             if len(SYSTEM_TEST_CERTS) > (i+1):
 412                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 413                 f.write(SYSTEM_TEST_CERTS[i+1])
 414                 f.close()
 415
 416             def write(name, value):
 417                 open(os.path.join(basedir, name), "w").write(value+"\n")
 418             if i == 0:
 419                 # clients[0] runs a webserver and a helper, no key_generator
 420                 write("webport", "tcp:0:interface=127.0.0.1")
 421                 write("run_helper", "yes")
 422                 write("keepalive_timeout", "600")
 423             if i == 3:
 424                 # clients[3] runs a webserver and uses a helper, uses
 425                 # key_generator
 426                 write("webport", "tcp:0:interface=127.0.0.1")
 427                 write("disconnect_timeout", "1800")
 428                 if self.key_generator_furl:
 429                     kgf = "%s\n" % (self.key_generator_furl,)
 430                     write("key_generator.furl", kgf)
 431             write("introducer.furl", self.introducer_furl)
 432             if self.stats_gatherer_furl:
 433                 write("stats_gatherer.furl", self.stats_gatherer_furl)
 434
 435         # give subclasses a chance to append liens to the node's tahoe.cfg
 436         # files before they are launched.
 437         self._set_up_nodes_extra_config()
 438
 439         # start clients[0], wait for it's tub to be ready (at which point it
 440         # will have registered the helper furl).
 441         c = self.add_service(client.Client(basedir=basedirs[0]))
 442         self.clients.append(c)
 443         d = c.when_tub_ready()
 444         def _ready(res):
 445             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 446             helper_furl = f.read()
 447             f.close()
 448             self.helper_furl = helper_furl
 449             if self.numclients >= 4:
 450                 f = open(os.path.join(basedirs[3],"helper.furl"), "w")
 451                 f.write(helper_furl)
 452                 f.close()
 453
 454             # this starts the rest of the clients
 455             for i in range(1, self.numclients):
 456                 c = self.add_service(client.Client(basedir=basedirs[i]))
 457                 self.clients.append(c)
 458             log.msg("STARTING")
 459             return self.wait_for_connections()
 460         d.addCallback(_ready)
 461         def _connected(res):
 462             log.msg("CONNECTED")
 463             # now find out where the web port was
 464             l = self.clients[0].getServiceNamed("webish").listener
 465             port = l._port.getHost().port
 466             self.webish_url = "http://localhost:%d/" % port
 467             if self.numclients >=4:
 468                 # and the helper-using webport
 469                 l = self.clients[3].getServiceNamed("webish").listener
 470                 port = l._port.getHost().port
 471                 self.helper_webish_url = "http://localhost:%d/" % port
 472         d.addCallback(_connected)
 473         return d
 474
 475     def _set_up_nodes_extra_config(self):
 476         # for overriding by subclasses
 477         pass
 478
 479     def _grab_stats(self, res):
 480         d = self.stats_gatherer.poll()
 481         return d
 482
 483     def bounce_client(self, num):
 484         c = self.clients[num]
 485         d = c.disownServiceParent()
 486         # I think windows requires a moment to let the connection really stop
 487         # and the port number made available for re-use. TODO: examine the
 488         # behavior, see if this is really the problem, see if we can do
 489         # better than blindly waiting for a second.
 490         d.addCallback(self.stall, 1.0)
 491         def _stopped(res):
 492             new_c = client.Client(basedir=self.getdir("client%d" % num))
 493             self.clients[num] = new_c
 494             self.add_service(new_c)
 495             return new_c.when_tub_ready()
 496         d.addCallback(_stopped)
 497         d.addCallback(lambda res: self.wait_for_connections())
 498         def _maybe_get_webport(res):
 499             if num == 0:
 500                 # now find out where the web port was
 501                 l = self.clients[0].getServiceNamed("webish").listener
 502                 port = l._port.getHost().port
 503                 self.webish_url = "http://localhost:%d/" % port
 504         d.addCallback(_maybe_get_webport)
 505         return d
 506
 507     def add_extra_node(self, client_num, helper_furl=None,
 508                        add_to_sparent=False):
 509         # usually this node is *not* parented to our self.sparent, so we can
 510         # shut it down separately from the rest, to exercise the
 511         # connection-lost code
 512         basedir = self.getdir("client%d" % client_num)
 513         if not os.path.isdir(basedir):
 514             fileutil.make_dirs(basedir)
 515         open(os.path.join(basedir, "introducer.furl"), "w").write(self.introducer_furl)
 516         if helper_furl:
 517             f = open(os.path.join(basedir, "helper.furl") ,"w")
 518             f.write(helper_furl+"\n")
 519             f.close()
 520
 521         c = client.Client(basedir=basedir)
 522         self.clients.append(c)
 523         self.numclients += 1
 524         if add_to_sparent:
 525             c.setServiceParent(self.sparent)
 526         else:
 527             c.startService()
 528         d = self.wait_for_connections()
 529         d.addCallback(lambda res: c)
 530         return d
 531
 532     def _check_connections(self):
 533         for c in self.clients:
 534             ic = c.introducer_client
 535             if not ic.connected_to_introducer():
 536                 return False
 537             if len(ic.get_all_peerids()) != self.numclients:
 538                 return False
 539         return True
 540
 541     def wait_for_connections(self, ignored=None):
 542         # TODO: replace this with something that takes a list of peerids and
 543         # fires when they've all been heard from, instead of using a count
 544         # and a threshold
 545         return self.poll(self._check_connections, timeout=200)
 546
 547
 548 # our system test uses the same Tub certificates each time, to avoid the
 549 # overhead of key generation
 550 SYSTEM_TEST_CERTS = [
 551 """-----BEGIN CERTIFICATE-----
 552 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 553 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 554 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 555 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 556 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 557 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 558 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 559 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 560 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 561 -----END CERTIFICATE-----
 562 -----BEGIN RSA PRIVATE KEY-----
 563 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 564 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 565 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 566 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 567 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 568 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 569 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 570 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 571 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 572 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 573 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 574 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 575 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 576 -----END RSA PRIVATE KEY-----
 577 """, # 0
 578 """-----BEGIN CERTIFICATE-----
 579 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 580 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 581 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 582 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 583 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 584 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 585 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 586 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 587 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 588 -----END CERTIFICATE-----
 589 -----BEGIN RSA PRIVATE KEY-----
 590 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 591 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 592 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 593 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 594 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 595 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 596 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 597 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 598 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 599 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 600 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 601 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 602 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 603 -----END RSA PRIVATE KEY-----
 604 """, # 1
 605 """-----BEGIN CERTIFICATE-----
 606 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 607 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 608 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 609 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 610 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 611 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 612 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 613 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 614 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 615 -----END CERTIFICATE-----
 616 -----BEGIN RSA PRIVATE KEY-----
 617 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 618 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 619 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 620 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 621 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 622 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 623 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 624 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 625 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 626 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 627 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 628 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 629 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 630 -----END RSA PRIVATE KEY-----
 631 """, # 2
 632 """-----BEGIN CERTIFICATE-----
 633 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 634 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 635 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 636 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 637 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 638 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 639 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 640 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 641 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 642 -----END CERTIFICATE-----
 643 -----BEGIN RSA PRIVATE KEY-----
 644 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 645 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 646 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 647 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 648 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 649 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 650 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 651 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 652 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 653 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 654 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 655 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 656 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 657 -----END RSA PRIVATE KEY-----
 658 """, # 3
 659 """-----BEGIN CERTIFICATE-----
 660 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 661 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 662 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 663 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 664 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 665 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 666 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 667 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 668 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 669 -----END CERTIFICATE-----
 670 -----BEGIN RSA PRIVATE KEY-----
 671 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 672 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 673 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 674 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 675 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 676 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 677 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 678 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 679 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 680 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 681 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 682 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 683 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 684 -----END RSA PRIVATE KEY-----
 685 """, # 4
 686 """-----BEGIN CERTIFICATE-----
 687 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 688 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 689 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 690 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 691 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 692 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 693 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 694 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 695 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 696 -----END CERTIFICATE-----
 697 -----BEGIN RSA PRIVATE KEY-----
 698 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 699 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 700 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 701 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 702 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 703 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 704 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 705 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 706 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 707 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 708 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 709 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 710 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 711 -----END RSA PRIVATE KEY-----
 712 """, # 5
 713 """-----BEGIN CERTIFICATE-----
 714 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 715 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 716 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 717 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 718 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 719 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 720 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 721 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 722 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 723 -----END CERTIFICATE-----
 724 -----BEGIN RSA PRIVATE KEY-----
 725 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 726 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 727 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 728 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 729 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 730 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 731 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 732 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 733 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 734 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 735 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 736 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 737 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 738 -----END RSA PRIVATE KEY-----
 739 """, #6
 740 """-----BEGIN CERTIFICATE-----
 741 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 742 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 743 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 744 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 745 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 746 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 747 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 748 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 749 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 750 -----END CERTIFICATE-----
 751 -----BEGIN RSA PRIVATE KEY-----
 752 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 753 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 754 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 755 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 756 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 757 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 758 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 759 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 760 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 761 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 762 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 763 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 764 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 765 -----END RSA PRIVATE KEY-----
 766 """, #7
 767 """-----BEGIN CERTIFICATE-----
 768 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 769 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 770 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 771 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 772 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 773 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 774 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 775 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 776 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 777 -----END CERTIFICATE-----
 778 -----BEGIN RSA PRIVATE KEY-----
 779 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 780 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 781 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 782 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 783 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 784 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 785 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 786 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 787 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 788 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 789 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 790 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 791 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 792 -----END RSA PRIVATE KEY-----
 793 """, #8
 794 """-----BEGIN CERTIFICATE-----
 795 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 796 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 797 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 798 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 799 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 800 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 801 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 802 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 803 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 804 -----END CERTIFICATE-----
 805 -----BEGIN RSA PRIVATE KEY-----
 806 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 807 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 808 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 809 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 810 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 811 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 812 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 813 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 814 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 815 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 816 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 817 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 818 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 819 -----END RSA PRIVATE KEY-----
 820 """, #9
 821 """-----BEGIN CERTIFICATE-----
 822 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 823 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 824 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 825 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 826 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 827 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 828 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 829 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 830 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 831 -----END CERTIFICATE-----
 832 -----BEGIN RSA PRIVATE KEY-----
 833 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 834 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 835 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 836 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 837 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 838 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 839 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
 840 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
 841 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
 842 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
 843 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
 844 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
 845 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
 846 -----END RSA PRIVATE KEY-----
 847 """, #10
 848 """-----BEGIN CERTIFICATE-----
 849 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 850 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 851 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
 852 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
 853 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
 854 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 855 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
 856 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
 857 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
 858 -----END CERTIFICATE-----
 859 -----BEGIN RSA PRIVATE KEY-----
 860 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
 861 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
 862 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
 863 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
 864 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
 865 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
 866 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
 867 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
 868 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
 869 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
 870 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
 871 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
 872 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
 873 -----END RSA PRIVATE KEY-----
 874 """, #11
 875 ]
 876
 877 # To disable the pre-computed tub certs, uncomment this line.
 878 #SYSTEM_TEST_CERTS = []
 879
 880 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
 881
 882 class ShareManglingMixin(SystemTestMixin):
 883
 884     def setUp(self):
 885         # Set self.basedir to a temp dir which has the name of the current
 886         # test method in its name.
 887         self.basedir = self.mktemp()
 888
 889         d = defer.maybeDeferred(SystemTestMixin.setUp, self)
 890         d.addCallback(lambda x: self.set_up_nodes())
 891
 892         def _upload_a_file(ignored):
 893             cl0 = self.clients[0]
 894             # We need multiple segments to test crypttext hash trees that are
 895             # non-trivial (i.e. they have more than just one hash in them).
 896             cl0.DEFAULT_ENCODING_PARAMETERS['max_segment_size'] = 12
 897             d2 = cl0.upload(immutable.upload.Data(TEST_DATA, convergence=""))
 898             def _after_upload(u):
 899                 self.uri = IURI(u.uri)
 900                 return cl0.create_node_from_uri(self.uri)
 901             d2.addCallback(_after_upload)
 902             return d2
 903         d.addCallback(_upload_a_file)
 904
 905         def _stash_it(filenode):
 906             self.filenode = filenode
 907         d.addCallback(_stash_it)
 908         return d
 909
 910     def find_shares(self, unused=None):
 911         """Locate shares on disk. Returns a dict that maps
 912         (clientnum,sharenum) to a string that contains the share container
 913         (copied directly from the disk, containing leases etc). You can
 914         modify this dict and then call replace_shares() to modify the shares.
 915         """
 916         shares = {} # k: (i, sharenum), v: data
 917
 918         for i, c in enumerate(self.clients):
 919             sharedir = c.getServiceNamed("storage").sharedir
 920             for (dirp, dirns, fns) in os.walk(sharedir):
 921                 for fn in fns:
 922                     try:
 923                         sharenum = int(fn)
 924                     except TypeError:
 925                         # Whoops, I guess that's not a share file then.
 926                         pass
 927                     else:
 928                         data = open(os.path.join(sharedir, dirp, fn), "rb").read()
 929                         shares[(i, sharenum)] = data
 930
 931         return shares
 932
 933     def replace_shares(self, newshares, storage_index):
 934         """Replace shares on disk. Takes a dictionary in the same form
 935         as find_shares() returns."""
 936
 937         for i, c in enumerate(self.clients):
 938             sharedir = c.getServiceNamed("storage").sharedir
 939             for (dirp, dirns, fns) in os.walk(sharedir):
 940                 for fn in fns:
 941                     try:
 942                         sharenum = int(fn)
 943                     except TypeError:
 944                         # Whoops, I guess that's not a share file then.
 945                         pass
 946                     else:
 947                         pathtosharefile = os.path.join(sharedir, dirp, fn)
 948                         os.unlink(pathtosharefile)
 949             for ((clientnum, sharenum), newdata) in newshares.iteritems():
 950                 if clientnum == i:
 951                     fullsharedirp=os.path.join(sharedir, storage_index_to_dir(storage_index))
 952                     fileutil.make_dirs(fullsharedirp)
 953                     wf = open(os.path.join(fullsharedirp, str(sharenum)), "wb")
 954                     wf.write(newdata)
 955                     wf.close()
 956
 957     def _delete_a_share(self, unused=None, sharenum=None):
 958         """ Delete one share. """
 959
 960         shares = self.find_shares()
 961         ks = shares.keys()
 962         if sharenum is not None:
 963             k = [ key for key in shares.keys() if key[1] == sharenum ][0]
 964         else:
 965             k = random.choice(ks)
 966         del shares[k]
 967         self.replace_shares(shares, storage_index=self.uri.storage_index)
 968
 969         return unused
 970
 971     def _corrupt_a_share(self, unused, corruptor_func, sharenum):
 972         shares = self.find_shares()
 973         ks = [ key for key in shares.keys() if key[1] == sharenum ]
 974         assert ks, (shares.keys(), sharenum)
 975         k = ks[0]
 976         shares[k] = corruptor_func(shares[k])
 977         self.replace_shares(shares, storage_index=self.uri.storage_index)
 978         return corruptor_func
 979
 980     def _corrupt_all_shares(self, unused, corruptor_func):
 981         """ All shares on disk will be corrupted by corruptor_func. """
 982         shares = self.find_shares()
 983         for k in shares.keys():
 984             self._corrupt_a_share(unused, corruptor_func, k[1])
 985         return corruptor_func
 986
 987     def _corrupt_a_random_share(self, unused, corruptor_func):
 988         """ Exactly one share on disk will be corrupted by corruptor_func. """
 989         shares = self.find_shares()
 990         ks = shares.keys()
 991         k = random.choice(ks)
 992         self._corrupt_a_share(unused, corruptor_func, k[1])
 993         return k[1]
 994
 995     def _count_reads(self):
 996         sum_of_read_counts = 0
 997         for thisclient in self.clients:
 998             counters = thisclient.stats_provider.get_stats()['counters']
 999             sum_of_read_counts += counters.get('storage_server.read', 0)
1000         return sum_of_read_counts
1001
1002     def _count_allocates(self):
1003         sum_of_allocate_counts = 0
1004         for thisclient in self.clients:
1005             counters = thisclient.stats_provider.get_stats()['counters']
1006             sum_of_allocate_counts += counters.get('storage_server.allocate', 0)
1007         return sum_of_allocate_counts
1008
1009     def _count_writes(self):
1010         sum_of_write_counts = 0
1011         for thisclient in self.clients:
1012             counters = thisclient.stats_provider.get_stats()['counters']
1013             sum_of_write_counts += counters.get('storage_server.write', 0)
1014         return sum_of_write_counts
1015
1016     def _download_and_check_plaintext(self, unused=None):
1017         self.downloader = self.clients[1].getServiceNamed("downloader")
1018         d = self.downloader.download_to_data(self.uri)
1019
1020         def _after_download(result):
1021             self.failUnlessEqual(result, TEST_DATA)
1022         d.addCallback(_after_download)
1023         return d
1024
1025 class ShouldFailMixin:
1026     def shouldFail(self, expected_failure, which, substring,
1027                    callable, *args, **kwargs):
1028         """Assert that a function call raises some exception. This is a
1029         Deferred-friendly version of TestCase.assertRaises() .
1030
1031         Suppose you want to verify the following function:
1032
1033          def broken(a, b, c):
1034              if a < 0:
1035                  raise TypeError('a must not be negative')
1036              return defer.succeed(b+c)
1037
1038         You can use:
1039             d = self.shouldFail(TypeError, 'test name',
1040                                 'a must not be negative',
1041                                 broken, -4, 5, c=12)
1042         in your test method. The 'test name' string will be included in the
1043         error message, if any, because Deferred chains frequently make it
1044         difficult to tell which assertion was tripped.
1045
1046         The substring= argument, if not None, must appear inside the
1047         stringified Failure, or the test will fail.
1048         """
1049
1050         assert substring is None or isinstance(substring, str)
1051         d = defer.maybeDeferred(callable, *args, **kwargs)
1052         def done(res):
1053             if isinstance(res, failure.Failure):
1054                 res.trap(expected_failure)
1055                 if substring:
1056                     self.failUnless(substring in str(res),
1057                                     "substring '%s' not in '%s'"
1058                                     % (substring, str(res)))
1059             else:
1060                 self.fail("%s was supposed to raise %s, not get '%s'" %
1061                           (which, expected_failure, res))
1062         d.addBoth(done)
1063         return d
1064
1065 class WebErrorMixin:
1066     def explain_web_error(self, f):
1067         # an error on the server side causes the client-side getPage() to
1068         # return a failure(t.web.error.Error), and its str() doesn't show the
1069         # response body, which is where the useful information lives. Attach
1070         # this method as an errback handler, and it will reveal the hidden
1071         # message.
1072         f.trap(WebError)
1073         print "Web Error:", f.value, ":", f.value.response
1074         return f
1075 class ErrorMixin(WebErrorMixin):
1076     def explain_error(self, f):
1077         if f.check(defer.FirstError):
1078             print "First Error:", f.value.subFailure
1079         return f
1080
1081 class MemoryConsumer:
1082     implements(IConsumer)
1083     def __init__(self):
1084         self.chunks = []
1085         self.done = False
1086     def registerProducer(self, p, streaming):
1087         if streaming:
1088             # call resumeProducing once to start things off
1089             p.resumeProducing()
1090         else:
1091             while not self.done:
1092                 p.resumeProducing()
1093     def write(self, data):
1094         self.chunks.append(data)
1095     def unregisterProducer(self):
1096         self.done = True
1097
1098 def download_to_data(n, offset=0, size=None):
1099     d = n.read(MemoryConsumer(), offset, size)
1100     d.addCallback(lambda mc: "".join(mc.chunks))
1101     return d
1102
1103 def corrupt_field(data, offset, size, debug=False):
1104     if random.random() < 0.5:
1105         newdata = testutil.flip_one_bit(data, offset, size)
1106         if debug:
1107             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1108         return newdata
1109     else:
1110         newval = testutil.insecurerandstr(size)
1111         if debug:
1112             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1113         return data[:offset]+newval+data[offset+size:]
1114
1115 def _corrupt_nothing(data):
1116     """ Leave the data pristine. """
1117     return data
1118
1119 def _corrupt_file_version_number(data):
1120     """ Scramble the file data -- the share file version number have one bit flipped or else
1121     will be changed to a random value."""
1122     return corrupt_field(data, 0x00, 4)
1123
1124 def _corrupt_size_of_file_data(data):
1125     """ Scramble the file data -- the field showing the size of the share data within the file
1126     will be set to one smaller. """
1127     return corrupt_field(data, 0x04, 4)
1128
1129 def _corrupt_sharedata_version_number(data):
1130     """ Scramble the file data -- the share data version number will have one bit flipped or
1131     else will be changed to a random value, but not 1 or 2."""
1132     return corrupt_field(data, 0x0c, 4)
1133     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1134     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1135     newsharevernum = sharevernum
1136     while newsharevernum in (1, 2):
1137         newsharevernum = random.randrange(0, 2**32)
1138     newsharevernumbytes = struct.pack(">L", newsharevernum)
1139     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1140
1141 def _corrupt_sharedata_version_number_to_plausible_version(data):
1142     """ Scramble the file data -- the share data version number will
1143     be changed to 2 if it is 1 or else to 1 if it is 2."""
1144     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1145     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1146     if sharevernum == 1:
1147         newsharevernum = 2
1148     else:
1149         newsharevernum = 1
1150     newsharevernumbytes = struct.pack(">L", newsharevernum)
1151     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1152
1153 def _corrupt_segment_size(data):
1154     """ Scramble the file data -- the field showing the size of the segment will have one
1155     bit flipped or else be changed to a random value. """
1156     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1157     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1158     if sharevernum == 1:
1159         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1160     else:
1161         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1162
1163 def _corrupt_size_of_sharedata(data):
1164     """ Scramble the file data -- the field showing the size of the data within the share
1165     data will have one bit flipped or else will be changed to a random value. """
1166     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1167     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1168     if sharevernum == 1:
1169         return corrupt_field(data, 0x0c+0x08, 4)
1170     else:
1171         return corrupt_field(data, 0x0c+0x0c, 8)
1172
1173 def _corrupt_offset_of_sharedata(data):
1174     """ Scramble the file data -- the field showing the offset of the data within the share
1175     data will have one bit flipped or else be changed to a random value. """
1176     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1177     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1178     if sharevernum == 1:
1179         return corrupt_field(data, 0x0c+0x0c, 4)
1180     else:
1181         return corrupt_field(data, 0x0c+0x14, 8)
1182
1183 def _corrupt_offset_of_ciphertext_hash_tree(data):
1184     """ Scramble the file data -- the field showing the offset of the ciphertext hash tree
1185     within the share data will have one bit flipped or else be changed to a random value.
1186     """
1187     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1188     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1189     if sharevernum == 1:
1190         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1191     else:
1192         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1193
1194 def _corrupt_offset_of_block_hashes(data):
1195     """ Scramble the file data -- the field showing the offset of the block hash tree within
1196     the share data will have one bit flipped or else will be changed to a random value. """
1197     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1198     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1199     if sharevernum == 1:
1200         return corrupt_field(data, 0x0c+0x18, 4)
1201     else:
1202         return corrupt_field(data, 0x0c+0x2c, 8)
1203
1204 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data):
1205     """ Scramble the file data -- the field showing the offset of the block hash tree within the
1206     share data will have a multiple of hash size subtracted from it, thus causing the downloader
1207     to download an incomplete crypttext hash tree."""
1208     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1209     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1210     if sharevernum == 1:
1211         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1212         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1213         newvalstr = struct.pack(">L", newval)
1214         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1215     else:
1216         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1217         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1218         newvalstr = struct.pack(">Q", newval)
1219         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1220
1221 def _corrupt_offset_of_share_hashes(data):
1222     """ Scramble the file data -- the field showing the offset of the share hash tree within
1223     the share data will have one bit flipped or else will be changed to a random value. """
1224     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1225     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1226     if sharevernum == 1:
1227         return corrupt_field(data, 0x0c+0x1c, 4)
1228     else:
1229         return corrupt_field(data, 0x0c+0x34, 8)
1230
1231 def _corrupt_offset_of_uri_extension(data):
1232     """ Scramble the file data -- the field showing the offset of the uri extension will
1233     have one bit flipped or else will be changed to a random value. """
1234     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1235     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1236     if sharevernum == 1:
1237         return corrupt_field(data, 0x0c+0x20, 4)
1238     else:
1239         return corrupt_field(data, 0x0c+0x3c, 8)
1240
1241 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1242     """ Scramble the file data -- the field showing the offset of the uri extension will be set
1243     to the size of the file minus 3.  This means when the client tries to read the length field
1244     from that location it will get a short read -- the result string will be only 3 bytes long,
1245     not the 4 or 8 bytes necessary to do a successful struct.unpack."""
1246     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1247     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1248     # The "-0x0c" in here is to skip the server-side header in the share file, which the client doesn't see when seeking and reading.
1249     if sharevernum == 1:
1250         if debug:
1251             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1252         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1253     else:
1254         if debug:
1255             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1256         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1257
1258 def _corrupt_share_data(data):
1259     """ Scramble the file data -- the field containing the share data itself will have one
1260     bit flipped or else will be changed to a random value. """
1261     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1262     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1263     if sharevernum == 1:
1264         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1265
1266         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1267     else:
1268         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1269
1270         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1271
1272 def _corrupt_crypttext_hash_tree(data):
1273     """ Scramble the file data -- the field containing the crypttext hash tree will have one
1274     bit flipped or else will be changed to a random value.
1275     """
1276     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1277     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1278     if sharevernum == 1:
1279         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1280         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1281     else:
1282         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1283         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1284
1285     return corrupt_field(data, crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset)
1286
1287 def _corrupt_block_hashes(data):
1288     """ Scramble the file data -- the field containing the block hash tree will have one bit
1289     flipped or else will be changed to a random value.
1290     """
1291     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1292     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1293     if sharevernum == 1:
1294         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1295         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1296     else:
1297         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1298         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1299
1300     return corrupt_field(data, blockhashesoffset, sharehashesoffset-blockhashesoffset)
1301
1302 def _corrupt_share_hashes(data):
1303     """ Scramble the file data -- the field containing the share hash chain will have one
1304     bit flipped or else will be changed to a random value.
1305     """
1306     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1307     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1308     if sharevernum == 1:
1309         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1310         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1311     else:
1312         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1313         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1314
1315     return corrupt_field(data, sharehashesoffset, uriextoffset-sharehashesoffset)
1316
1317 def _corrupt_length_of_uri_extension(data):
1318     """ Scramble the file data -- the field showing the length of the uri extension will
1319     have one bit flipped or else will be changed to a random value. """
1320     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1321     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1322     if sharevernum == 1:
1323         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1324         return corrupt_field(data, uriextoffset, 4)
1325     else:
1326         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1327         return corrupt_field(data, uriextoffset, 8)
1328
1329 def _corrupt_uri_extension(data):
1330     """ Scramble the file data -- the field containing the uri extension will have one bit
1331     flipped or else will be changed to a random value. """
1332     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1333     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1334     if sharevernum == 1:
1335         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1336         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1337     else:
1338         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1339         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1340
1341     return corrupt_field(data, uriextoffset, uriextlen)