src/allmydata/test/common.py

   1 import os, random, struct
   2 from zope.interface import implements
   3 from twisted.internet import defer
   4 from twisted.internet.interfaces import IConsumer
   5 from twisted.python import failure
   6 from twisted.application import service
   7 from twisted.web.error import Error as WebError
   8 from foolscap.api import flushEventualQueue, fireEventually
   9 from allmydata import uri, dirnode, client
  10 from allmydata.introducer.server import IntroducerNode
  11 from allmydata.interfaces import IURI, IMutableFileNode, IFileNode, \
  12      FileTooLargeError, NotEnoughSharesError, ICheckable
  13 from allmydata.check_results import CheckResults, CheckAndRepairResults, \
  14      DeepCheckResults, DeepCheckAndRepairResults
  15 from allmydata.mutable.common import CorruptShareError
  16 from allmydata.mutable.layout import unpack_header
  17 from allmydata.storage.server import storage_index_to_dir
  18 from allmydata.storage.mutable import MutableShareFile
  19 from allmydata.util import hashutil, log, fileutil, pollmixin
  20 from allmydata.util.assertutil import precondition
  21 from allmydata.stats import StatsGathererService
  22 from allmydata.key_generator import KeyGeneratorService
  23 import common_util as testutil
  24 from allmydata import immutable
  25
  26
  27 def flush_but_dont_ignore(res):
  28     d = flushEventualQueue()
  29     def _done(ignored):
  30         return res
  31     d.addCallback(_done)
  32     return d
  33
  34 class FakeCHKFileNode:
  35     """I provide IFileNode, but all of my data is stored in a class-level
  36     dictionary."""
  37     implements(IFileNode)
  38     all_contents = {}
  39     bad_shares = {}
  40
  41     def __init__(self, u, thisclient):
  42         precondition(IURI.providedBy(u), u)
  43         self.client = thisclient
  44         self.my_uri = u
  45         self.storage_index = u.storage_index
  46
  47     def get_uri(self):
  48         return self.my_uri.to_string()
  49     def get_readonly_uri(self):
  50         return self.my_uri.to_string()
  51     def get_verify_cap(self):
  52         return self.my_uri.get_verify_cap()
  53     def get_repair_cap(self):
  54         return self.my_uri.get_verify_cap()
  55     def get_storage_index(self):
  56         return self.storage_index
  57
  58     def check(self, monitor, verify=False, add_lease=False):
  59         r = CheckResults(self.my_uri, self.storage_index)
  60         is_bad = self.bad_shares.get(self.storage_index, None)
  61         data = {}
  62         data["count-shares-needed"] = 3
  63         data["count-shares-expected"] = 10
  64         data["count-good-share-hosts"] = 10
  65         data["count-wrong-shares"] = 0
  66         nodeid = "\x00"*20
  67         data["list-corrupt-shares"] = []
  68         data["sharemap"] = {1: [nodeid]}
  69         data["servers-responding"] = [nodeid]
  70         data["count-recoverable-versions"] = 1
  71         data["count-unrecoverable-versions"] = 0
  72         if is_bad:
  73              r.set_healthy(False)
  74              r.set_recoverable(True)
  75              data["count-shares-good"] = 9
  76              data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
  77              r.problems = failure.Failure(CorruptShareError(is_bad))
  78         else:
  79              r.set_healthy(True)
  80              r.set_recoverable(True)
  81              data["count-shares-good"] = 10
  82              r.problems = []
  83         r.set_data(data)
  84         r.set_needs_rebalancing(False)
  85         return defer.succeed(r)
  86     def check_and_repair(self, monitor, verify=False, add_lease=False):
  87         d = self.check(verify)
  88         def _got(cr):
  89             r = CheckAndRepairResults(self.storage_index)
  90             r.pre_repair_results = r.post_repair_results = cr
  91             return r
  92         d.addCallback(_got)
  93         return d
  94
  95     def is_mutable(self):
  96         return False
  97     def is_readonly(self):
  98         return True
  99
 100     def download(self, target):
 101         if self.my_uri.to_string() not in self.all_contents:
 102             f = failure.Failure(NotEnoughSharesError(None, 0, 3))
 103             target.fail(f)
 104             return defer.fail(f)
 105         data = self.all_contents[self.my_uri.to_string()]
 106         target.open(len(data))
 107         target.write(data)
 108         target.close()
 109         return defer.maybeDeferred(target.finish)
 110     def download_to_data(self):
 111         if self.my_uri.to_string() not in self.all_contents:
 112             return defer.fail(NotEnoughSharesError(None, 0, 3))
 113         data = self.all_contents[self.my_uri.to_string()]
 114         return defer.succeed(data)
 115     def get_size(self):
 116         try:
 117             data = self.all_contents[self.my_uri.to_string()]
 118         except KeyError, le:
 119             raise NotEnoughSharesError(le, 0, 3)
 120         return len(data)
 121     def read(self, consumer, offset=0, size=None):
 122         d = self.download_to_data()
 123         def _got(data):
 124             start = offset
 125             if size is not None:
 126                 end = offset + size
 127             else:
 128                 end = len(data)
 129             consumer.write(data[start:end])
 130             return consumer
 131         d.addCallback(_got)
 132         return d
 133
 134 def make_chk_file_uri(size):
 135     return uri.CHKFileURI(key=os.urandom(16),
 136                           uri_extension_hash=os.urandom(32),
 137                           needed_shares=3,
 138                           total_shares=10,
 139                           size=size)
 140
 141 def create_chk_filenode(thisclient, contents):
 142     u = make_chk_file_uri(len(contents))
 143     n = FakeCHKFileNode(u, thisclient)
 144     FakeCHKFileNode.all_contents[u.to_string()] = contents
 145     return n
 146
 147
 148 class FakeMutableFileNode:
 149     """I provide IMutableFileNode, but all of my data is stored in a
 150     class-level dictionary."""
 151
 152     implements(IMutableFileNode, ICheckable)
 153     MUTABLE_SIZELIMIT = 10000
 154     all_contents = {}
 155     bad_shares = {}
 156
 157     def __init__(self, thisclient):
 158         self.client = thisclient
 159         self.my_uri = make_mutable_file_uri()
 160         self.storage_index = self.my_uri.storage_index
 161     def create(self, initial_contents, key_generator=None, keysize=None):
 162         if len(initial_contents) > self.MUTABLE_SIZELIMIT:
 163             raise FileTooLargeError("SDMF is limited to one segment, and "
 164                                     "%d > %d" % (len(initial_contents),
 165                                                  self.MUTABLE_SIZELIMIT))
 166         self.all_contents[self.storage_index] = initial_contents
 167         return defer.succeed(self)
 168     def init_from_uri(self, myuri):
 169         self.my_uri = IURI(myuri)
 170         self.storage_index = self.my_uri.storage_index
 171         return self
 172     def get_uri(self):
 173         return self.my_uri.to_string()
 174     def get_readonly(self):
 175         return self.my_uri.get_readonly()
 176     def get_readonly_uri(self):
 177         return self.my_uri.get_readonly().to_string()
 178     def get_verify_cap(self):
 179         return self.my_uri.get_verify_cap()
 180     def is_readonly(self):
 181         return self.my_uri.is_readonly()
 182     def is_mutable(self):
 183         return self.my_uri.is_mutable()
 184     def get_writekey(self):
 185         return "\x00"*16
 186     def get_size(self):
 187         return "?" # TODO: see mutable.MutableFileNode.get_size
 188     def get_size_of_best_version(self):
 189         return defer.succeed(len(self.all_contents[self.storage_index]))
 190
 191     def get_storage_index(self):
 192         return self.storage_index
 193
 194     def check(self, monitor, verify=False, add_lease=False):
 195         r = CheckResults(self.my_uri, self.storage_index)
 196         is_bad = self.bad_shares.get(self.storage_index, None)
 197         data = {}
 198         data["count-shares-needed"] = 3
 199         data["count-shares-expected"] = 10
 200         data["count-good-share-hosts"] = 10
 201         data["count-wrong-shares"] = 0
 202         data["list-corrupt-shares"] = []
 203         nodeid = "\x00"*20
 204         data["sharemap"] = {"seq1-abcd-sh0": [nodeid]}
 205         data["servers-responding"] = [nodeid]
 206         data["count-recoverable-versions"] = 1
 207         data["count-unrecoverable-versions"] = 0
 208         if is_bad:
 209              r.set_healthy(False)
 210              r.set_recoverable(True)
 211              data["count-shares-good"] = 9
 212              r.problems = failure.Failure(CorruptShareError("peerid",
 213                                                             0, # shnum
 214                                                             is_bad))
 215         else:
 216              r.set_healthy(True)
 217              r.set_recoverable(True)
 218              data["count-shares-good"] = 10
 219              r.problems = []
 220         r.set_data(data)
 221         r.set_needs_rebalancing(False)
 222         return defer.succeed(r)
 223
 224     def check_and_repair(self, monitor, verify=False, add_lease=False):
 225         d = self.check(verify)
 226         def _got(cr):
 227             r = CheckAndRepairResults(self.storage_index)
 228             r.pre_repair_results = r.post_repair_results = cr
 229             return r
 230         d.addCallback(_got)
 231         return d
 232
 233     def deep_check(self, verify=False, add_lease=False):
 234         d = self.check(verify)
 235         def _done(r):
 236             dr = DeepCheckResults(self.storage_index)
 237             dr.add_check(r, [])
 238             return dr
 239         d.addCallback(_done)
 240         return d
 241
 242     def deep_check_and_repair(self, verify=False, add_lease=False):
 243         d = self.check_and_repair(verify)
 244         def _done(r):
 245             dr = DeepCheckAndRepairResults(self.storage_index)
 246             dr.add_check(r, [])
 247             return dr
 248         d.addCallback(_done)
 249         return d
 250
 251     def download_best_version(self):
 252         return defer.succeed(self.all_contents[self.storage_index])
 253     def overwrite(self, new_contents):
 254         if len(new_contents) > self.MUTABLE_SIZELIMIT:
 255             raise FileTooLargeError("SDMF is limited to one segment, and "
 256                                     "%d > %d" % (len(new_contents),
 257                                                  self.MUTABLE_SIZELIMIT))
 258         assert not self.is_readonly()
 259         self.all_contents[self.storage_index] = new_contents
 260         return defer.succeed(None)
 261     def modify(self, modifier):
 262         # this does not implement FileTooLargeError, but the real one does
 263         return defer.maybeDeferred(self._modify, modifier)
 264     def _modify(self, modifier):
 265         assert not self.is_readonly()
 266         old_contents = self.all_contents[self.storage_index]
 267         self.all_contents[self.storage_index] = modifier(old_contents, None, True)
 268         return None
 269
 270     def download(self, target):
 271         if self.storage_index not in self.all_contents:
 272             f = failure.Failure(NotEnoughSharesError(None, 0, 3))
 273             target.fail(f)
 274             return defer.fail(f)
 275         data = self.all_contents[self.storage_index]
 276         target.open(len(data))
 277         target.write(data)
 278         target.close()
 279         return defer.maybeDeferred(target.finish)
 280     def download_to_data(self):
 281         if self.storage_index not in self.all_contents:
 282             return defer.fail(NotEnoughSharesError(None, 0, 3))
 283         data = self.all_contents[self.storage_index]
 284         return defer.succeed(data)
 285
 286 def make_mutable_file_uri():
 287     return uri.WriteableSSKFileURI(writekey=os.urandom(16),
 288                                    fingerprint=os.urandom(32))
 289 def make_verifier_uri():
 290     return uri.SSKVerifierURI(storage_index=os.urandom(16),
 291                               fingerprint=os.urandom(32))
 292
 293 class FakeDirectoryNode(dirnode.DirectoryNode):
 294     """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
 295     backing store, so it doesn't go to the grid. The child data is still
 296     encrypted and serialized, so this isn't useful for tests that want to
 297     look inside the dirnodes and check their contents.
 298     """
 299     filenode_class = FakeMutableFileNode
 300
 301 class LoggingServiceParent(service.MultiService):
 302     def log(self, *args, **kwargs):
 303         return log.msg(*args, **kwargs)
 304
 305
 306 class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 307
 308     # SystemTestMixin tests tend to be a lot of work, and we have a few
 309     # buildslaves that are pretty slow, and every once in a while these tests
 310     # run up against the default 120 second timeout. So increase the default
 311     # timeout. Individual test cases can override this, of course.
 312     timeout = 300
 313
 314     def setUp(self):
 315         self.sparent = service.MultiService()
 316         self.sparent.startService()
 317
 318         self.stats_gatherer = None
 319         self.stats_gatherer_furl = None
 320         self.key_generator_svc = None
 321         self.key_generator_furl = None
 322
 323     def tearDown(self):
 324         log.msg("shutting down SystemTest services")
 325         d = self.sparent.stopService()
 326         d.addBoth(flush_but_dont_ignore)
 327         return d
 328
 329     def getdir(self, subdir):
 330         return os.path.join(self.basedir, subdir)
 331
 332     def add_service(self, s):
 333         s.setServiceParent(self.sparent)
 334         return s
 335
 336     def set_up_nodes(self, NUMCLIENTS=5,
 337                      use_stats_gatherer=False, use_key_generator=False):
 338         self.numclients = NUMCLIENTS
 339         iv_dir = self.getdir("introducer")
 340         if not os.path.isdir(iv_dir):
 341             fileutil.make_dirs(iv_dir)
 342             f = open(os.path.join(iv_dir, "webport"), "w")
 343             f.write("tcp:0:interface=127.0.0.1\n")
 344             f.close()
 345             if SYSTEM_TEST_CERTS:
 346                 os.mkdir(os.path.join(iv_dir, "private"))
 347                 f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
 348                 f.write(SYSTEM_TEST_CERTS[0])
 349                 f.close()
 350         iv = IntroducerNode(basedir=iv_dir)
 351         self.introducer = self.add_service(iv)
 352         d = self.introducer.when_tub_ready()
 353         d.addCallback(self._get_introducer_web)
 354         if use_stats_gatherer:
 355             d.addCallback(self._set_up_stats_gatherer)
 356         if use_key_generator:
 357             d.addCallback(self._set_up_key_generator)
 358         d.addCallback(self._set_up_nodes_2)
 359         if use_stats_gatherer:
 360             d.addCallback(self._grab_stats)
 361         return d
 362
 363     def _get_introducer_web(self, res):
 364         f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
 365         self.introweb_url = f.read().strip()
 366         f.close()
 367
 368     def _set_up_stats_gatherer(self, res):
 369         statsdir = self.getdir("stats_gatherer")
 370         fileutil.make_dirs(statsdir)
 371         self.stats_gatherer_svc = StatsGathererService(statsdir)
 372         self.stats_gatherer = self.stats_gatherer_svc.stats_gatherer
 373         self.add_service(self.stats_gatherer_svc)
 374
 375         d = fireEventually()
 376         sgf = os.path.join(statsdir, 'stats_gatherer.furl')
 377         def check_for_furl():
 378             return os.path.exists(sgf)
 379         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 380         def get_furl(junk):
 381             self.stats_gatherer_furl = file(sgf, 'rb').read().strip()
 382         d.addCallback(get_furl)
 383         return d
 384
 385     def _set_up_key_generator(self, res):
 386         kgsdir = self.getdir("key_generator")
 387         fileutil.make_dirs(kgsdir)
 388
 389         self.key_generator_svc = KeyGeneratorService(kgsdir,
 390                                                      display_furl=False,
 391                                                      default_key_size=522)
 392         self.key_generator_svc.key_generator.pool_size = 4
 393         self.key_generator_svc.key_generator.pool_refresh_delay = 60
 394         self.add_service(self.key_generator_svc)
 395
 396         d = fireEventually()
 397         def check_for_furl():
 398             return os.path.exists(os.path.join(kgsdir, 'key_generator.furl'))
 399         d.addCallback(lambda junk: self.poll(check_for_furl, timeout=30))
 400         def get_furl(junk):
 401             kgf = os.path.join(kgsdir, 'key_generator.furl')
 402             self.key_generator_furl = file(kgf, 'rb').read().strip()
 403         d.addCallback(get_furl)
 404         return d
 405
 406     def _set_up_nodes_2(self, res):
 407         q = self.introducer
 408         self.introducer_furl = q.introducer_url
 409         self.clients = []
 410         basedirs = []
 411         for i in range(self.numclients):
 412             basedir = self.getdir("client%d" % i)
 413             basedirs.append(basedir)
 414             fileutil.make_dirs(os.path.join(basedir, "private"))
 415             if len(SYSTEM_TEST_CERTS) > (i+1):
 416                 f = open(os.path.join(basedir, "private", "node.pem"), "w")
 417                 f.write(SYSTEM_TEST_CERTS[i+1])
 418                 f.close()
 419
 420             def write(name, value):
 421                 open(os.path.join(basedir, name), "w").write(value+"\n")
 422             if i == 0:
 423                 # clients[0] runs a webserver and a helper, no key_generator
 424                 write("webport", "tcp:0:interface=127.0.0.1")
 425                 write("run_helper", "yes")
 426                 write("keepalive_timeout", "600")
 427             if i == 3:
 428                 # clients[3] runs a webserver and uses a helper, uses
 429                 # key_generator
 430                 write("webport", "tcp:0:interface=127.0.0.1")
 431                 write("disconnect_timeout", "1800")
 432                 if self.key_generator_furl:
 433                     kgf = "%s\n" % (self.key_generator_furl,)
 434                     write("key_generator.furl", kgf)
 435             write("introducer.furl", self.introducer_furl)
 436             if self.stats_gatherer_furl:
 437                 write("stats_gatherer.furl", self.stats_gatherer_furl)
 438
 439         # give subclasses a chance to append liens to the node's tahoe.cfg
 440         # files before they are launched.
 441         self._set_up_nodes_extra_config()
 442
 443         # start clients[0], wait for it's tub to be ready (at which point it
 444         # will have registered the helper furl).
 445         c = self.add_service(client.Client(basedir=basedirs[0]))
 446         self.clients.append(c)
 447         c.DEFAULT_MUTABLE_KEYSIZE = 522
 448         d = c.when_tub_ready()
 449         def _ready(res):
 450             f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
 451             helper_furl = f.read()
 452             f.close()
 453             self.helper_furl = helper_furl
 454             if self.numclients >= 4:
 455                 f = open(os.path.join(basedirs[3],"helper.furl"), "w")
 456                 f.write(helper_furl)
 457                 f.close()
 458
 459             # this starts the rest of the clients
 460             for i in range(1, self.numclients):
 461                 c = self.add_service(client.Client(basedir=basedirs[i]))
 462                 self.clients.append(c)
 463                 c.DEFAULT_MUTABLE_KEYSIZE = 522
 464             log.msg("STARTING")
 465             return self.wait_for_connections()
 466         d.addCallback(_ready)
 467         def _connected(res):
 468             log.msg("CONNECTED")
 469             # now find out where the web port was
 470             l = self.clients[0].getServiceNamed("webish").listener
 471             port = l._port.getHost().port
 472             self.webish_url = "http://localhost:%d/" % port
 473             if self.numclients >=4:
 474                 # and the helper-using webport
 475                 l = self.clients[3].getServiceNamed("webish").listener
 476                 port = l._port.getHost().port
 477                 self.helper_webish_url = "http://localhost:%d/" % port
 478         d.addCallback(_connected)
 479         return d
 480
 481     def _set_up_nodes_extra_config(self):
 482         # for overriding by subclasses
 483         pass
 484
 485     def _grab_stats(self, res):
 486         d = self.stats_gatherer.poll()
 487         return d
 488
 489     def bounce_client(self, num):
 490         c = self.clients[num]
 491         d = c.disownServiceParent()
 492         # I think windows requires a moment to let the connection really stop
 493         # and the port number made available for re-use. TODO: examine the
 494         # behavior, see if this is really the problem, see if we can do
 495         # better than blindly waiting for a second.
 496         d.addCallback(self.stall, 1.0)
 497         def _stopped(res):
 498             new_c = client.Client(basedir=self.getdir("client%d" % num))
 499             self.clients[num] = new_c
 500             new_c.DEFAULT_MUTABLE_KEYSIZE = 522
 501             self.add_service(new_c)
 502             return new_c.when_tub_ready()
 503         d.addCallback(_stopped)
 504         d.addCallback(lambda res: self.wait_for_connections())
 505         def _maybe_get_webport(res):
 506             if num == 0:
 507                 # now find out where the web port was
 508                 l = self.clients[0].getServiceNamed("webish").listener
 509                 port = l._port.getHost().port
 510                 self.webish_url = "http://localhost:%d/" % port
 511         d.addCallback(_maybe_get_webport)
 512         return d
 513
 514     def add_extra_node(self, client_num, helper_furl=None,
 515                        add_to_sparent=False):
 516         # usually this node is *not* parented to our self.sparent, so we can
 517         # shut it down separately from the rest, to exercise the
 518         # connection-lost code
 519         basedir = self.getdir("client%d" % client_num)
 520         if not os.path.isdir(basedir):
 521             fileutil.make_dirs(basedir)
 522         open(os.path.join(basedir, "introducer.furl"), "w").write(self.introducer_furl)
 523         if helper_furl:
 524             f = open(os.path.join(basedir, "helper.furl") ,"w")
 525             f.write(helper_furl+"\n")
 526             f.close()
 527
 528         c = client.Client(basedir=basedir)
 529         self.clients.append(c)
 530         c.DEFAULT_MUTABLE_KEYSIZE = 522
 531         self.numclients += 1
 532         if add_to_sparent:
 533             c.setServiceParent(self.sparent)
 534         else:
 535             c.startService()
 536         d = self.wait_for_connections()
 537         d.addCallback(lambda res: c)
 538         return d
 539
 540     def _check_connections(self):
 541         for c in self.clients:
 542             if not c.connected_to_introducer():
 543                 return False
 544             sb = c.get_storage_broker()
 545             if len(sb.get_all_servers()) != self.numclients:
 546                 return False
 547         return True
 548
 549     def wait_for_connections(self, ignored=None):
 550         # TODO: replace this with something that takes a list of peerids and
 551         # fires when they've all been heard from, instead of using a count
 552         # and a threshold
 553         return self.poll(self._check_connections, timeout=200)
 554
 555
 556 # our system test uses the same Tub certificates each time, to avoid the
 557 # overhead of key generation
 558 SYSTEM_TEST_CERTS = [
 559 """-----BEGIN CERTIFICATE-----
 560 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 561 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 562 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR
 563 2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu
 564 Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg
 565 DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB
 566 gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds
 567 Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS
 568 2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ==
 569 -----END CERTIFICATE-----
 570 -----BEGIN RSA PRIVATE KEY-----
 571 MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R
 572 x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8
 573 /bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB
 574 AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V
 575 QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ
 576 3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ
 577 IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6
 578 bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw
 579 iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy
 580 bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S
 581 cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ
 582 WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP
 583 fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk
 584 -----END RSA PRIVATE KEY-----
 585 """, # 0
 586 """-----BEGIN CERTIFICATE-----
 587 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 588 aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU
 589 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW
 590 kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8
 591 n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO
 592 iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 593 gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq
 594 dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh
 595 OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ==
 596 -----END CERTIFICATE-----
 597 -----BEGIN RSA PRIVATE KEY-----
 598 MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3
 599 p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ
 600 sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB
 601 AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2
 602 qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9
 603 h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z
 604 UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l
 605 8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7
 606 2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj
 607 uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+
 608 hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek
 609 4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ
 610 JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d
 611 -----END RSA PRIVATE KEY-----
 612 """, # 1
 613 """-----BEGIN CERTIFICATE-----
 614 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 615 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 616 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz
 617 DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo
 618 q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv
 619 dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB
 620 gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw
 621 AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD
 622 b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ==
 623 -----END CERTIFICATE-----
 624 -----BEGIN RSA PRIVATE KEY-----
 625 MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/
 626 jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa
 627 hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB
 628 AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65
 629 CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt
 630 1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb
 631 wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf
 632 TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN
 633 aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz
 634 g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb
 635 nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa
 636 P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl
 637 Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1
 638 -----END RSA PRIVATE KEY-----
 639 """, # 2
 640 """-----BEGIN CERTIFICATE-----
 641 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 642 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 643 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS
 644 qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi
 645 CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm
 646 E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 647 gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50
 648 xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH//
 649 xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A==
 650 -----END CERTIFICATE-----
 651 -----BEGIN RSA PRIVATE KEY-----
 652 MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO
 653 nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh
 654 gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB
 655 AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX
 656 8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U
 657 1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/
 658 ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR
 659 2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5
 660 XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum
 661 n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1
 662 XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk
 663 mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY
 664 vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg=
 665 -----END RSA PRIVATE KEY-----
 666 """, # 3
 667 """-----BEGIN CERTIFICATE-----
 668 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 669 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 670 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn
 671 jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP
 672 62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI
 673 sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB
 674 gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz
 675 w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59
 676 0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA==
 677 -----END CERTIFICATE-----
 678 -----BEGIN RSA PRIVATE KEY-----
 679 MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek
 680 2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4
 681 sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB
 682 AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79
 683 TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n
 684 7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU
 685 7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL
 686 kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q
 687 oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I
 688 VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D
 689 KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I
 690 eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37
 691 fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs=
 692 -----END RSA PRIVATE KEY-----
 693 """, # 4
 694 """-----BEGIN CERTIFICATE-----
 695 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 696 aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU
 697 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF
 698 zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe
 699 4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y
 700 ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB
 701 gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE
 702 Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS
 703 CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A==
 704 -----END CERTIFICATE-----
 705 -----BEGIN RSA PRIVATE KEY-----
 706 MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd
 707 aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE
 708 STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB
 709 AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h
 710 36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg
 711 UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC
 712 Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK
 713 rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2
 714 xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw
 715 /tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H
 716 B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0
 717 Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7
 718 w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg==
 719 -----END RSA PRIVATE KEY-----
 720 """, # 5
 721 """-----BEGIN CERTIFICATE-----
 722 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 723 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 724 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA
 725 NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF
 726 3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV
 727 VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB
 728 gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT
 729 BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H
 730 j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA==
 731 -----END CERTIFICATE-----
 732 -----BEGIN RSA PRIVATE KEY-----
 733 MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI
 734 yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI
 735 E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB
 736 AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx
 737 f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp
 738 0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw
 739 hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK
 740 zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw
 741 IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn
 742 m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo
 743 xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX
 744 5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp
 745 Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl
 746 -----END RSA PRIVATE KEY-----
 747 """, #6
 748 """-----BEGIN CERTIFICATE-----
 749 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 750 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 751 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA
 752 jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je
 753 qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff
 754 J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB
 755 gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV
 756 WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az
 757 ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA==
 758 -----END CERTIFICATE-----
 759 -----BEGIN RSA PRIVATE KEY-----
 760 MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt
 761 1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2
 762 O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB
 763 AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz
 764 oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf
 765 TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH
 766 V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr
 767 VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb
 768 BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3
 769 3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE
 770 BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH
 771 LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/
 772 8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs=
 773 -----END RSA PRIVATE KEY-----
 774 """, #7
 775 """-----BEGIN CERTIFICATE-----
 776 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 777 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 778 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn
 779 5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC
 780 EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1
 781 I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB
 782 gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT
 783 3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu
 784 Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g==
 785 -----END CERTIFICATE-----
 786 -----BEGIN RSA PRIVATE KEY-----
 787 MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn
 788 zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT
 789 HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB
 790 AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t
 791 qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2
 792 dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW
 793 6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R
 794 0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld
 795 YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y
 796 biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY
 797 5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0
 798 7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE
 799 qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE
 800 -----END RSA PRIVATE KEY-----
 801 """, #8
 802 """-----BEGIN CERTIFICATE-----
 803 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 804 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 805 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0
 806 CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV
 807 kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3
 808 pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 809 gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8
 810 A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ
 811 ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw==
 812 -----END CERTIFICATE-----
 813 -----BEGIN RSA PRIVATE KEY-----
 814 MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM
 815 LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W
 816 bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB
 817 AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u
 818 VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB
 819 dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk
 820 lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg
 821 Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF
 822 OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso
 823 5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS
 824 sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb
 825 Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk
 826 dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp
 827 -----END RSA PRIVATE KEY-----
 828 """, #9
 829 """-----BEGIN CERTIFICATE-----
 830 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 831 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 832 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT
 833 kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP
 834 wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR
 835 BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 836 gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4
 837 pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l
 838 OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ==
 839 -----END CERTIFICATE-----
 840 -----BEGIN RSA PRIVATE KEY-----
 841 MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF
 842 pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+
 843 Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB
 844 AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK
 845 A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv
 846 tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz
 847 BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg
 848 GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT
 849 tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F
 850 xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1
 851 lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs
 852 MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe
 853 JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g==
 854 -----END RSA PRIVATE KEY-----
 855 """, #10
 856 """-----BEGIN CERTIFICATE-----
 857 MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro
 858 aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU
 859 DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J
 860 XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY
 861 NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt
 862 o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB
 863 gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK
 864 bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls
 865 q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA==
 866 -----END CERTIFICATE-----
 867 -----BEGIN RSA PRIVATE KEY-----
 868 MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU
 869 zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG
 870 DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB
 871 AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer
 872 MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c
 873 UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT
 874 bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj
 875 hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+
 876 1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb
 877 QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW
 878 MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08
 879 2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY
 880 N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd
 881 -----END RSA PRIVATE KEY-----
 882 """, #11
 883 ]
 884
 885 # To disable the pre-computed tub certs, uncomment this line.
 886 #SYSTEM_TEST_CERTS = []
 887
 888 TEST_DATA="\x02"*(immutable.upload.Uploader.URI_LIT_SIZE_THRESHOLD+1)
 889
 890 class ShareManglingMixin(SystemTestMixin):
 891
 892     def setUp(self):
 893         # Set self.basedir to a temp dir which has the name of the current
 894         # test method in its name.
 895         self.basedir = self.mktemp()
 896
 897         d = defer.maybeDeferred(SystemTestMixin.setUp, self)
 898         d.addCallback(lambda x: self.set_up_nodes())
 899
 900         def _upload_a_file(ignored):
 901             cl0 = self.clients[0]
 902             # We need multiple segments to test crypttext hash trees that are
 903             # non-trivial (i.e. they have more than just one hash in them).
 904             cl0.DEFAULT_ENCODING_PARAMETERS['max_segment_size'] = 12
 905             d2 = cl0.upload(immutable.upload.Data(TEST_DATA, convergence=""))
 906             def _after_upload(u):
 907                 self.uri = IURI(u.uri)
 908                 return cl0.create_node_from_uri(self.uri)
 909             d2.addCallback(_after_upload)
 910             return d2
 911         d.addCallback(_upload_a_file)
 912
 913         def _stash_it(filenode):
 914             self.filenode = filenode
 915         d.addCallback(_stash_it)
 916         return d
 917
 918     def find_shares(self, unused=None):
 919         """Locate shares on disk. Returns a dict that maps
 920         (clientnum,sharenum) to a string that contains the share container
 921         (copied directly from the disk, containing leases etc). You can
 922         modify this dict and then call replace_shares() to modify the shares.
 923         """
 924         shares = {} # k: (i, sharenum), v: data
 925
 926         for i, c in enumerate(self.clients):
 927             sharedir = c.getServiceNamed("storage").sharedir
 928             for (dirp, dirns, fns) in os.walk(sharedir):
 929                 for fn in fns:
 930                     try:
 931                         sharenum = int(fn)
 932                     except TypeError:
 933                         # Whoops, I guess that's not a share file then.
 934                         pass
 935                     else:
 936                         data = open(os.path.join(sharedir, dirp, fn), "rb").read()
 937                         shares[(i, sharenum)] = data
 938
 939         return shares
 940
 941     def replace_shares(self, newshares, storage_index):
 942         """Replace shares on disk. Takes a dictionary in the same form
 943         as find_shares() returns."""
 944
 945         for i, c in enumerate(self.clients):
 946             sharedir = c.getServiceNamed("storage").sharedir
 947             for (dirp, dirns, fns) in os.walk(sharedir):
 948                 for fn in fns:
 949                     try:
 950                         sharenum = int(fn)
 951                     except TypeError:
 952                         # Whoops, I guess that's not a share file then.
 953                         pass
 954                     else:
 955                         pathtosharefile = os.path.join(sharedir, dirp, fn)
 956                         os.unlink(pathtosharefile)
 957             for ((clientnum, sharenum), newdata) in newshares.iteritems():
 958                 if clientnum == i:
 959                     fullsharedirp=os.path.join(sharedir, storage_index_to_dir(storage_index))
 960                     fileutil.make_dirs(fullsharedirp)
 961                     wf = open(os.path.join(fullsharedirp, str(sharenum)), "wb")
 962                     wf.write(newdata)
 963                     wf.close()
 964
 965     def _delete_a_share(self, unused=None, sharenum=None):
 966         """ Delete one share. """
 967
 968         shares = self.find_shares()
 969         ks = shares.keys()
 970         if sharenum is not None:
 971             k = [ key for key in shares.keys() if key[1] == sharenum ][0]
 972         else:
 973             k = random.choice(ks)
 974         del shares[k]
 975         self.replace_shares(shares, storage_index=self.uri.storage_index)
 976
 977         return unused
 978
 979     def _corrupt_a_share(self, unused, corruptor_func, sharenum):
 980         shares = self.find_shares()
 981         ks = [ key for key in shares.keys() if key[1] == sharenum ]
 982         assert ks, (shares.keys(), sharenum)
 983         k = ks[0]
 984         shares[k] = corruptor_func(shares[k])
 985         self.replace_shares(shares, storage_index=self.uri.storage_index)
 986         return corruptor_func
 987
 988     def _corrupt_all_shares(self, unused, corruptor_func):
 989         """ All shares on disk will be corrupted by corruptor_func. """
 990         shares = self.find_shares()
 991         for k in shares.keys():
 992             self._corrupt_a_share(unused, corruptor_func, k[1])
 993         return corruptor_func
 994
 995     def _corrupt_a_random_share(self, unused, corruptor_func):
 996         """ Exactly one share on disk will be corrupted by corruptor_func. """
 997         shares = self.find_shares()
 998         ks = shares.keys()
 999         k = random.choice(ks)
1000         self._corrupt_a_share(unused, corruptor_func, k[1])
1001         return k[1]
1002
1003     def _count_reads(self):
1004         sum_of_read_counts = 0
1005         for thisclient in self.clients:
1006             counters = thisclient.stats_provider.get_stats()['counters']
1007             sum_of_read_counts += counters.get('storage_server.read', 0)
1008         return sum_of_read_counts
1009
1010     def _count_allocates(self):
1011         sum_of_allocate_counts = 0
1012         for thisclient in self.clients:
1013             counters = thisclient.stats_provider.get_stats()['counters']
1014             sum_of_allocate_counts += counters.get('storage_server.allocate', 0)
1015         return sum_of_allocate_counts
1016
1017     def _count_writes(self):
1018         sum_of_write_counts = 0
1019         for thisclient in self.clients:
1020             counters = thisclient.stats_provider.get_stats()['counters']
1021             sum_of_write_counts += counters.get('storage_server.write', 0)
1022         return sum_of_write_counts
1023
1024     def _download_and_check_plaintext(self, unused=None):
1025         self.downloader = self.clients[1].getServiceNamed("downloader")
1026         d = self.downloader.download_to_data(self.uri)
1027
1028         def _after_download(result):
1029             self.failUnlessEqual(result, TEST_DATA)
1030         d.addCallback(_after_download)
1031         return d
1032
1033 class ShouldFailMixin:
1034     def shouldFail(self, expected_failure, which, substring,
1035                    callable, *args, **kwargs):
1036         """Assert that a function call raises some exception. This is a
1037         Deferred-friendly version of TestCase.assertRaises() .
1038
1039         Suppose you want to verify the following function:
1040
1041          def broken(a, b, c):
1042              if a < 0:
1043                  raise TypeError('a must not be negative')
1044              return defer.succeed(b+c)
1045
1046         You can use:
1047             d = self.shouldFail(TypeError, 'test name',
1048                                 'a must not be negative',
1049                                 broken, -4, 5, c=12)
1050         in your test method. The 'test name' string will be included in the
1051         error message, if any, because Deferred chains frequently make it
1052         difficult to tell which assertion was tripped.
1053
1054         The substring= argument, if not None, must appear inside the
1055         stringified Failure, or the test will fail.
1056         """
1057
1058         assert substring is None or isinstance(substring, str)
1059         d = defer.maybeDeferred(callable, *args, **kwargs)
1060         def done(res):
1061             if isinstance(res, failure.Failure):
1062                 res.trap(expected_failure)
1063                 if substring:
1064                     self.failUnless(substring in str(res),
1065                                     "substring '%s' not in '%s'"
1066                                     % (substring, str(res)))
1067             else:
1068                 self.fail("%s was supposed to raise %s, not get '%s'" %
1069                           (which, expected_failure, res))
1070         d.addBoth(done)
1071         return d
1072
1073 class WebErrorMixin:
1074     def explain_web_error(self, f):
1075         # an error on the server side causes the client-side getPage() to
1076         # return a failure(t.web.error.Error), and its str() doesn't show the
1077         # response body, which is where the useful information lives. Attach
1078         # this method as an errback handler, and it will reveal the hidden
1079         # message.
1080         f.trap(WebError)
1081         print "Web Error:", f.value, ":", f.value.response
1082         return f
1083
1084     def _shouldHTTPError(self, res, which, validator):
1085         if isinstance(res, failure.Failure):
1086             res.trap(WebError)
1087             return validator(res)
1088         else:
1089             self.fail("%s was supposed to Error, not get '%s'" % (which, res))
1090
1091     def shouldHTTPError(self, which,
1092                         code=None, substring=None, response_substring=None,
1093                         callable=None, *args, **kwargs):
1094         # returns a Deferred with the response body
1095         assert substring is None or isinstance(substring, str)
1096         assert callable
1097         def _validate(f):
1098             if code is not None:
1099                 self.failUnlessEqual(f.value.status, str(code))
1100             if substring:
1101                 code_string = str(f)
1102                 self.failUnless(substring in code_string,
1103                                 "substring '%s' not in '%s'"
1104                                 % (substring, code_string))
1105             response_body = f.value.response
1106             if response_substring:
1107                 self.failUnless(response_substring in response_body,
1108                                 "response substring '%s' not in '%s'"
1109                                 % (response_substring, response_body))
1110             return response_body
1111         d = defer.maybeDeferred(callable, *args, **kwargs)
1112         d.addBoth(self._shouldHTTPError, which, _validate)
1113         return d
1114
1115 class ErrorMixin(WebErrorMixin):
1116     def explain_error(self, f):
1117         if f.check(defer.FirstError):
1118             print "First Error:", f.value.subFailure
1119         return f
1120
1121 class MemoryConsumer:
1122     implements(IConsumer)
1123     def __init__(self):
1124         self.chunks = []
1125         self.done = False
1126     def registerProducer(self, p, streaming):
1127         if streaming:
1128             # call resumeProducing once to start things off
1129             p.resumeProducing()
1130         else:
1131             while not self.done:
1132                 p.resumeProducing()
1133     def write(self, data):
1134         self.chunks.append(data)
1135     def unregisterProducer(self):
1136         self.done = True
1137
1138 def download_to_data(n, offset=0, size=None):
1139     d = n.read(MemoryConsumer(), offset, size)
1140     d.addCallback(lambda mc: "".join(mc.chunks))
1141     return d
1142
1143 def corrupt_field(data, offset, size, debug=False):
1144     if random.random() < 0.5:
1145         newdata = testutil.flip_one_bit(data, offset, size)
1146         if debug:
1147             log.msg("testing: corrupting offset %d, size %d flipping one bit orig: %r, newdata: %r" % (offset, size, data[offset:offset+size], newdata[offset:offset+size]))
1148         return newdata
1149     else:
1150         newval = testutil.insecurerandstr(size)
1151         if debug:
1152             log.msg("testing: corrupting offset %d, size %d randomizing field, orig: %r, newval: %r" % (offset, size, data[offset:offset+size], newval))
1153         return data[:offset]+newval+data[offset+size:]
1154
1155 def _corrupt_nothing(data):
1156     """ Leave the data pristine. """
1157     return data
1158
1159 def _corrupt_file_version_number(data):
1160     """ Scramble the file data -- the share file version number have one bit flipped or else
1161     will be changed to a random value."""
1162     return corrupt_field(data, 0x00, 4)
1163
1164 def _corrupt_size_of_file_data(data):
1165     """ Scramble the file data -- the field showing the size of the share data within the file
1166     will be set to one smaller. """
1167     return corrupt_field(data, 0x04, 4)
1168
1169 def _corrupt_sharedata_version_number(data):
1170     """ Scramble the file data -- the share data version number will have one bit flipped or
1171     else will be changed to a random value, but not 1 or 2."""
1172     return corrupt_field(data, 0x0c, 4)
1173     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1174     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1175     newsharevernum = sharevernum
1176     while newsharevernum in (1, 2):
1177         newsharevernum = random.randrange(0, 2**32)
1178     newsharevernumbytes = struct.pack(">L", newsharevernum)
1179     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1180
1181 def _corrupt_sharedata_version_number_to_plausible_version(data):
1182     """ Scramble the file data -- the share data version number will
1183     be changed to 2 if it is 1 or else to 1 if it is 2."""
1184     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1185     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1186     if sharevernum == 1:
1187         newsharevernum = 2
1188     else:
1189         newsharevernum = 1
1190     newsharevernumbytes = struct.pack(">L", newsharevernum)
1191     return data[:0x0c] + newsharevernumbytes + data[0x0c+4:]
1192
1193 def _corrupt_segment_size(data):
1194     """ Scramble the file data -- the field showing the size of the segment will have one
1195     bit flipped or else be changed to a random value. """
1196     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1197     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1198     if sharevernum == 1:
1199         return corrupt_field(data, 0x0c+0x04, 4, debug=False)
1200     else:
1201         return corrupt_field(data, 0x0c+0x04, 8, debug=False)
1202
1203 def _corrupt_size_of_sharedata(data):
1204     """ Scramble the file data -- the field showing the size of the data within the share
1205     data will have one bit flipped or else will be changed to a random value. """
1206     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1207     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1208     if sharevernum == 1:
1209         return corrupt_field(data, 0x0c+0x08, 4)
1210     else:
1211         return corrupt_field(data, 0x0c+0x0c, 8)
1212
1213 def _corrupt_offset_of_sharedata(data):
1214     """ Scramble the file data -- the field showing the offset of the data within the share
1215     data will have one bit flipped or else be changed to a random value. """
1216     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1217     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1218     if sharevernum == 1:
1219         return corrupt_field(data, 0x0c+0x0c, 4)
1220     else:
1221         return corrupt_field(data, 0x0c+0x14, 8)
1222
1223 def _corrupt_offset_of_ciphertext_hash_tree(data):
1224     """ Scramble the file data -- the field showing the offset of the ciphertext hash tree
1225     within the share data will have one bit flipped or else be changed to a random value.
1226     """
1227     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1228     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1229     if sharevernum == 1:
1230         return corrupt_field(data, 0x0c+0x14, 4, debug=False)
1231     else:
1232         return corrupt_field(data, 0x0c+0x24, 8, debug=False)
1233
1234 def _corrupt_offset_of_block_hashes(data):
1235     """ Scramble the file data -- the field showing the offset of the block hash tree within
1236     the share data will have one bit flipped or else will be changed to a random value. """
1237     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1238     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1239     if sharevernum == 1:
1240         return corrupt_field(data, 0x0c+0x18, 4)
1241     else:
1242         return corrupt_field(data, 0x0c+0x2c, 8)
1243
1244 def _corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes(data):
1245     """ Scramble the file data -- the field showing the offset of the block hash tree within the
1246     share data will have a multiple of hash size subtracted from it, thus causing the downloader
1247     to download an incomplete crypttext hash tree."""
1248     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1249     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1250     if sharevernum == 1:
1251         curval = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1252         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1253         newvalstr = struct.pack(">L", newval)
1254         return data[:0x0c+0x18]+newvalstr+data[0x0c+0x18+4:]
1255     else:
1256         curval = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1257         newval = random.randrange(0, max(1, (curval/hashutil.CRYPTO_VAL_SIZE)/2))*hashutil.CRYPTO_VAL_SIZE
1258         newvalstr = struct.pack(">Q", newval)
1259         return data[:0x0c+0x2c]+newvalstr+data[0x0c+0x2c+8:]
1260
1261 def _corrupt_offset_of_share_hashes(data):
1262     """ Scramble the file data -- the field showing the offset of the share hash tree within
1263     the share data will have one bit flipped or else will be changed to a random value. """
1264     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1265     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1266     if sharevernum == 1:
1267         return corrupt_field(data, 0x0c+0x1c, 4)
1268     else:
1269         return corrupt_field(data, 0x0c+0x34, 8)
1270
1271 def _corrupt_offset_of_uri_extension(data):
1272     """ Scramble the file data -- the field showing the offset of the uri extension will
1273     have one bit flipped or else will be changed to a random value. """
1274     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1275     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1276     if sharevernum == 1:
1277         return corrupt_field(data, 0x0c+0x20, 4)
1278     else:
1279         return corrupt_field(data, 0x0c+0x3c, 8)
1280
1281 def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False):
1282     """ Scramble the file data -- the field showing the offset of the uri extension will be set
1283     to the size of the file minus 3.  This means when the client tries to read the length field
1284     from that location it will get a short read -- the result string will be only 3 bytes long,
1285     not the 4 or 8 bytes necessary to do a successful struct.unpack."""
1286     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1287     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1288     # The "-0x0c" in here is to skip the server-side header in the share file, which the client doesn't see when seeking and reading.
1289     if sharevernum == 1:
1290         if debug:
1291             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x2c, 4, struct.unpack(">L", data[0x2c:0x2c+4])[0], len(data)-0x0c-3, len(data)))
1292         return data[:0x2c] + struct.pack(">L", len(data)-0x0c-3) + data[0x2c+4:]
1293     else:
1294         if debug:
1295             log.msg("testing: corrupting offset %d, size %d, changing %d to %d (len(data) == %d)" % (0x48, 8, struct.unpack(">Q", data[0x48:0x48+8])[0], len(data)-0x0c-3, len(data)))
1296         return data[:0x48] + struct.pack(">Q", len(data)-0x0c-3) + data[0x48+8:]
1297
1298 def _corrupt_mutable_share_data(data):
1299     prefix = data[:32]
1300     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
1301     data_offset = MutableShareFile.DATA_OFFSET
1302     sharetype = data[data_offset:data_offset+1]
1303     assert sharetype == "\x00", "non-SDMF mutable shares not supported"
1304     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
1305      ig_datalen, offsets) = unpack_header(data[data_offset:])
1306     assert version == 0, "this function only handles v0 SDMF files"
1307     start = data_offset + offsets["share_data"]
1308     length = data_offset + offsets["enc_privkey"] - start
1309     return corrupt_field(data, start, length)
1310
1311 def _corrupt_share_data(data):
1312     """ Scramble the file data -- the field containing the share data itself will have one
1313     bit flipped or else will be changed to a random value. """
1314     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1315     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways, not v%d." % sharevernum
1316     if sharevernum == 1:
1317         sharedatasize = struct.unpack(">L", data[0x0c+0x08:0x0c+0x08+4])[0]
1318
1319         return corrupt_field(data, 0x0c+0x24, sharedatasize)
1320     else:
1321         sharedatasize = struct.unpack(">Q", data[0x0c+0x08:0x0c+0x0c+8])[0]
1322
1323         return corrupt_field(data, 0x0c+0x44, sharedatasize)
1324
1325 def _corrupt_crypttext_hash_tree(data):
1326     """ Scramble the file data -- the field containing the crypttext hash tree will have one
1327     bit flipped or else will be changed to a random value.
1328     """
1329     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1330     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1331     if sharevernum == 1:
1332         crypttexthashtreeoffset = struct.unpack(">L", data[0x0c+0x14:0x0c+0x14+4])[0]
1333         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1334     else:
1335         crypttexthashtreeoffset = struct.unpack(">Q", data[0x0c+0x24:0x0c+0x24+8])[0]
1336         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1337
1338     return corrupt_field(data, crypttexthashtreeoffset, blockhashesoffset-crypttexthashtreeoffset)
1339
1340 def _corrupt_block_hashes(data):
1341     """ Scramble the file data -- the field containing the block hash tree will have one bit
1342     flipped or else will be changed to a random value.
1343     """
1344     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1345     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1346     if sharevernum == 1:
1347         blockhashesoffset = struct.unpack(">L", data[0x0c+0x18:0x0c+0x18+4])[0]
1348         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1349     else:
1350         blockhashesoffset = struct.unpack(">Q", data[0x0c+0x2c:0x0c+0x2c+8])[0]
1351         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1352
1353     return corrupt_field(data, blockhashesoffset, sharehashesoffset-blockhashesoffset)
1354
1355 def _corrupt_share_hashes(data):
1356     """ Scramble the file data -- the field containing the share hash chain will have one
1357     bit flipped or else will be changed to a random value.
1358     """
1359     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1360     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1361     if sharevernum == 1:
1362         sharehashesoffset = struct.unpack(">L", data[0x0c+0x1c:0x0c+0x1c+4])[0]
1363         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1364     else:
1365         sharehashesoffset = struct.unpack(">Q", data[0x0c+0x34:0x0c+0x34+8])[0]
1366         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1367
1368     return corrupt_field(data, sharehashesoffset, uriextoffset-sharehashesoffset)
1369
1370 def _corrupt_length_of_uri_extension(data):
1371     """ Scramble the file data -- the field showing the length of the uri extension will
1372     have one bit flipped or else will be changed to a random value. """
1373     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1374     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1375     if sharevernum == 1:
1376         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1377         return corrupt_field(data, uriextoffset, 4)
1378     else:
1379         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1380         return corrupt_field(data, uriextoffset, 8)
1381
1382 def _corrupt_uri_extension(data):
1383     """ Scramble the file data -- the field containing the uri extension will have one bit
1384     flipped or else will be changed to a random value. """
1385     sharevernum = struct.unpack(">L", data[0x0c:0x0c+4])[0]
1386     assert sharevernum in (1, 2), "This test is designed to corrupt immutable shares of v1 or v2 in specific ways."
1387     if sharevernum == 1:
1388         uriextoffset = struct.unpack(">L", data[0x0c+0x20:0x0c+0x20+4])[0]
1389         uriextlen = struct.unpack(">L", data[0x0c+uriextoffset:0x0c+uriextoffset+4])[0]
1390     else:
1391         uriextoffset = struct.unpack(">Q", data[0x0c+0x3c:0x0c+0x3c+8])[0]
1392         uriextlen = struct.unpack(">Q", data[0x0c+uriextoffset:0x0c+uriextoffset+8])[0]
1393
1394     return corrupt_field(data, uriextoffset, uriextlen)