results.sort()
return results
+ def get_encoding_parameters(self):
+ if not self.introducer_client:
+ return None
+ return self.introducer_client.encoding_parameters
+
def connected_to_introducer(self):
if self.introducer_client:
return self.introducer_client.connected_to_introducer()
return self.required_shares
def decode(self, some_shares, their_shareids):
- precondition(len(some_shares) == len(their_shareids), len(some_shares), len(their_shareids))
- precondition(len(some_shares) == self.required_shares, len(some_shares), self.required_shares)
- return defer.succeed(self.decoder.decode(some_shares, [int(s) for s in their_shareids]))
+ precondition(len(some_shares) == len(their_shareids),
+ len(some_shares), len(their_shareids))
+ precondition(len(some_shares) == self.required_shares,
+ len(some_shares), self.required_shares)
+ data = self.decoder.decode(some_shares,
+ [int(s) for s in their_shareids])
+ return defer.succeed(data)
all_encoders = {
self.TOTAL_SHARES = n
self.uri_extension_data = {}
+ def set_params(self, encoding_parameters):
+ k,d,n = encoding_parameters
+ self.NEEDED_SHARES = k
+ self.SHARES_OF_HAPPINESS = d
+ self.TOTAL_SHARES = n
+
def setup(self, infile, encryption_key):
self.infile = infile
assert isinstance(encryption_key, str)
class RIIntroducerClient(RemoteInterface):
def new_peers(furls=SetOf(FURL)):
return None
+ def set_encoding_parameters(parameters=(int, int, int)):
+ """Advise the client of the recommended k-of-n encoding parameters
+ for this grid. 'parameters' is a tuple of (k, desired, n), where 'n'
+ is the total number of shares that will be created for any given
+ file, while 'k' is the number of shares that must be retrieved to
+ recover that file, and 'desired' is the minimum number of shares that
+ must be placed before the uploader will consider its job a success.
+ n/k is the expansion ratio, while k determines the robustness.
+
+ Introducers should specify 'n' according to the expected size of the
+ grid (there is no point to producing more shares than there are
+ peers), and k according to the desired reliability-vs-overhead goals.
+
+ Note that the current encoding technology requires k>=2.
+ """
+ return None
class RIIntroducer(RemoteInterface):
def hello(node=RIIntroducerClient, furl=FURL):
class Introducer(service.MultiService, Referenceable):
implements(RIIntroducer)
+ name = "introducer"
def __init__(self):
service.MultiService.__init__(self)
self.nodes = set()
self.furls = set()
+ self._encoding_parameters = None
+
+ def set_encoding_parameters(self, parameters):
+ self._encoding_parameters = parameters
def remote_hello(self, node, furl):
log.msg("introducer: new contact at %s, node is %s" % (furl, node))
node.notifyOnDisconnect(_remove)
self.furls.add(furl)
node.callRemote("new_peers", self.furls)
+ if self._encoding_parameters is not None:
+ node.callRemote("set_encoding_parameters",
+ self._encoding_parameters)
for othernode in self.nodes:
othernode.callRemote("new_peers", set([furl]))
self.nodes.add(node)
self._connected = False
self.connection_observers = observer.ObserverList()
+ self.encoding_parameters = None
def startService(self):
service.Service.startService(self)
for furl in furls:
self._new_peer(furl)
+ def remote_set_encoding_parameters(self, parameters):
+ self.encoding_parameters = parameters
+
def stopService(self):
service.Service.stopService(self)
self.introducer_reconnector.stopConnecting()
PORTNUMFILE = "introducer.port"
NODETYPE = "introducer"
VDRIVEDIR = "vdrive"
+ ENCODING_PARAMETERS_FILE = "encoding_parameters"
+ DEFAULT_K, DEFAULT_DESIRED, DEFAULT_N = 3, 7, 10
def __init__(self, basedir="."):
node.Node.__init__(self, basedir)
self.urls = {}
+ self.read_encoding_parameters()
def tub_ready(self):
- r = self.add_service(Introducer())
+ i = Introducer()
+ r = self.add_service(i)
self.urls["introducer"] = self.tub.registerReference(r, "introducer")
self.log(" introducer is at %s" % self.urls["introducer"])
f = open(os.path.join(self.basedir, "introducer.furl"), "w")
f.write(self.urls["vdrive"] + "\n")
f.close()
+ encoding_parameters = self.read_encoding_parameters()
+ i.set_encoding_parameters(encoding_parameters)
+
+ def read_encoding_parameters(self):
+ k, desired, n = self.DEFAULT_K, self.DEFAULT_DESIRED, self.DEFAULT_N
+ PARAM_FILE = os.path.join(self.basedir, self.ENCODING_PARAMETERS_FILE)
+ if os.path.exists(PARAM_FILE):
+ f = open(PARAM_FILE, "r")
+ data = f.read().strip()
+ f.close()
+ k,desired,n = data.split()
+ k = int(k); desired = int(desired); n = int(n)
+ return k, desired, n
+
+import os
from twisted.trial import unittest
from foolscap.eventual import fireEventually, flushEventualQueue
class Basic(testutil.SignalMixin, unittest.TestCase):
def test_loadable(self):
- q = introducer_and_vdrive.IntroducerAndVdrive()
+ basedir = "introducer_and_vdrive.Basic.test_loadable"
+ os.mkdir(basedir)
+ q = introducer_and_vdrive.IntroducerAndVdrive(basedir)
d = fireEventually(None)
d.addCallback(lambda res: q.startService())
+ d.addCallback(lambda res: q.when_tub_ready())
+ def _check_parameters(res):
+ i = q.getServiceNamed("introducer")
+ self.failUnlessEqual(i._encoding_parameters, (3, 7, 10))
+ d.addCallback(_check_parameters)
+ d.addCallback(lambda res: q.stopService())
+ d.addCallback(flushEventualQueue)
+ return d
+
+ def test_set_parameters(self):
+ basedir = "introducer_and_vdrive.Basic.test_set_parameters"
+ os.mkdir(basedir)
+ f = open(os.path.join(basedir, "encoding_parameters"), "w")
+ f.write("25 75 100")
+ f.close()
+ q = introducer_and_vdrive.IntroducerAndVdrive(basedir)
+ d = fireEventually(None)
+ d.addCallback(lambda res: q.startService())
+ d.addCallback(lambda res: q.when_tub_ready())
+ def _check_parameters(res):
+ i = q.getServiceNamed("introducer")
+ self.failUnlessEqual(i._encoding_parameters, (25, 75, 100))
+ d.addCallback(_check_parameters)
d.addCallback(lambda res: q.stopService())
d.addCallback(flushEventualQueue)
return d
self.failUnless("size: %d\n" % len(self.data) in output)
self.failUnless("num_segments: 1\n" in output)
# segment_size is always a multiple of needed_shares
- self.failUnless("segment_size: 125\n" in output)
- self.failUnless("total_shares: 100\n" in output)
+ self.failUnless("segment_size: 114\n" in output)
+ self.failUnless("total_shares: 10\n" in output)
# keys which are supposed to be present
for key in ("size", "num_segments", "segment_size",
"needed_shares", "total_shares",
def get_permuted_peers(self, storage_index):
return [ ("%20d"%fakeid, "%20d"%fakeid, FakePeer(self.mode),)
for fakeid in range(50) ]
+ def get_encoding_parameters(self):
+ return None
DATA = """
Once upon a time, there was a beautiful princess named Buttercup. She lived
self._client = client
self._options = options
- def set_params(self, needed_shares, shares_of_happiness, total_shares):
+ def set_params(self, encoding_parameters):
+ self._encoding_parameters = encoding_parameters
+
+ needed_shares, shares_of_happiness, total_shares = encoding_parameters
self.needed_shares = needed_shares
self.shares_of_happiness = shares_of_happiness
self.total_shares = total_shares
def setup_encoder(self):
self._encoder = encode.Encoder(self._options)
+ self._encoder.set_params(self._encoding_parameters)
self._encoder.setup(self._filehandle, self._encryption_key)
share_size = self._encoder.get_share_size()
block_size = self._encoder.get_block_size()
uploader_class = FileUploader
URI_LIT_SIZE_THRESHOLD = 55
- needed_shares = 25 # Number of shares required to reconstruct a file.
- desired_shares = 75 # We will abort an upload unless we can allocate space for at least this many.
- total_shares = 100 # Total number of shares created by encoding. If everybody has room then this is is how many we will upload.
+ DEFAULT_ENCODING_PARAMETERS = (25, 75, 100)
+ # this is a tuple of (needed, desired, total). 'needed' is the number of
+ # shares required to reconstruct a file. 'desired' means that we will
+ # abort an upload unless we can allocate space for at least this many.
+ # 'total' is the total number of shares created by encoding. If everybody
+ # has room then this is is how many we will upload.
def compute_id_strings(self, f):
# return a list of (plaintext_hash, encryptionkey, crypttext_hash)
else:
u = self.uploader_class(self.parent, options)
u.set_filehandle(fh)
- u.set_params(self.needed_shares, self.desired_shares,
- self.total_shares)
+ encoding_parameters = self.parent.get_encoding_parameters()
+ if not encoding_parameters:
+ encoding_parameters = self.DEFAULT_ENCODING_PARAMETERS
+ u.set_params(encoding_parameters)
plaintext_hash, key, crypttext_hash = self.compute_id_strings(fh)
u.set_encryption_key(key)
u.set_id_strings(crypttext_hash, plaintext_hash)
projects / tahoe-lafs / tahoe-lafs.git / commitdiff