From 94cdf703e854b30cb6a9351720c9c050a99edb8e Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Tue, 16 Apr 2013 06:59:10 +0000 Subject: [PATCH] new feature: preferred storage servers --- docs/configuration.rst | 18 ++++++++++++++++++ src/allmydata/client.py | 3 ++- src/allmydata/storage_client.py | 8 ++++++-- src/allmydata/test/test_client.py | 14 ++++++++++++-- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 26d8c58b..79272a41 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -386,6 +386,24 @@ Client Configuration .. _performance.rst: performance.rst .. _mutable.rst: specifications/mutable.rst +``peers.preferred = (string, optional)`` + + This is an optional comma-separated list of storage server node IDs that + will be tried first when selecting storage servers for reading or writing. + + Every selected node, preferred or not, will still receive the same number + of shares (one, if there are ``N`` or more servers accepting uploads). + Preferred nodes are simply moved to the front of the server selection lists + computed for each file. + + This is useful if a subset of your nodes have different availability or + connectivity characteristics than the rest of the grid. For instance, if + there are more than ``N`` servers on the grid, and ``K`` or more of them + are at a single physical location, it would make sense for clients at that + location to prefer their local servers so that they can maintain access to + all of their uploads without using the internet. + + Frontend Configuration ====================== diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 64b09c07..e06dad66 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -342,7 +342,8 @@ class Client(node.Node, pollmixin.PollMixin): def init_client_storage_broker(self): # create a StorageFarmBroker object, for use by Uploader/Downloader # (and everybody else who wants to use storage servers) - sb = storage_client.StorageFarmBroker(self.tub, permute_peers=True) + preferred_peers = [p.strip() for p in self.get_config("client", "peers.preferred", "").split(",") if p != ""] + sb = storage_client.StorageFarmBroker(self.tub, permute_peers=True, preferred_peers=preferred_peers) self.storage_broker = sb # load static server specifications from tahoe.cfg, if any. diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 39753e0c..4a6d6928 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -62,10 +62,11 @@ class StorageFarmBroker: I'm also responsible for subscribing to the IntroducerClient to find out about new servers as they are announced by the Introducer. """ - def __init__(self, tub, permute_peers): + def __init__(self, tub, permute_peers, preferred_peers): self.tub = tub assert permute_peers # False not implemented yet self.permute_peers = permute_peers + self.preferred_peers = preferred_peers # self.servers maps serverid -> IServer, and keeps track of all the # storage servers that we've heard about. Each descriptor manages its # own Reconnector, and will give us a RemoteReference when we ask @@ -124,7 +125,10 @@ class StorageFarmBroker: def _permuted(server): seed = server.get_permutation_seed() return sha1(peer_selection_index + seed).digest() - return sorted(self.get_connected_servers(), key=_permuted) + connected_servers = self.get_connected_servers() + preferred_servers = frozenset([s for s in connected_servers if s.get_longname() in self.preferred_peers]) + unpreferred_servers = connected_servers - preferred_servers + return sorted(preferred_servers, key=_permuted) + sorted(unpreferred_servers, key=_permuted) def get_all_serverids(self): return frozenset(self.servers.keys()) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 531215f6..104f064f 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -6,7 +6,7 @@ import allmydata from allmydata.node import Node, OldConfigError, OldConfigOptionError, MissingConfigEntry, UnescapedHashError from allmydata import client from allmydata.storage_client import StorageFarmBroker -from allmydata.util import base32, fileutil +from allmydata.util import base32, fileutil, idlib from allmydata.interfaces import IFilesystemNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, IDirectoryNode from foolscap.api import flushEventualQueue @@ -177,7 +177,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): return [ s.get_longname() for s in sb.get_servers_for_psi(key) ] def test_permute(self): - sb = StorageFarmBroker(None, True) + sb = StorageFarmBroker(None, True, []) for k in ["%d" % i for i in range(5)]: ann = {"anonymous-storage-FURL": "pb://abcde@nowhere/fake", "permutation-seed-base32": base32.b2a(k) } @@ -188,6 +188,16 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): sb.servers.clear() self.failUnlessReallyEqual(self._permute(sb, "one"), []) + def test_permute_with_preferred(self): + sb = StorageFarmBroker(None, True, map(idlib.nodeid_b2a, ['1','4'])) + for k in ["%d" % i for i in range(5)]: + sb.test_add_rref(k, "rref") + + self.failUnlessReallyEqual(self._permute(sb, "one"), ['1','4','3','0','2']) + self.failUnlessReallyEqual(self._permute(sb, "two"), ['4','1','0','2','3']) + sb.servers.clear() + self.failUnlessReallyEqual(self._permute(sb, "one"), []) + def test_versions(self): basedir = "test_client.Basic.test_versions" os.mkdir(basedir) -- 2.45.2