From 0fab511be531c6f34d4f8f24bb133afbaf78d1b0 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 21 Nov 2008 20:28:12 -0700 Subject: [PATCH] upload: don't use servers which can't support the share size we need. This ought to avoid #439 problems. Some day we'll have a storage server which advertises support for a larger share size. No tests yet. --- src/allmydata/immutable/upload.py | 20 ++++++++++++++++++-- src/allmydata/test/test_upload.py | 6 ++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 6ebcdc68..3b653d8a 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -164,8 +164,6 @@ class Tahoe2PeerSelector: if not peers: raise NotEnoughSharesError("client gave us zero peers") - # figure out how much space to ask for - # this needed_hashes computation should mirror # Encoder.send_all_share_hash_trees. We use an IncompleteHashTree # (instead of a HashTree) because we don't require actual hashing @@ -173,6 +171,24 @@ class Tahoe2PeerSelector: ht = hashtree.IncompleteHashTree(total_shares) num_share_hashes = len(ht.needed_hashes(0, include_leaf=True)) + # figure out how much space to ask for + allocated_size = layout.allocated_size(share_size, + num_segments, + num_share_hashes, + EXTENSION_SIZE) + # filter the list of peers according to which ones can accomodate + # this request. This excludes older peers (which used a 4-byte size + # field) from getting large shares (for files larger than about + # 12GiB). See #439 for details. + def _get_maxsize(peer): + (peerid, conn) = peer + v1 = conn.version["http://allmydata.org/tahoe/protocols/storage/v1"] + return v1["maximum-immutable-share-size"] + peers = [peer for peer in peers + if _get_maxsize(peer) >= allocated_size] + if not peers: + raise NotEnoughSharesError("no peers could accept an allocated_size of %d" % allocated_size) + # decide upon the renewal/cancel secrets, to include them in the # allocat_buckets query. client_renewal_secret = client.get_renewal_secret() diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 6671b361..a9536b68 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -7,6 +7,7 @@ from twisted.python import log from twisted.internet import defer from foolscap import eventual +import allmydata from allmydata import uri, monitor from allmydata.immutable import upload from allmydata.interfaces import IFileURI, FileTooLargeError, NotEnoughSharesError @@ -81,6 +82,11 @@ class FakeStorageServer: self.mode = mode self.allocated = [] self.queries = 0 + self.version = { "http://allmydata.org/tahoe/protocols/storage/v1" : + { "maximum-immutable-share-size": 2**32 }, + "application-version": str(allmydata.__version__), + } + def callRemote(self, methname, *args, **kwargs): def _call(): meth = getattr(self, methname) -- 2.45.2