From 6e3396fb88c7137037c3fcd497239d72450a2247 Mon Sep 17 00:00:00 2001 From: Zooko O'Whielacronx Date: Sat, 10 Jan 2009 11:46:23 -0700 Subject: [PATCH] immutable: redefine the "sharemap" member of the upload results to be a map from shnum to set of serverids It used to be a map from shnum to a string saying "placed this share on XYZ server". The new definition is more in keeping with the "sharemap" object that results from immutable file checking and repair, and it is more useful to the repairer, which is a consumer of immutable upload results. --- src/allmydata/immutable/upload.py | 25 ++++++++++--------------- src/allmydata/interfaces.py | 8 +++++++- src/allmydata/web/status.py | 6 ++++-- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 5149ba9d..41b338e3 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -48,8 +48,8 @@ class UploadResults(Copyable, RemoteCopy): def __init__(self): self.timings = {} # dict of name to number of seconds - self.sharemap = {} # dict of shnum to placement string - self.servermap = {} # dict of peerid to set(shnums) + self.sharemap = {} # k: shnum, v: set(serverid) + self.servermap = {} # k: serverid, v: set(shnum) self.file_size = None self.ciphertext_fetched = None # how much the helper fetched self.uri = None @@ -654,6 +654,9 @@ class CHKUploader: self._upload_status.set_active(True) self._upload_status.set_results(self._results) + # locate_all_shareholders() will create the following attribute: + # self._peer_trackers = {} # k: shnum, v: instance of PeerTracker + def log(self, *args, **kwargs): if "parent" not in kwargs: kwargs["parent"] = self._log_number @@ -735,22 +738,16 @@ class CHKUploader: """ self.log("_send_shares, used_peers is %s" % (used_peers,)) # record already-present shares in self._results - for (shnum, peerid) in already_peers.items(): - peerid_s = idlib.shortnodeid_b2a(peerid) - self._results.sharemap[shnum] = "Found on [%s]" % peerid_s - if peerid not in self._results.servermap: - self._results.servermap[peerid] = set() - self._results.servermap[peerid].add(shnum) self._results.preexisting_shares = len(already_peers) - self._sharemap = {} + self._peer_trackers = {} # k: shnum, v: instance of PeerTracker for peer in used_peers: assert isinstance(peer, PeerTracker) buckets = {} for peer in used_peers: buckets.update(peer.buckets) for shnum in peer.buckets: - self._sharemap[shnum] = peer + self._peer_trackers[shnum] = peer assert len(buckets) == sum([len(peer.buckets) for peer in used_peers]) encoder.set_shareholders(buckets) @@ -758,13 +755,11 @@ class CHKUploader: """ Returns a Deferred that will fire with the UploadResults instance. """ r = self._results for shnum in self._encoder.get_shares_placed(): - peer_tracker = self._sharemap[shnum] + peer_tracker = self._peer_trackers[shnum] peerid = peer_tracker.peerid peerid_s = idlib.shortnodeid_b2a(peerid) - r.sharemap[shnum] = "Placed on [%s]" % peerid_s - if peerid not in r.servermap: - r.servermap[peerid] = set() - r.servermap[peerid].add(shnum) + r.sharemap.setdefault(shnum, set()).add(peerid) + r.servermap.setdefault(peerid, set()).add(shnum) r.pushed_shares = len(self._encoder.get_shares_placed()) now = time.time() r.file_size = self._encoder.file_size diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 12972742..54176c69 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1460,7 +1460,13 @@ class IUploadResults(Interface): .file_size : the size of the file, in bytes .uri : the CHK read-cap for the file .ciphertext_fetched : how many bytes were fetched by the helper - .sharemap : dict mapping share number to placement string + .renamed_sharemap: dict mapping share identifier to set of serverids + (binary strings). This indicates which servers were given + which shares. For immutable files, the shareid is an + integer (the share number, from 0 to N-1). For mutable + files, it is a string of the form 'seq%d-%s-sh%d', + containing the sequence number, the roothash, and the + share number. .servermap : dict mapping server peerid to a set of share numbers .timings : dict of timing information, mapping name to seconds (float) total : total upload time, start to finish diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index f97ca515..9f508813 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -46,8 +46,10 @@ class UploadResultsRendererMixin(RateAndTimeMixin): if sharemap is None: return "None" l = T.ul() - for shnum in sorted(sharemap.keys()): - l[T.li["%d -> %s" % (shnum, sharemap[shnum])]] + for shnum, peerids in sorted(sharemap.items()): + for peerid in peerids: + peerid_s = idlib.shortnodeid_b2a(peerid) + l[T.li["%d -> %s" % (shnum, peerid_s)]] return l d.addCallback(_render) return d -- 2.45.2