From 3cb361e233054121c3b94b695d77d5636a8ea4c5 Mon Sep 17 00:00:00 2001
From: Brian Warner <warner@allmydata.com>
Date: Wed, 7 May 2008 18:36:37 -0700
Subject: [PATCH] dirnode: use the concurrency limiter in t=manifest and
t=deep-size, allow 10 retrievals in parallel
---
src/allmydata/dirnode.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py
index 99379779..96eb8916 100644
--- a/src/allmydata/dirnode.py
+++ b/src/allmydata/dirnode.py
@@ -10,6 +10,7 @@ from allmydata.interfaces import IMutableFileNode, IDirectoryNode,\
IURI, IFileNode, IMutableFileURI, IVerifierURI, IFilesystemNode
from allmydata.util import hashutil
from allmydata.util.hashutil import netstring
+from allmydata.util.limiter import ConcurrencyLimiter
from allmydata.uri import NewDirectoryURI
from pycryptopp.cipher.aes import AES
@@ -425,12 +426,14 @@ class NewDirectoryNode:
(directories and files) reachable from this one."""
# this is just a tree-walker, except that following each edge
- # requires a Deferred.
+ # requires a Deferred. We use a ConcurrencyLimiter to make sure the
+ # fan-out doesn't cause problems.
manifest = set()
manifest.add(self.get_verifier())
+ limiter = ConcurrencyLimiter(10) # allow 10 in parallel
- d = self._build_manifest_from_node(self, manifest)
+ d = self._build_manifest_from_node(self, manifest, limiter)
def _done(res):
# LIT nodes have no verifier-capability: their data is stored
# inside the URI itself, so there is no need to refresh anything.
@@ -443,8 +446,8 @@ class NewDirectoryNode:
d.addCallback(_done)
return d
- def _build_manifest_from_node(self, node, manifest):
- d = node.list()
+ def _build_manifest_from_node(self, node, manifest, limiter):
+ d = limiter.add(node.list)
def _got_list(res):
dl = []
for name, (child, metadata) in res.iteritems():
@@ -453,7 +456,8 @@ class NewDirectoryNode:
manifest.add(verifier)
if IDirectoryNode.providedBy(child):
dl.append(self._build_manifest_from_node(child,
- manifest))
+ manifest,
+ limiter))
if dl:
return defer.DeferredList(dl)
d.addCallback(_got_list)
--
2.45.2