From: Brian Warner <warner@allmydata.com>
Date: Wed, 13 Feb 2008 04:28:52 +0000 (-0700)
Subject: webish.py: fix for #237: when listing large directories, insert a turn break once... 
X-Git-Tag: allmydata-tahoe-0.8.0~76
X-Git-Url: https://git.rkrishnan.org/class-simplejson.JSONDecoder-index.html?a=commitdiff_plain;h=124b2160b930c180fc2abc70497be35c4728a71e;p=tahoe-lafs%2Ftahoe-lafs.git

webish.py: fix for #237: when listing large directories, insert a turn break once every 100 children, to work around non-optimized tail recursion Deferreds
---

diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py
index a7728863..fa374e90 100644
--- a/src/allmydata/webish.py
+++ b/src/allmydata/webish.py
@@ -18,6 +18,7 @@ from allmydata import get_package_versions_string
 from zope.interface import implements, Interface
 import urllib
 from formless import webform
+from foolscap.eventual import fireEventually
 
 from nevow.util import resource_filename
 
@@ -209,6 +210,25 @@ class Directory(rend.Page):
     def data_children(self, ctx, data):
         d = self._dirnode.list()
         d.addCallback(lambda dict: sorted(dict.items()))
+        def _stall_some(items):
+            # Deferreds don't optimize out tail recursion, and the way
+            # Nevow's flattener handles Deferreds doesn't take this into
+            # account. As a result, large lists of Deferreds that fire in the
+            # same turn (i.e. the output of defer.succeed) will cause a stack
+            # overflow. To work around this, we insert a turn break after
+            # every 100 items, using foolscap's fireEventually(). This gives
+            # the stack a chance to be popped. It would also work to put
+            # every item in its own turn, but that'd be a lot more
+            # inefficient. This addresses ticket #237, for which I was never
+            # able to create a failing unit test.
+            output = []
+            for i,item in enumerate(items):
+                if i % 100 == 0:
+                    output.append(fireEventually(item))
+                else:
+                    output.append(item)
+            return output
+        d.addCallback(_stall_some)
         return d
 
     def render_row(self, ctx, data):