From 684966d103b7cc95f4895f106c3719e790a806b7 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Thu, 9 Aug 2007 18:26:17 -0700 Subject: [PATCH] encode.py: add a reactor turn barrier between segments, to allow Deferreds to retire and free their arguments, all in the name of reducing memory footprint --- src/allmydata/encode.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/allmydata/encode.py b/src/allmydata/encode.py index e1de2eae..b79ef370 100644 --- a/src/allmydata/encode.py +++ b/src/allmydata/encode.py @@ -3,6 +3,7 @@ from zope.interface import implements from twisted.internet import defer from twisted.python import log +from foolscap import eventual from allmydata import uri from allmydata.hashtree import HashTree from allmydata.util import mathutil, hashutil @@ -212,8 +213,9 @@ class Encoder(object): # that we sent to that landlord. self.share_root_hashes = [None] * self.num_shares - d = defer.maybeDeferred(self._uploadable.set_segment_size, - self.segment_size) + d = eventual.fireEventually() + d.addCallback(lambda res: + self._uploadable.set_segment_size(self.segment_size)) for l in self.landlords.values(): d.addCallback(lambda res, l=l: l.start()) @@ -225,9 +227,11 @@ class Encoder(object): # use this form instead: d.addCallback(lambda res, i=i: self._encode_segment(i)) d.addCallback(self._send_segment, i) + d.addCallback(self._turn_barrier) last_segnum = self.num_segments - 1 d.addCallback(lambda res: self._encode_tail_segment(last_segnum)) d.addCallback(self._send_segment, last_segnum) + d.addCallback(self._turn_barrier) d.addCallback(lambda res: self.finish_hashing()) @@ -243,6 +247,16 @@ class Encoder(object): d.addCallbacks(lambda res: self.done(), self.err) return d + def _turn_barrier(self, res): + # putting this method in a Deferred chain imposes a guaranteed + # reactor turn between the pre- and post- portions of that chain. + # This can be useful to limit memory consumption: since Deferreds do + # not do tail recursion, code which uses defer.succeed(result) for + # consistency will cause objects to live for longer than you might + # normally expect. + + return eventual.fireEventually(res) + def _encode_segment(self, segnum): codec = self._codec -- 2.45.2