From 32547bbd102cfc00c05a25a77e3c90cbf029e489 Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Fri, 4 Apr 2014 15:34:20 +0100 Subject: [PATCH] Move a utility function into allmydata.util. Signed-off-by: Daira Hopwood --- .../storage/backends/cloud/cloud_common.py | 19 +----------------- src/allmydata/test/test_storage.py | 5 ----- src/allmydata/test/test_util.py | 11 +++++++++- src/allmydata/util/listutil.py | 20 +++++++++++++++++++ 4 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 src/allmydata/util/listutil.py diff --git a/src/allmydata/storage/backends/cloud/cloud_common.py b/src/allmydata/storage/backends/cloud/cloud_common.py index c3d06068..f07e3814 100644 --- a/src/allmydata/storage/backends/cloud/cloud_common.py +++ b/src/allmydata/storage/backends/cloud/cloud_common.py @@ -16,6 +16,7 @@ from allmydata.interfaces import IShareBase from allmydata.util import log from allmydata.util.assertutil import precondition, _assert from allmydata.util.deferredutil import eventually_callback, eventually_errback, eventual_chain, gatherResults +from allmydata.util.listutil import concat from allmydata.storage.common import si_b2a, NUM_RE @@ -458,24 +459,6 @@ class CommonContainerMixin: return self._do_request('DELETE object', self._delete_object, object_name) -def concat(seqs): - """ - O(n), rather than O(n^2), concatenation of list-like things, returning a list. - I can't believe this isn't built in. - """ - total_len = 0 - for seq in seqs: - total_len += len(seq) - result = [None]*total_len - i = 0 - for seq in seqs: - for x in seq: - result[i] = x - i += 1 - _assert(i == total_len, i=i, total_len=total_len) - return result - - class ContainerListMixin: """ S3 has a limitation of 1000 object entries returned on each list (GET Bucket) request. diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 59ba8332..dc23c73b 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1,6 +1,5 @@ import time, os.path, platform, re, simplejson, struct, itertools, urllib -from collections import deque from cStringIO import StringIO import thread @@ -365,10 +364,6 @@ class Seek(unittest.TestCase, WorkdirMixin): class CloudCommon(unittest.TestCase, ShouldFailMixin, WorkdirMixin): - def test_concat(self): - x = deque([[1, 2], (), xrange(3, 6)]) - self.failUnlessEqual(cloud_common.concat(x), [1, 2, 3, 4, 5]) - def test_list_objects_truncated_badly(self): # If a container misbehaves by not producing listings with increasing keys, # that should cause an incident. diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 95fae0ec..bcda0cbb 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -2,7 +2,9 @@ def foo(): pass # keep the line number constant import os, time, sys +from collections import deque from StringIO import StringIO + from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python.failure import Failure @@ -12,7 +14,7 @@ from pycryptopp.hash.sha256 import SHA256 as _hash from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil from allmydata.util import assertutil, fileutil, deferredutil, abbreviate from allmydata.util import limiter, time_format, pollmixin, cachedir -from allmydata.util import statistics, dictutil, pipeline +from allmydata.util import statistics, dictutil, listutil, pipeline from allmydata.util import log as tahoe_log from allmydata.util.spans import Spans, overlap, DataSpans @@ -1431,6 +1433,13 @@ class DictUtil(unittest.TestCase): self.failUnlessEqual(d["one"], 1) self.failUnlessEqual(d.get_aux("one"), None) + +class ListUtil(unittest.TestCase): + def test_concat(self): + x = deque([[1, 2], (), xrange(3, 6)]) + self.failUnlessEqual(listutil.concat(x), [1, 2, 3, 4, 5]) + + class Pipeline(unittest.TestCase): def pause(self, *args, **kwargs): d = defer.Deferred() diff --git a/src/allmydata/util/listutil.py b/src/allmydata/util/listutil.py new file mode 100644 index 00000000..56b0ee25 --- /dev/null +++ b/src/allmydata/util/listutil.py @@ -0,0 +1,20 @@ + +from allmydata.util.assertutil import _assert + + +def concat(seqs): + """ + O(n), rather than O(n^2), concatenation of list-like things, returning a list. + I can't believe this isn't built in. + """ + total_len = 0 + for seq in seqs: + total_len += len(seq) + result = [None]*total_len + i = 0 + for seq in seqs: + for x in seq: + result[i] = x + i += 1 + _assert(i == total_len, i=i, total_len=total_len) + return result -- 2.45.2