From ce4610c3e6a5a55f524a475853753aa54c1dfcbf Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 20 Jan 2007 15:50:21 -0700 Subject: [PATCH] more filetree work, more tests now pass --- src/allmydata/filetree/directory.py | 42 ++++++++++++------------- src/allmydata/filetree/interfaces.py | 22 +++++++++++-- src/allmydata/filetree/opener.py | 4 ++- src/allmydata/filetree/redirect.py | 16 ++++++---- src/allmydata/filetree/vdrive.py | 19 ++++++----- src/allmydata/test/test_filetree_new.py | 30 +++++++++++++----- 6 files changed, 89 insertions(+), 44 deletions(-) diff --git a/src/allmydata/filetree/directory.py b/src/allmydata/filetree/directory.py index 5360ee97..859e36f7 100644 --- a/src/allmydata/filetree/directory.py +++ b/src/allmydata/filetree/directory.py @@ -3,7 +3,7 @@ from zope.interface import implements from twisted.internet import defer from cStringIO import StringIO from allmydata.filetree.interfaces import ( - INode, IDirectoryNode, ISubTree, + INode, INodeMaker, IDirectoryNode, ISubTree, ICHKDirectoryNode, ISSKDirectoryNode, NoSuchChildError, ) @@ -19,17 +19,14 @@ from allmydata.util import bencode # each time the vdrive changes, update the local drive to match, and # vice versa. -# from the itertools 'recipes' page -from itertools import izip, tee -def pairwise(iterable): - "s -> (s0,s1), (s1,s2), (s2, s3), ..." - a, b = tee(iterable) - try: - b.next() - except StopIteration: - pass +from itertools import islice, izip +def in_pairs(iterable): + "s -> (s0,s1), (s2,s3), (s4,s5), ..." + a = islice(iterable, 0, None, 2) + b = islice(iterable, 1, None, 2) return izip(a, b) + class SubTreeNode: implements(INode, IDirectoryNode) @@ -95,15 +92,16 @@ class SubTreeNode: data.append(self.children[name].serialize_node()) return data - def populate_node(self, data, node_maker): + def populate_dirnode(self, data, node_maker): + assert INodeMaker(node_maker) assert len(data) % 2 == 0 - for (name, child_data) in pairwise(data): + for (name, child_data) in in_pairs(data): if isinstance(child_data, (list, tuple)): child = SubTreeNode(self.enclosing_tree) - child.populate_node(child_data) + child.populate_dirnode(child_data, node_maker) else: assert isinstance(child_data, str) - child = node_maker(child_data) + child = node_maker.make_node_from_serialized(child_data) self.children[name] = child @@ -141,7 +139,7 @@ class _DirectorySubTree(object): def _populate_from_data(self, data, node_maker): self.root = SubTreeNode(self) - self.root.populate_node(bencode.bdecode(data), node_maker) + self.root.populate_dirnode(bencode.bdecode(data), node_maker) return self def serialize_subtree_to_file(self, f): @@ -205,14 +203,16 @@ class LocalFileSubTree(_DirectorySubTree): f = open(self.filename, "rb") data = f.read() f.close() - return defer.succeed(self._populate_from_data(node_maker)) + d = defer.succeed(data) + d.addCallback(self._populate_from_data, node_maker) + return d def create_node_now(self): return LocalFileSubTreeNode().new(self.filename) def _update(self): f = open(self.filename, "wb") - self.serialize_to_file(f) + self.serialize_subtree_to_file(f) f.close() def update_now(self, uploader): @@ -258,7 +258,7 @@ class CHKDirectorySubTree(_DirectorySubTree): def update_now(self, uploader): f = StringIO() - self.serialize_to_file(f) + self.serialize_subtree_to_file(f) data = f.getvalue() d = uploader.upload_data(data) def _uploaded(uri): @@ -271,7 +271,7 @@ class CHKDirectorySubTree(_DirectorySubTree): # this is the CHK form old_uri = self.uri f, filename = workqueue.create_tempfile(".chkdir") - self.serialize_to_file(f) + self.serialize_subtree_to_file(f) f.close() boxname = workqueue.create_boxname() workqueue.add_upload_chk(filename, boxname) @@ -339,7 +339,7 @@ class SSKDirectorySubTree(_DirectorySubTree): raise RuntimeError("This SSKDirectorySubTree is not mutable") f = StringIO() - self.serialize_to_file(f) + self.serialize_subtree_to_file(f) data = f.getvalue() self.version += 1 @@ -350,7 +350,7 @@ class SSKDirectorySubTree(_DirectorySubTree): def update(self, workqueue): # this is the SSK form f, filename = workqueue.create_tempfile(".sskdir") - self.serialize_to_file(f) + self.serialize_subtree_to_file(f) f.close() oldversion = self.version diff --git a/src/allmydata/filetree/interfaces.py b/src/allmydata/filetree/interfaces.py index 353b77a0..49d19504 100644 --- a/src/allmydata/filetree/interfaces.py +++ b/src/allmydata/filetree/interfaces.py @@ -17,8 +17,11 @@ class INode(Interface): .prefix .""" def populate_node(body, node_maker): """vdrive.make_node_from_serialized() will first use the prefix from - the .prefix attribute to decide what kind of Node to create. It will - then call this function with the body to populate the new Node.""" + the .prefix attribute to decide what kind of Node to create. They + will then call this populate_node() method with the body to populate + the new Node. 'node_maker' provides INodeMaker, which provides that + same make_node_from_serialized function to create any internal child + nodes that might be necessary.""" class IFileNode(Interface): """This is a file which can be retrieved.""" @@ -190,6 +193,21 @@ class ISubTree(Interface): (SSKDirectorySubTrees and redirections). """ +class INodeMaker(Interface): + def make_node_from_serialized(serialized): + """Turn a string into an INode, which contains information about + the file or directory (like a URI), but does not contain the actual + contents. An IOpener can be used later to retrieve the contents + (which means downloading the file if this is an IFileNode, or + perhaps creating a new subtree from the contents).""" + +class ISubTreeMaker(Interface): + def make_subtree_from_node(node, parent_is_mutable): + """Turn an INode into an ISubTree (using an internal opener to + download the data, if necessary). + This returns a Deferred that fires with the ISubTree instance. + """ + #class IMutableSubTree(Interface): # def mutation_affects_parent(): # """This returns True for CHK nodes where you must inform the parent diff --git a/src/allmydata/filetree/opener.py b/src/allmydata/filetree/opener.py index a536d709..1f052c11 100644 --- a/src/allmydata/filetree/opener.py +++ b/src/allmydata/filetree/opener.py @@ -3,7 +3,7 @@ from zope.interface import implements from twisted.internet import defer from allmydata.filetree import interfaces, directory, redirect #from allmydata.filetree.file import CHKFile, MutableSSKFile, ImmutableSSKFile -from allmydata.filetree.interfaces import INode, IDirectoryNode +from allmydata.filetree.interfaces import INode, IDirectoryNode, INodeMaker all_openable_subtree_types = [ directory.LocalFileSubTree, @@ -27,6 +27,7 @@ class Opener(object): def _create(self, node, parent_is_mutable, node_maker): assert INode(node) + assert INodeMaker(node_maker) for subtree_class in all_openable_subtree_types: if isinstance(node, subtree_class.node_class): subtree = subtree_class() @@ -41,6 +42,7 @@ class Opener(object): def open(self, node, parent_is_mutable, node_maker): assert INode(node) assert not isinstance(node, IDirectoryNode) + assert INodeMaker(node_maker) # is it in cache? To check this we need to use the node's serialized # form, since nodes are instances and don't compare by value diff --git a/src/allmydata/filetree/redirect.py b/src/allmydata/filetree/redirect.py index f1217563..b7fa12a9 100644 --- a/src/allmydata/filetree/redirect.py +++ b/src/allmydata/filetree/redirect.py @@ -3,7 +3,7 @@ from cStringIO import StringIO from zope.interface import implements from twisted.internet import defer -from allmydata.filetree.interfaces import ISubTree +from allmydata.filetree.interfaces import ISubTree, INodeMaker from allmydata.filetree.basenode import BaseDataNode from allmydata.util import bencode @@ -33,7 +33,8 @@ class _BaseRedirection(object): return self.child_node.serialize_node() def _populate_from_data(self, data, node_maker): - self.child_node = node_maker(data) + assert INodeMaker(node_maker) + self.child_node = node_maker.make_node_from_serialized(data) return self @@ -64,7 +65,9 @@ class LocalFileRedirection(_BaseRedirection): # note: we don't cache the contents of the file. TODO: consider # doing this based upon mtime. It is important that we be able to # notice if the file has been changed. - return defer.succeed(self._populate_from_data(data, node_maker)) + d = defer.succeed(data) + d.addCallback(self._populate_from_data, node_maker) + return d def is_mutable(self): return True @@ -161,10 +164,11 @@ class QueenOrLocalFileRedirection(_BaseRedirection): # TODO: queen? # TODO: pubsub so we can cache the queen's results d = self._queen.callRemote("lookup_handle", self.handle) - d.addCallback(self._choose_winner, local_version_and_data, node_maker) + d.addCallback(self._choose_winner, local_version_and_data) + d.addCallback(self._populate_from_data, node_maker) return d - def _choose_winner(self, queen_version_and_data, local_version_and_data, node_maker): + def _choose_winner(self, queen_version_and_data, local_version_and_data): queen_version, queen_data = bencode.bdecode(queen_version_and_data) local_version, local_data = bencode.bdecode(local_version_and_data) if queen_version > local_version: @@ -174,7 +178,7 @@ class QueenOrLocalFileRedirection(_BaseRedirection): data = local_data self.version = local_version # NOTE: two layers of bencoding here, TODO - return self._populate_from_data(data, node_maker) + return data def is_mutable(self): return True diff --git a/src/allmydata/filetree/vdrive.py b/src/allmydata/filetree/vdrive.py index cc0e6a9d..3868ee2b 100644 --- a/src/allmydata/filetree/vdrive.py +++ b/src/allmydata/filetree/vdrive.py @@ -1,8 +1,8 @@ from zope.interface import implements -from allmydata.filetree import opener, directory, redirect +from allmydata.filetree import opener, directory, file, redirect from allmydata.filetree.interfaces import ( - IVirtualDrive, INode, ISubTree, IFileNode, IDirectoryNode, + IVirtualDrive, INodeMaker, INode, ISubTree, IFileNode, IDirectoryNode, NoSuchDirectoryError, NoSuchChildError, PathAlreadyExistsError, PathDoesNotExistError, ) @@ -12,8 +12,11 @@ from allmydata.upload import IUploadable # node specification strings (found inside the serialized form of subtrees) # into Nodes (which live in the in-RAM form of subtrees). all_node_types = [ + directory.LocalFileSubTreeNode, directory.CHKDirectorySubTreeNode, directory.SSKDirectorySubTreeNode, + file.CHKFileNode, + file.SSKFileNode, redirect.LocalFileRedirectionNode, redirect.QueenRedirectionNode, redirect.HTTPRedirectionNode, @@ -21,7 +24,7 @@ all_node_types = [ ] class VirtualDrive(object): - implements(IVirtualDrive) + implements(IVirtualDrive, INodeMaker) def __init__(self, workqueue, downloader, root_node): assert INode(root_node) @@ -33,6 +36,8 @@ class VirtualDrive(object): self.root_node = root_node # these are called when loading and creating nodes + + # INodeMaker def make_node_from_serialized(self, serialized): # this turns a string into an INode, which contains information about # the file or directory (like a URI), but does not contain the actual @@ -47,14 +52,14 @@ class VirtualDrive(object): for node_class in all_node_types: if prefix == node_class.prefix: node = node_class() - node.populate_node(body, self.make_node_from_serialized) + node.populate_node(body, self) return node - raise RuntimeError("unable to handle subtree type '%s'" % prefix) + raise RuntimeError("unable to handle node type '%s'" % prefix) + # ISubTreeMaker def make_subtree_from_node(self, node, parent_is_mutable): assert INode(node) - return self.opener.open(node, parent_is_mutable, - self.make_subtree_from_node) + return self.opener.open(node, parent_is_mutable, self) # these methods are used to walk through our subtrees diff --git a/src/allmydata/test/test_filetree_new.py b/src/allmydata/test/test_filetree_new.py index 92925b11..3f933479 100644 --- a/src/allmydata/test/test_filetree_new.py +++ b/src/allmydata/test/test_filetree_new.py @@ -1,7 +1,7 @@ #from zope.interface import implements from twisted.trial import unittest -#from twisted.internet import defer +from twisted.internet import defer #from allmydata.filetree.interfaces import IOpener, IDirectoryNode #from allmydata.filetree.directory import (ImmutableDirectorySubTree, # SubTreeNode, @@ -314,8 +314,15 @@ class Redirect(unittest.TestCase): import os.path from allmydata.filetree import directory, redirect, vdrive -from allmydata.filetree.interfaces import (ISubTree, INode, IDirectoryNode) +from allmydata.filetree.interfaces import (ISubTree, INode, IDirectoryNode, IFileNode) from allmydata.filetree.file import CHKFileNode +from allmydata.util import bencode + +class InPairs(unittest.TestCase): + def test_in_pairs(self): + l = range(8) + pairs = list(directory.in_pairs(l)) + self.failUnlessEqual(pairs, [(0,1), (2,3), (4,5), (6,7)]) class Stuff(unittest.TestCase): @@ -372,11 +379,14 @@ class Stuff(unittest.TestCase): del root, subdir1, subdir2, subdir3, subdir4 # leaving file1 for later use - # now serialize it and reconstruct it + # now serialize it and examine the results f = StringIO() subtree.serialize_subtree_to_file(f) data = f.getvalue() #print data + unpacked = bencode.bdecode(data) + #print unpacked + del f, data, unpacked node = subtree.create_node_now() self.failUnless(isinstance(node, directory.LocalFileSubTreeNode)) @@ -384,9 +394,14 @@ class Stuff(unittest.TestCase): self.failUnless(isinstance(node_s, str)) self.failUnless(node_s.startswith("LocalFileDirectory:")) self.failUnless("dirtree.save" in node_s) + del node, node_s + + d = defer.maybeDeferred(subtree.update_now, None) + def _updated(node): + # now reconstruct it + return v.make_subtree_from_node(node, False) + d.addCallback(_updated) - # now reconstruct it - d = v.make_subtree_from_node(node, False) def _opened(new_subtree): res = new_subtree.get_node_for_path([]) (found_path, root, remaining_path) = res @@ -396,7 +411,9 @@ class Stuff(unittest.TestCase): self.failUnless(IDirectoryNode.providedBy(root)) self.failUnlessEqual(root.list(), ["foo.txt", "subdir1"]) file1a = root.get("foo.txt") - self.failUnless(isinstance(CHKFileNode, file1a)) + self.failUnless(INode(file1a)) + self.failUnless(isinstance(file1a, CHKFileNode)) + self.failUnless(IFileNode(file1a)) self.failUnlessEqual(file1a.get_uri(), "uri1") subdir1 = root.get("subdir1") subdir2 = subdir1.get("subdir2") @@ -405,7 +422,6 @@ class Stuff(unittest.TestCase): self.failUnlessEqual(subdir2.list(), []) d.addCallback(_opened) return d - testDirectory.todo = "not working yet" def testVdrive(self): # create some stuff, see if we can import everything -- 2.45.2