From 1d9a58977f1271d48cc510d47a8346cd7b08ff1b Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 21 Jul 2007 15:40:36 -0700 Subject: [PATCH] uri: implement URI-processing classes, IFileURI/IDirnodeURI, use internally --- src/allmydata/dirnode.py | 105 ++++++----- src/allmydata/download.py | 42 +++-- src/allmydata/interfaces.py | 36 +++- src/allmydata/scripts/debug.py | 20 ++- src/allmydata/test/test_dirnode.py | 34 ++-- src/allmydata/test/test_encode.py | 17 +- src/allmydata/test/test_system.py | 14 +- src/allmydata/test/test_upload.py | 32 ++-- src/allmydata/test/test_uri.py | 150 +++++++++++----- src/allmydata/test/test_web.py | 52 +++--- src/allmydata/upload.py | 21 +-- src/allmydata/uri.py | 272 ++++++++++++++++++++--------- src/allmydata/vdrive.py | 7 +- src/allmydata/webish.py | 23 +-- 14 files changed, 520 insertions(+), 305 deletions(-) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index cbc35964..e88beccf 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -5,7 +5,8 @@ from twisted.application import service from twisted.internet import defer from foolscap import Referenceable from allmydata import uri -from allmydata.interfaces import RIVirtualDriveServer, IDirectoryNode, IFileNode +from allmydata.interfaces import RIVirtualDriveServer, \ + IDirectoryNode, IFileNode, IFileURI, IDirnodeURI, IURI from allmydata.util import bencode, idlib, hashutil, fileutil from allmydata.Crypto.Cipher import AES @@ -48,7 +49,8 @@ class VirtualDriveServer(service.MultiService, Referenceable): def get_public_root_uri(self): if self._root: - return uri.pack_dirnode_uri(self._myfurl, self._root) + u = uri.DirnodeURI(self._myfurl, self._root) + return u.to_string() raise NoPublicRootError remote_get_public_root_uri = get_public_root_uri @@ -121,16 +123,14 @@ class NotMutableError(Exception): def create_directory_node(client, diruri): - assert uri.is_dirnode_uri(diruri) - if uri.is_mutable_dirnode_uri(diruri): - dirnode_class = MutableDirectoryNode - else: - dirnode_class = ImmutableDirectoryNode - (furl, key) = uri.unpack_dirnode_uri(diruri) - d = client.tub.getReference(furl) + u = IURI(diruri) + assert IDirnodeURI.providedBy(u) + d = client.tub.getReference(u.furl) def _got(rref): - dirnode = dirnode_class(diruri, client, rref, key) - return dirnode + if isinstance(u, uri.DirnodeURI): + return MutableDirectoryNode(u, client, rref) + else: # uri.ReadOnlyDirnodeURI + return ImmutableDirectoryNode(u, client, rref) d.addCallback(_got) return d @@ -163,12 +163,14 @@ def decrypt(key, data): class ImmutableDirectoryNode: implements(IDirectoryNode) - def __init__(self, myuri, client, rref, readkey): - self._uri = myuri + def __init__(self, myuri, client, rref): + u = IDirnodeURI(myuri) + assert u.is_readonly() + self._uri = u.to_string() self._client = client self._tub = client.tub self._rref = rref - self._readkey = readkey + self._readkey = u.readkey self._writekey = None self._write_enabler = None self._index = hashutil.dir_index_hash(self._readkey) @@ -188,10 +190,7 @@ class ImmutableDirectoryNode: def get_immutable_uri(self): # return the dirnode URI for a read-only form of this directory - if self._mutable: - return uri.make_immutable_dirnode_uri(self._uri) - else: - return self._uri + return IDirnodeURI(self._uri).get_readonly().to_string() def __hash__(self): return hash((self.__class__, self._uri)) @@ -256,7 +255,9 @@ class ImmutableDirectoryNode: E_name = self._encrypt(self._readkey, name) E_write = "" if self._writekey and write_child: + assert isinstance(write_child, str) E_write = self._encrypt(self._writekey, write_child) + assert isinstance(read_child, str) E_read = self._encrypt(self._readkey, read_child) d = self._rref.callRemote("set", self._index, self._write_enabler, H_name, E_name, E_write, E_read) @@ -280,30 +281,28 @@ class ImmutableDirectoryNode: return d def _create_node(self, child_uri): - if uri.is_dirnode_uri(child_uri): - return create_directory_node(self._client, child_uri) + u = IURI(child_uri) + if IDirnodeURI.providedBy(u): + return create_directory_node(self._client, u) else: - return defer.succeed(FileNode(child_uri, self._client)) + return defer.succeed(FileNode(u, self._client)) def _split_uri(self, child_uri): - if uri.is_dirnode_uri(child_uri): - if uri.is_mutable_dirnode_uri(child_uri): - write = child_uri - read = uri.make_immutable_dirnode_uri(child_uri) - else: - write = None - read = child_uri - return (write, read) - return (None, child_uri) # file + u = IURI(child_uri) + if u.is_mutable() and not u.is_readonly(): + write = u.to_string() + else: + write = None + read = u.get_readonly().to_string() + return (write, read) def create_empty_directory(self, name): if not self._mutable: return defer.fail(NotMutableError()) child_writekey = hashutil.random_key() - my_furl, parent_writekey = uri.unpack_dirnode_uri(self._uri) - child_uri = uri.pack_dirnode_uri(my_furl, child_writekey) - child = MutableDirectoryNode(child_uri, self._client, self._rref, - child_writekey) + furl = IDirnodeURI(self._uri).furl + u = uri.DirnodeURI(furl, child_writekey) + child = MutableDirectoryNode(u, self._client, self._rref) d = self._rref.callRemote("create_directory", child._index, child._write_enabler) d.addCallback(lambda index: self.set_node(name, child)) @@ -362,8 +361,8 @@ class ImmutableDirectoryNode: return d def get_refresh_capability(self): - ro_uri = self.get_immutable_uri() - furl, rk = uri.unpack_dirnode_uri(ro_uri) + u = IDirnodeURI(self._uri).get_readonly() + rk = u.readkey wk, we, rk, index = hashutil.generate_dirnode_keys_from_readkey(rk) return "DIR-REFRESH:%s" % idlib.b2a(index) @@ -384,21 +383,28 @@ class ImmutableDirectoryNode: class MutableDirectoryNode(ImmutableDirectoryNode): implements(IDirectoryNode) - def __init__(self, myuri, client, rref, writekey): - readkey = hashutil.dir_read_key_hash(writekey) - ImmutableDirectoryNode.__init__(self, myuri, client, rref, readkey) - self._writekey = writekey - self._write_enabler = hashutil.dir_write_enabler_hash(writekey) + def __init__(self, myuri, client, rref): + u = IDirnodeURI(myuri) + assert not u.is_readonly() + self._writekey = u.writekey + self._write_enabler = hashutil.dir_write_enabler_hash(u.writekey) + readkey = hashutil.dir_read_key_hash(u.writekey) + self._uri = u.to_string() + self._client = client + self._tub = client.tub + self._rref = rref + self._readkey = readkey + self._index = hashutil.dir_index_hash(self._readkey) self._mutable = True def create_directory(client, furl): write_key = hashutil.random_key() (wk, we, rk, index) = \ hashutil.generate_dirnode_keys_from_writekey(write_key) - myuri = uri.pack_dirnode_uri(furl, wk) + u = uri.DirnodeURI(furl, wk) d = client.tub.getReference(furl) def _got_vdrive_server(vdrive_server): - node = MutableDirectoryNode(myuri, client, vdrive_server, wk) + node = MutableDirectoryNode(u, client, vdrive_server) d2 = vdrive_server.callRemote("create_directory", index, we) d2.addCallback(lambda res: node) return d2 @@ -409,12 +415,16 @@ class FileNode: implements(IFileNode) def __init__(self, uri, client): - self.uri = uri + u = IFileURI(uri) + self.uri = u.to_string() self._client = client def get_uri(self): return self.uri + def get_size(self): + return IFileURI(self.uri).get_size() + def __hash__(self): return hash((self.__class__, self.uri)) def __cmp__(self, them): @@ -425,10 +435,9 @@ class FileNode: return cmp(self.uri, them.uri) def get_refresh_capability(self): - t = uri.get_uri_type(self.uri) - if t == "CHK": - d = uri.unpack_uri(self.uri) - return "CHK-REFRESH:%s" % idlib.b2a(d['storage_index']) + u = IFileURI(self.uri) + if isinstance(u, uri.CHKFileURI): + return "CHK-REFRESH:%s" % idlib.b2a(u.storage_index) return None def download(self, target): diff --git a/src/allmydata/download.py b/src/allmydata/download.py index f76b13c3..12acd64f 100644 --- a/src/allmydata/download.py +++ b/src/allmydata/download.py @@ -9,7 +9,7 @@ from allmydata.util import idlib, mathutil, hashutil from allmydata.util.assertutil import _assert from allmydata import codec, hashtree, storage, uri from allmydata.Crypto.Cipher import AES -from allmydata.interfaces import IDownloadTarget, IDownloader +from allmydata.interfaces import IDownloadTarget, IDownloader, IFileURI from allmydata.encode import NotEnoughPeersError class HaveAllPeersError(Exception): @@ -288,14 +288,14 @@ class FileDownloader: def __init__(self, client, u, downloadable): self._client = client - d = uri.unpack_uri(u) - self._storage_index = d['storage_index'] - self._uri_extension_hash = d['uri_extension_hash'] - self._total_shares = d['total_shares'] - self._size = d['size'] - self._num_needed_shares = d['needed_shares'] + u = IFileURI(u) + self._storage_index = u.storage_index + self._uri_extension_hash = u.uri_extension_hash + self._total_shares = u.total_shares + self._size = u.size + self._num_needed_shares = u.needed_shares - self._output = Output(downloadable, d['key'], self._size) + self._output = Output(downloadable, u.key, self._size) self.active_buckets = {} # k: shnum, v: bucket self._share_buckets = [] # list of (sharenum, bucket) tuples @@ -609,12 +609,13 @@ class FileDownloader: return self._output.finish() class LiteralDownloader: - def __init__(self, client, uri, downloadable): - self._uri = uri + def __init__(self, client, u, downloadable): + self._uri = IFileURI(u) + assert isinstance(self._uri, uri.LiteralFileURI) self._downloadable = downloadable def start(self): - data = uri.unpack_lit(self._uri) + data = self._uri.data self._downloadable.open(len(data)) self._downloadable.write(data) self._downloadable.close() @@ -625,16 +626,19 @@ class FileName: implements(IDownloadTarget) def __init__(self, filename): self._filename = filename + self.f = None def open(self, size): self.f = open(self._filename, "wb") return self.f def write(self, data): self.f.write(data) def close(self): - self.f.close() + if self.f: + self.f.close() def fail(self, why): - self.f.close() - os.unlink(self._filename) + if self.f: + self.f.close() + os.unlink(self._filename) def register_canceller(self, cb): pass # we won't use it def finish(self): @@ -690,14 +694,16 @@ class Downloader(service.MultiService): def download(self, u, t): assert self.parent assert self.running + u = IFileURI(u) t = IDownloadTarget(t) assert t.write assert t.close - utype = uri.get_uri_type(u) - if utype == "CHK": - dl = FileDownloader(self.parent, u, t) - elif utype == "LIT": + if isinstance(u, uri.LiteralFileURI): dl = LiteralDownloader(self.parent, u, t) + elif isinstance(u, uri.CHKFileURI): + dl = FileDownloader(self.parent, u, t) + else: + raise RuntimeError("I don't know how to download a %s" % u) d = dl.start() return d diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 1d4d2468..84e8de2e 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -228,6 +228,38 @@ class RIVirtualDriveServer(RemoteInterface): """ +class IURI(Interface): + def init_from_string(uri): + """Accept a string (as created by my to_string() method) and populate + this instance with its data. I am not normally called directly, + please use the module-level uri.from_string() function to convert + arbitrary URI strings into IURI-providing instances.""" + + def is_readonly(): + """Return False if this URI be used to modify the data. Return True + if this URI cannot be used to modify the data.""" + + def is_mutable(): + """Return True if the data can be modified by *somebody* (perhaps + someone who has a more powerful URI than this one).""" + + def get_readonly(): + """Return another IURI instance, which represents a read-only form of + this one. If is_readonly() is True, this returns self.""" + + def to_string(): + """Return a string of printable ASCII characters, suitable for + passing into init_from_string.""" + +class IDirnodeURI(Interface): + """I am a URI which represents a dirnode.""" + +class IFileURI(Interface): + """I am a URI which represents a filenode.""" + def get_size(): + """Return the length (in bytes) of the file that I represent.""" + + class IFileNode(Interface): def download(target): """Download the file's contents to a given IDownloadTarget""" @@ -239,6 +271,8 @@ class IFileNode(Interface): """Return the URI that can be used by others to get access to this file. """ + def get_size(): + """Return the length (in bytes) of the data this node represents.""" def get_refresh_capability(): """Return a string that represents the 'refresh capability' for this @@ -788,7 +822,7 @@ class IVirtualDrive(Interface): """ def get_node(uri): - """Transform a URI into an IDirectoryNode or IFileNode. + """Transform a URI (or IURI) into an IDirectoryNode or IFileNode. This returns a Deferred that will fire with an instance that provides either IDirectoryNode or IFileNode, as appropriate.""" diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 773c6748..eae0afc7 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -91,8 +91,8 @@ def dump_root_dirnode(config, out=sys.stdout, err=sys.stderr): try: f = open(root_dirnode_file, "rb") key = f.read() - rooturi = uri.pack_dirnode_uri("fakeFURL", key) - print >>out, rooturi + rooturi = uri.DirnodeURI("fakeFURL", key) + print >>out, rooturi.to_string() return 0 except EnvironmentError: print >>out, "unable to read root dirnode file from %s" % \ @@ -100,22 +100,24 @@ def dump_root_dirnode(config, out=sys.stdout, err=sys.stderr): return 1 def dump_directory_node(config, out=sys.stdout, err=sys.stderr): - from allmydata import uri, dirnode + from allmydata import dirnode from allmydata.util import hashutil, idlib + from allmydata.interfaces import IDirnodeURI basedir = config['basedirs'][0] - dir_uri = config['uri'] + dir_uri = IDirnodeURI(config['uri']) verbose = config['verbose'] - furl, key = uri.unpack_dirnode_uri(dir_uri) - if uri.is_mutable_dirnode_uri(dir_uri): - wk, we, rk, index = hashutil.generate_dirnode_keys_from_writekey(key) + if dir_uri.is_readonly(): + wk, we, rk, index = \ + hashutil.generate_dirnode_keys_from_readkey(dir_uri.readkey) else: - wk, we, rk, index = hashutil.generate_dirnode_keys_from_readkey(key) + wk, we, rk, index = \ + hashutil.generate_dirnode_keys_from_writekey(dir_uri.writekey) filename = os.path.join(basedir, "vdrive", idlib.b2a(index)) print >>out - print >>out, "dirnode uri: %s" % dir_uri + print >>out, "dirnode uri: %s" % dir_uri.to_string() print >>out, "filename : %s" % filename print >>out, "index : %s" % idlib.b2a(index) if wk: diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 1a645e52..d6b69cd2 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -6,7 +6,7 @@ from twisted.internet import defer from twisted.python import failure from allmydata import uri, dirnode from allmydata.util import hashutil -from allmydata.interfaces import IDirectoryNode +from allmydata.interfaces import IDirectoryNode, IDirnodeURI from allmydata.scripts import runner from allmydata.dirnode import VirtualDriveServer, \ ChildAlreadyPresentError, BadWriteEnablerError, NoPublicRootError @@ -20,13 +20,13 @@ class DirectoryNode(unittest.TestCase): vds.set_furl("myFURL") root_uri = vds.get_public_root_uri() - self.failUnless(uri.is_dirnode_uri(root_uri)) - self.failUnless(uri.is_mutable_dirnode_uri(root_uri)) - furl, key = uri.unpack_dirnode_uri(root_uri) - self.failUnlessEqual(furl, "myFURL") - self.failUnlessEqual(len(key), hashutil.KEYLEN) + u = IDirnodeURI(root_uri) + self.failIf(u.is_readonly()) + self.failUnlessEqual(u.furl, "myFURL") + self.failUnlessEqual(len(u.writekey), hashutil.KEYLEN) - wk, we, rk, index = hashutil.generate_dirnode_keys_from_writekey(key) + wk, we, rk, index = \ + hashutil.generate_dirnode_keys_from_writekey(u.writekey) empty_list = vds.list(index) self.failUnlessEqual(empty_list, []) @@ -78,10 +78,10 @@ class DirectoryNode(unittest.TestCase): vds2 = VirtualDriveServer(basedir) vds2.set_furl("myFURL") root_uri2 = vds.get_public_root_uri() - self.failUnless(uri.is_mutable_dirnode_uri(root_uri2)) - furl2, key2 = uri.unpack_dirnode_uri(root_uri2) + u2 = IDirnodeURI(root_uri2) + self.failIf(u2.is_readonly()) (wk2, we2, rk2, index2) = \ - hashutil.generate_dirnode_keys_from_writekey(key2) + hashutil.generate_dirnode_keys_from_writekey(u2.writekey) self.failUnlessEqual(sorted(vds2.list(index2)), [ ("name2", "", "read2"), ]) @@ -173,8 +173,18 @@ class Test(unittest.TestCase): self.failUnlessEqual(res, {}) d.addCallback(_listed) - file1 = uri.pack_uri("11" + " "*30, "k"*16, "e"*32, 25, 100, 12345) - file2 = uri.pack_uri("2i" + " "*30, "k"*16, "e"*32, 25, 100, 12345) + file1 = uri.CHKFileURI(storage_index="11" + " "*30, + key="k"*16, + uri_extension_hash="e"*32, + needed_shares=25, + total_shares=100, + size=12345).to_string() + file2 = uri.CHKFileURI(storage_index="2i" + " "*30, + key="k"*16, + uri_extension_hash="e"*32, + needed_shares=25, + total_shares=100, + size=12345).to_string() file2_node = dirnode.FileNode(file2, None) d.addCallback(lambda res: rootnode.set_uri("foo", file1)) # root/ diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py index ca91614e..2cdec5d6 100644 --- a/src/allmydata/test/test_encode.py +++ b/src/allmydata/test/test_encode.py @@ -3,12 +3,9 @@ from zope.interface import implements from twisted.trial import unittest from twisted.internet import defer from twisted.python.failure import Failure -from allmydata import encode, upload, download, hashtree +from allmydata import encode, upload, download, hashtree, uri from allmydata.util import hashutil -from allmydata.uri import pack_uri -from allmydata.Crypto.Cipher import AES from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader -from cStringIO import StringIO class LostPeerError(Exception): pass @@ -308,12 +305,12 @@ class Roundtrip(unittest.TestCase): if "corrupt_key" in recover_mode: key = flip_bit(key) - URI = pack_uri(storage_index="S" * 32, - key=key, - uri_extension_hash=uri_extension_hash, - needed_shares=e.required_shares, - total_shares=e.num_shares, - size=e.file_size) + URI = uri.CHKFileURI(storage_index="S" * 32, + key=key, + uri_extension_hash=uri_extension_hash, + needed_shares=e.required_shares, + total_shares=e.num_shares, + size=e.file_size).to_string() client = None target = download.Data() fd = download.FileDownloader(client, URI, target) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 1fa99a33..432d503d 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -8,7 +8,7 @@ from allmydata import client, uri, download, upload from allmydata.introducer_and_vdrive import IntroducerAndVdrive from allmydata.util import idlib, fileutil, testutil from allmydata.scripts import runner -from allmydata.interfaces import IDirectoryNode, IFileNode +from allmydata.interfaces import IDirectoryNode, IFileNode, IFileURI from allmydata.dirnode import NotMutableError from foolscap.eventual import flushEventualQueue from twisted.python import log @@ -224,10 +224,14 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase): def mangle_uri(self, gooduri): # change the storage index, which means we'll be asking about the # wrong file, so nobody will have any shares - d = uri.unpack_uri(gooduri) - assert len(d['storage_index']) == 32 - d['storage_index'] = self.flip_bit(d['storage_index']) - return uri.pack_uri(**d) + u = IFileURI(gooduri) + u2 = uri.CHKFileURI(storage_index=self.flip_bit(u.storage_index), + key=u.key, + uri_extension_hash=u.uri_extension_hash, + needed_shares=u.needed_shares, + total_shares=u.total_shares, + size=u.size) + return u2.to_string() # TODO: add a test which mangles the uri_extension_hash instead, and # should fail due to not being able to get a valid uri_extension block. diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 7b8b4bec..3b335d1f 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -5,8 +5,8 @@ from twisted.python.failure import Failure from twisted.internet import defer from cStringIO import StringIO -from allmydata import upload, encode -from allmydata.uri import unpack_uri, unpack_lit +from allmydata import upload, encode, uri +from allmydata.interfaces import IFileURI from allmydata.util.assertutil import precondition from foolscap import eventual @@ -154,21 +154,19 @@ class GoodServer(unittest.TestCase): self.u.running = True self.u.parent = self.node - def _check_small(self, uri, size): - self.failUnless(isinstance(uri, str)) - self.failUnless(uri.startswith("URI:LIT:")) - d = unpack_lit(uri) - self.failUnlessEqual(len(d), size) - - def _check_large(self, uri, size): - self.failUnless(isinstance(uri, str)) - self.failUnless(uri.startswith("URI:")) - d = unpack_uri(uri) - self.failUnless(isinstance(d['storage_index'], str)) - self.failUnlessEqual(len(d['storage_index']), 32) - self.failUnless(isinstance(d['key'], str)) - self.failUnlessEqual(len(d['key']), 16) - self.failUnlessEqual(d['size'], size) + def _check_small(self, newuri, size): + u = IFileURI(newuri) + self.failUnless(isinstance(u, uri.LiteralFileURI)) + self.failUnlessEqual(len(u.data), size) + + def _check_large(self, newuri, size): + u = IFileURI(newuri) + self.failUnless(isinstance(u, uri.CHKFileURI)) + self.failUnless(isinstance(u.storage_index, str)) + self.failUnlessEqual(len(u.storage_index), 32) + self.failUnless(isinstance(u.key, str)) + self.failUnlessEqual(len(u.key), 16) + self.failUnlessEqual(u.size, size) def get_data(self, size): return DATA[:size] diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index 815aa3e0..8aa01938 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -2,23 +2,50 @@ from twisted.trial import unittest from allmydata import uri from allmydata.util import hashutil +from allmydata.interfaces import IURI, IFileURI, IDirnodeURI -class LIT(unittest.TestCase): +class Literal(unittest.TestCase): def test_pack(self): data = "This is some small data" - u = uri.pack_lit(data) - self.failUnlessEqual(uri.get_uri_type(u), "LIT") - self.failUnlessEqual(uri.unpack_lit(u), data) - self.failUnless(uri.is_filenode_uri(u)) - self.failUnlessEqual(uri.get_filenode_size(u), len(data)) + u = uri.LiteralFileURI(data) + self.failUnless(IURI.providedBy(u)) + self.failUnless(IFileURI.providedBy(u)) + self.failIf(IDirnodeURI.providedBy(u)) + self.failUnlessEqual(u.data, data) + self.failUnlessEqual(u.get_size(), len(data)) + self.failUnless(u.is_readonly()) + self.failIf(u.is_mutable()) + + u2 = uri.from_string(u.to_string()) + self.failUnless(IURI.providedBy(u2)) + self.failUnless(IFileURI.providedBy(u2)) + self.failIf(IDirnodeURI.providedBy(u2)) + self.failUnlessEqual(u2.data, data) + self.failUnlessEqual(u2.get_size(), len(data)) + self.failUnless(u.is_readonly()) + self.failIf(u.is_mutable()) def test_nonascii(self): data = "This contains \x00 and URI:LIT: and \n, oh my." - u = uri.pack_lit(data) - self.failUnlessEqual(uri.get_uri_type(u), "LIT") - self.failUnlessEqual(uri.unpack_lit(u), data) + u = uri.LiteralFileURI(data) + self.failUnless(IURI.providedBy(u)) + self.failUnless(IFileURI.providedBy(u)) + self.failIf(IDirnodeURI.providedBy(u)) + self.failUnlessEqual(u.data, data) + self.failUnlessEqual(u.get_size(), len(data)) + self.failUnless(u.is_readonly()) + self.failIf(u.is_mutable()) + + u2 = uri.from_string(u.to_string()) + self.failUnless(IURI.providedBy(u2)) + self.failUnless(IFileURI.providedBy(u2)) + self.failIf(IDirnodeURI.providedBy(u2)) + self.failUnlessEqual(u2.data, data) + self.failUnlessEqual(u2.get_size(), len(data)) + self.failUnless(u.is_readonly()) + self.failIf(u.is_mutable()) -class CHK(unittest.TestCase): +class CHKFile(unittest.TestCase): def test_pack(self): storage_index = hashutil.tagged_hash("foo", "bar") key = "\x00" * 16 @@ -26,23 +53,42 @@ class CHK(unittest.TestCase): needed_shares = 25 total_shares = 100 size = 1234 - u = uri.pack_uri(storage_index=storage_index, - key=key, - uri_extension_hash=uri_extension_hash, - needed_shares=needed_shares, - total_shares=total_shares, - size=size) - self.failUnlessEqual(uri.get_uri_type(u), "CHK") - d = uri.unpack_uri(u) - self.failUnlessEqual(d['storage_index'], storage_index) - self.failUnlessEqual(d['key'], key) - self.failUnlessEqual(d['uri_extension_hash'], uri_extension_hash) - self.failUnlessEqual(d['needed_shares'], needed_shares) - self.failUnlessEqual(d['total_shares'], total_shares) - self.failUnlessEqual(d['size'], size) + u = uri.CHKFileURI(storage_index=storage_index, + key=key, + uri_extension_hash=uri_extension_hash, + needed_shares=needed_shares, + total_shares=total_shares, + size=size) + self.failUnlessEqual(u.storage_index, storage_index) + self.failUnlessEqual(u.key, key) + self.failUnlessEqual(u.uri_extension_hash, uri_extension_hash) + self.failUnlessEqual(u.needed_shares, needed_shares) + self.failUnlessEqual(u.total_shares, total_shares) + self.failUnlessEqual(u.size, size) + self.failUnless(u.is_readonly()) + self.failIf(u.is_mutable()) + self.failUnless(IURI.providedBy(u)) + self.failUnless(IFileURI.providedBy(u)) + self.failIf(IDirnodeURI.providedBy(u)) + self.failUnlessEqual(u.get_size(), 1234) + self.failUnless(u.is_readonly()) + self.failIf(u.is_mutable()) - self.failUnless(uri.is_filenode_uri(u)) - self.failUnlessEqual(uri.get_filenode_size(u), size) + u2 = uri.from_string(u.to_string()) + self.failUnlessEqual(u2.storage_index, storage_index) + self.failUnlessEqual(u2.key, key) + self.failUnlessEqual(u2.uri_extension_hash, uri_extension_hash) + self.failUnlessEqual(u2.needed_shares, needed_shares) + self.failUnlessEqual(u2.total_shares, total_shares) + self.failUnlessEqual(u2.size, size) + self.failUnless(u2.is_readonly()) + self.failIf(u2.is_mutable()) + self.failUnless(IURI.providedBy(u2)) + self.failUnless(IFileURI.providedBy(u2)) + self.failIf(IDirnodeURI.providedBy(u2)) + self.failUnlessEqual(u2.get_size(), 1234) + self.failUnless(u2.is_readonly()) + self.failIf(u2.is_mutable()) class Extension(unittest.TestCase): def test_pack(self): @@ -64,26 +110,40 @@ class Dirnode(unittest.TestCase): furl = "pb://stuff@morestuff:stuff/andstuff" writekey = "\x01" * 16 - u = uri.pack_dirnode_uri(furl, writekey) - self.failUnless(uri.is_dirnode_uri(u)) - self.failIf(uri.is_dirnode_uri("NOT A DIRNODE URI")) - self.failIf(uri.is_dirnode_uri("URI:stuff")) - self.failUnless(uri.is_mutable_dirnode_uri(u)) - self.failIf(uri.is_mutable_dirnode_uri("NOT A DIRNODE URI")) - self.failIf(uri.is_mutable_dirnode_uri("URI:stuff")) - self.failUnlessEqual(uri.get_uri_type(u), "DIR") + u = uri.DirnodeURI(furl, writekey) + self.failUnlessEqual(u.furl, furl) + self.failUnlessEqual(u.writekey, writekey) + self.failIf(u.is_readonly()) + self.failUnless(u.is_mutable()) + self.failUnless(IURI.providedBy(u)) + self.failIf(IFileURI.providedBy(u)) + self.failUnless(IDirnodeURI.providedBy(u)) - rou = uri.make_immutable_dirnode_uri(u) - self.failUnless(uri.is_dirnode_uri(rou)) - self.failIf(uri.is_mutable_dirnode_uri(rou)) - self.failUnlessEqual(uri.get_uri_type(rou), "DIR-RO") + u2 = uri.from_string(u.to_string()) + self.failUnlessEqual(u2.furl, furl) + self.failUnlessEqual(u2.writekey, writekey) + self.failIf(u2.is_readonly()) + self.failUnless(u2.is_mutable()) + self.failUnless(IURI.providedBy(u2)) + self.failIf(IFileURI.providedBy(u2)) + self.failUnless(IDirnodeURI.providedBy(u2)) - d = uri.unpack_dirnode_uri(u) - self.failUnlessEqual(d[0], furl) - self.failUnlessEqual(d[1], writekey) + u3 = u2.get_readonly() + readkey = hashutil.dir_read_key_hash(writekey) + self.failUnlessEqual(u3.furl, furl) + self.failUnlessEqual(u3.readkey, readkey) + self.failUnless(u3.is_readonly()) + self.failUnless(u3.is_mutable()) + self.failUnless(IURI.providedBy(u3)) + self.failIf(IFileURI.providedBy(u3)) + self.failUnless(IDirnodeURI.providedBy(u3)) - d2 = uri.unpack_dirnode_uri(rou) - self.failUnlessEqual(d2[0], furl) - rk = hashutil.dir_read_key_hash(writekey) - self.failUnlessEqual(d2[1], rk) + u4 = uri.ReadOnlyDirnodeURI(furl, readkey) + self.failUnlessEqual(u4.furl, furl) + self.failUnlessEqual(u4.readkey, readkey) + self.failUnless(u4.is_readonly()) + self.failUnless(u4.is_mutable()) + self.failUnless(IURI.providedBy(u4)) + self.failIf(IFileURI.providedBy(u4)) + self.failUnless(IDirnodeURI.providedBy(u4)) diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index 0b5046ff..dbc9a845 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -48,6 +48,17 @@ class MyDownloader(service.Service): uri_counter = itertools.count() +def make_newuri(data): + n = uri_counter.next() + assert len(str(n)) < 5 + newuri = uri.CHKFileURI(storage_index="SI%05d" % n + "i"*25, + key="K"*16, + uri_extension_hash="EH" + "h"*30, + needed_shares=25, + total_shares=100, + size=len(data)) + return newuri.to_string() + class MyUploader(service.Service): implements(interfaces.IUploader) name = "uploader" @@ -59,31 +70,26 @@ class MyUploader(service.Service): d.addCallback(lambda size: uploadable.read(size)) d.addCallback(lambda data: "".join(data)) def _got_data(data): - uri = str(uri_counter.next()) - self.files[uri] = data + newuri = make_newuri(data) + self.files[newuri] = data uploadable.close() d.addCallback(_got_data) return d class MyDirectoryNode(dirnode.MutableDirectoryNode): - def __init__(self, nodes, files, client, uri=None): + def __init__(self, nodes, files, client, myuri=None): self._my_nodes = nodes self._my_files = files self._my_client = client - if uri is None: - uri = "URI:DIR:stuff/%s" % str(uri_counter.next()) - self._uri = str(uri) + if myuri is None: + u = uri.DirnodeURI("furl", "idx%s" % str(uri_counter.next())) + myuri = u.to_string() + self._uri = myuri self._my_nodes[self._uri] = self self.children = {} self._mutable = True - def get_immutable_uri(self): - return self.get_uri() + "RO" - - def get_refresh_capability(self): - return "refresh:" + self.get_uri() - def get(self, name): def _try(): uri = self.children[name] @@ -101,12 +107,12 @@ class MyDirectoryNode(dirnode.MutableDirectoryNode): d.addCallback(lambda size: uploadable.read(size)) d.addCallback(lambda data: "".join(data)) def _got_data(data): - uri = str(uri_counter.next()) - self._my_files[uri] = data - self._my_nodes[uri] = MyFileNode(uri, self._my_client) - self.children[name] = uri + newuri = make_newuri(data) + self._my_files[newuri] = data + self._my_nodes[newuri] = MyFileNode(newuri, self._my_client) + self.children[name] = newuri uploadable.close() - return self._my_nodes[uri] + return self._my_nodes[newuri] d.addCallback(_got_data) return d @@ -214,10 +220,12 @@ class Web(unittest.TestCase): def makefile(self, number): n = str(number) assert len(n) == 1 - newuri = uri.pack_uri("SI" + n*30, - "K" + n*15, - "EH" + n*30, - 25, 100, 123+number) + newuri = uri.CHKFileURI(storage_index="SI" + n*30, + key="K" + n*15, + uri_extension_hash="EH" + n*30, + needed_shares=25, + total_shares=100, + size=123+number).to_string() assert newuri not in self.nodes assert newuri not in self.files node = MyFileNode(newuri, self.s) @@ -230,7 +238,7 @@ class Web(unittest.TestCase): n = str(number) assert len(n) == 1 contents = "small data %s\n" % n - newuri = uri.pack_lit(contents) + newuri = uri.LiteralFileURI(contents).to_string() assert newuri not in self.nodes assert newuri not in self.files node = MyFileNode(newuri, self.s) diff --git a/src/allmydata/upload.py b/src/allmydata/upload.py index 8905da12..59b8db32 100644 --- a/src/allmydata/upload.py +++ b/src/allmydata/upload.py @@ -7,8 +7,7 @@ from twisted.application import service from foolscap import Referenceable from allmydata.util import idlib, hashutil -from allmydata import encode, storage, hashtree -from allmydata.uri import pack_uri, pack_lit +from allmydata import encode, storage, hashtree, uri from allmydata.interfaces import IUploadable, IUploader from cStringIO import StringIO @@ -321,13 +320,14 @@ class CHKUploader: self._encoder.set_shareholders(buckets) def _compute_uri(self, uri_extension_hash): - return pack_uri(storage_index=self._storage_index, - key=self._encryption_key, - uri_extension_hash=uri_extension_hash, - needed_shares=self.needed_shares, - total_shares=self.total_shares, - size=self._size, - ) + u = uri.CHKFileURI(storage_index=self._storage_index, + key=self._encryption_key, + uri_extension_hash=uri_extension_hash, + needed_shares=self.needed_shares, + total_shares=self.total_shares, + size=self._size, + ) + return u.to_string() def read_this_many_bytes(uploadable, size, prepend_data=[]): if size == 0: @@ -359,7 +359,8 @@ class LiteralUploader: def start(self): d = self._uploadable.get_size() d.addCallback(lambda size: read_this_many_bytes(self._uploadable, size)) - d.addCallback(lambda data: pack_lit("".join(data))) + d.addCallback(lambda data: uri.LiteralFileURI("".join(data))) + d.addCallback(lambda u: u.to_string()) return d def close(self): diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index c9d3bf47..47446198 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -1,66 +1,202 @@ import re +from zope.interface import implements +from twisted.python.components import registerAdapter from allmydata.util import idlib, hashutil - -def get_uri_type(uri): - assert uri.startswith("URI:") - if uri.startswith("URI:DIR:"): - return "DIR" - if uri.startswith("URI:DIR-RO:"): - return "DIR-RO" - if uri.startswith("URI:LIT:"): - return "LIT" - return "CHK" - -def is_filenode_uri(uri): - return get_uri_type(uri) in ("LIT", "CHK") - -def get_filenode_size(uri): - assert is_filenode_uri(uri) - t = get_uri_type(uri) - if t == "LIT": - return len(unpack_lit(uri)) - return unpack_uri(uri)['size'] - +from allmydata.interfaces import IURI, IDirnodeURI, IFileURI # the URI shall be an ascii representation of the file. It shall contain # enough information to retrieve and validate the contents. It shall be # expressed in a limited character set (namely [TODO]). -def pack_uri(storage_index, key, uri_extension_hash, - needed_shares, total_shares, size): - # applications should pass keyword parameters into this - assert isinstance(storage_index, str) - assert len(storage_index) == 32 # sha256 hash - assert isinstance(uri_extension_hash, str) - assert len(uri_extension_hash) == 32 # sha56 hash +class _BaseURI: + def __hash__(self): + return hash((self.__class__, self.to_string())) + def __cmp__(self, them): + if cmp(type(self), type(them)): + return cmp(type(self), type(them)) + if cmp(self.__class__, them.__class__): + return cmp(self.__class__, them.__class__) + return cmp(self.to_string(), them.to_string()) + +class CHKFileURI(_BaseURI): + implements(IURI, IFileURI) + + def __init__(self, **kwargs): + # construct me with kwargs, since there are so many of them + if not kwargs: + return + for name in ("storage_index", "key", "uri_extension_hash", + "needed_shares", "total_shares", "size"): + value = kwargs[name] + setattr(self, name, value) + + def init_from_string(self, uri): + assert uri.startswith("URI:CHK:"), uri + d = {} + (header_uri, header_chk, + storage_index_s, key_s, uri_extension_hash_s, + needed_shares_s, total_shares_s, size_s) = uri.split(":") + assert header_uri == "URI" + assert header_chk == "CHK" + self.storage_index = idlib.a2b(storage_index_s) + self.key = idlib.a2b(key_s) + self.uri_extension_hash = idlib.a2b(uri_extension_hash_s) + self.needed_shares = int(needed_shares_s) + self.total_shares = int(total_shares_s) + self.size = int(size_s) + return self + + def to_string(self): + assert isinstance(self.storage_index, str) + assert len(self.storage_index) == 32 # sha256 hash + + assert isinstance(self.uri_extension_hash, str) + assert len(self.uri_extension_hash) == 32 # sha56 hash + + assert isinstance(self.key, str) + assert len(self.key) == 16 # AES-128 + assert isinstance(self.needed_shares, int) + assert isinstance(self.total_shares, int) + assert isinstance(self.size, (int,long)) + + return ("URI:CHK:%s:%s:%s:%d:%d:%d" % + (idlib.b2a(self.storage_index), + idlib.b2a(self.key), + idlib.b2a(self.uri_extension_hash), + self.needed_shares, + self.total_shares, + self.size)) + + def is_readonly(self): + return True + def is_mutable(self): + return False + def get_readonly(self): + return self + + def get_size(self): + return self.size + +class LiteralFileURI(_BaseURI): + implements(IURI, IFileURI) + + def __init__(self, data=None): + if data is not None: + self.data = data + + def init_from_string(self, uri): + assert uri.startswith("URI:LIT:") + data_s = uri[len("URI:LIT:"):] + self.data = idlib.a2b(data_s) + return self + + def to_string(self): + return "URI:LIT:%s" % idlib.b2a(self.data) + + def is_readonly(self): + return True + def is_mutable(self): + return False + def get_readonly(self): + return self + + def get_size(self): + return len(self.data) + +class DirnodeURI(_BaseURI): + implements(IURI, IDirnodeURI) + + def __init__(self, furl=None, writekey=None): + if furl is not None or writekey is not None: + assert furl is not None + assert writekey is not None + self.furl = furl + self.writekey = writekey + + def init_from_string(self, uri): + # URI:DIR:furl:key + # but note that the furl contains colons + prefix = "URI:DIR:" + assert uri.startswith(prefix) + uri = uri[len(prefix):] + colon = uri.rindex(":") + self.furl = uri[:colon] + self.writekey = idlib.a2b(uri[colon+1:]) + return self + + def to_string(self): + return "URI:DIR:%s:%s" % (self.furl, idlib.b2a(self.writekey)) + + def is_readonly(self): + return False + def is_mutable(self): + return True + def get_readonly(self): + u = ReadOnlyDirnodeURI() + u.furl = self.furl + u.readkey = hashutil.dir_read_key_hash(self.writekey) + return u + +class ReadOnlyDirnodeURI(_BaseURI): + implements(IURI, IDirnodeURI) + + def __init__(self, furl=None, readkey=None): + if furl is not None or readkey is not None: + assert furl is not None + assert readkey is not None + self.furl = furl + self.readkey = readkey + + def init_from_string(self, uri): + # URI:DIR-RO:furl:key + # but note that the furl contains colons + prefix = "URI:DIR-RO:" + assert uri.startswith(prefix) + uri = uri[len(prefix):] + colon = uri.rindex(":") + self.furl = uri[:colon] + self.readkey = idlib.a2b(uri[colon+1:]) + return self + + def to_string(self): + return "URI:DIR-RO:%s:%s" % (self.furl, idlib.b2a(self.readkey)) + + def is_readonly(self): + return True + def is_mutable(self): + return True + def get_readonly(self): + return self + +def from_string(s): + if s.startswith("URI:CHK:"): + return CHKFileURI().init_from_string(s) + elif s.startswith("URI:LIT:"): + return LiteralFileURI().init_from_string(s) + elif s.startswith("URI:DIR:"): + return DirnodeURI().init_from_string(s) + elif s.startswith("URI:DIR-RO:"): + return ReadOnlyDirnodeURI().init_from_string(s) + else: + raise RuntimeError("unknown URI type: %s.." % s[:10]) - assert isinstance(key, str) - assert len(key) == 16 # AES-128 - assert isinstance(needed_shares, int) - assert isinstance(total_shares, int) - assert isinstance(size, (int,long)) +registerAdapter(from_string, str, IURI) - return "URI:%s:%s:%s:%d:%d:%d" % (idlib.b2a(storage_index), idlib.b2a(key), - idlib.b2a(uri_extension_hash), - needed_shares, total_shares, size) +def from_string_dirnode(s): + u = from_string(s) + assert IDirnodeURI.providedBy(u) + return u +registerAdapter(from_string_dirnode, str, IDirnodeURI) -def unpack_uri(uri): - assert uri.startswith("URI:"), uri - d = {} - (header, - storage_index_s, key_s, uri_extension_hash_s, - needed_shares_s, total_shares_s, size_s) = uri.split(":") - assert header == "URI" - d['storage_index'] = idlib.a2b(storage_index_s) - d['key'] = idlib.a2b(key_s) - d['uri_extension_hash'] = idlib.a2b(uri_extension_hash_s) - d['needed_shares'] = int(needed_shares_s) - d['total_shares'] = int(total_shares_s) - d['size'] = int(size_s) - return d +def from_string_filenode(s): + u = from_string(s) + assert IFileURI.providedBy(u) + return u + +registerAdapter(from_string_filenode, str, IFileURI) def pack_extension(data): @@ -108,39 +244,3 @@ def unpack_extension_readable(data): unpacked[k] = idlib.b2a(unpacked[k]) return unpacked -def pack_lit(data): - return "URI:LIT:%s" % idlib.b2a(data) - -def unpack_lit(uri): - assert uri.startswith("URI:LIT:") - data_s = uri[len("URI:LIT:"):] - return idlib.a2b(data_s) - - -def is_dirnode_uri(uri): - return uri.startswith("URI:DIR:") or uri.startswith("URI:DIR-RO:") -def is_mutable_dirnode_uri(uri): - return uri.startswith("URI:DIR:") -def unpack_dirnode_uri(uri): - assert is_dirnode_uri(uri) - # URI:DIR:furl:key - # but note that the furl contains colons - for prefix in ("URI:DIR:", "URI:DIR-RO:"): - if uri.startswith(prefix): - uri = uri[len(prefix):] - break - else: - assert 0 - colon = uri.rindex(":") - furl = uri[:colon] - key = uri[colon+1:] - return furl, idlib.a2b(key) - -def make_immutable_dirnode_uri(mutable_uri): - assert is_mutable_dirnode_uri(mutable_uri) - furl, writekey = unpack_dirnode_uri(mutable_uri) - readkey = hashutil.dir_read_key_hash(writekey) - return "URI:DIR-RO:%s:%s" % (furl, idlib.b2a(readkey)) - -def pack_dirnode_uri(furl, writekey): - return "URI:DIR:%s:%s" % (furl, idlib.b2a(writekey)) diff --git a/src/allmydata/vdrive.py b/src/allmydata/vdrive.py index b103a204..918660bf 100644 --- a/src/allmydata/vdrive.py +++ b/src/allmydata/vdrive.py @@ -2,8 +2,8 @@ import os from twisted.application import service from zope.interface import implements -from allmydata.interfaces import IVirtualDrive -from allmydata import dirnode, uri +from allmydata.interfaces import IVirtualDrive, IDirnodeURI, IURI +from allmydata import dirnode from twisted.internet import defer class NoGlobalVirtualDriveError(Exception): @@ -88,7 +88,8 @@ class VirtualDrive(service.MultiService): return self.get_node(self._private_uri) def get_node(self, node_uri): - if uri.is_dirnode_uri(node_uri): + node_uri = IURI(node_uri) + if IDirnodeURI.providedBy(node_uri): return dirnode.create_directory_node(self.parent, node_uri) else: return defer.succeed(dirnode.FileNode(node_uri, self.parent)) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index f8712cd4..731d5937 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -8,9 +8,8 @@ from nevow import inevow, rend, loaders, appserver, url, tags as T from nevow.static import File as nevow_File # TODO: merge with static.File? from allmydata.util import idlib, fileutil import simplejson -from allmydata.uri import unpack_uri, is_dirnode_uri from allmydata.interfaces import IDownloadTarget, IDirectoryNode, IFileNode -from allmydata import upload, download, uri +from allmydata import upload, download from zope.interface import implements, Interface import urllib from formless import webform @@ -103,19 +102,7 @@ class Directory(rend.Page): T.a(href=dlurl)[html.escape(name)]) ctx.fillSlots("type", "FILE") - - #uri = target.uri - #dl_uri_url = url.root.child("download_uri").child(uri) - ## add a filename= query argument to give it a Content-Type - #dl_uri_url = dl_uri_url.add("filename", name) - #ctx.fillSlots("uri", T.a(href=dl_uri_url)[html.escape(uri)]) - - #extract and display file size - try: - size = uri.get_filenode_size(target.get_uri()) - except AssertionError: - size = "?" - ctx.fillSlots("size", size) + ctx.fillSlots("size", target.get_size()) text_plain_link = "/uri/%s?filename=foo.txt" % uri_link text_plain_tag = T.a(href=text_plain_link)["text/plain"] @@ -322,7 +309,7 @@ class FileJSONMetadata(rend.Page): data = ("filenode", {'mutable': False, 'uri': file_uri, - 'size': uri.get_filenode_size(file_uri), + 'size': filenode.get_size(), }) return simplejson.dumps(data, indent=1) @@ -412,7 +399,7 @@ class DirectoryJSONMetadata(rend.Page): kiddata = ("filenode", {'mutable': False, 'uri': kiduri, - 'size': uri.get_filenode_size(kiduri), + 'size': childnode.get_size(), }) else: assert IDirectoryNode.providedBy(childnode) @@ -519,8 +506,6 @@ class POSTHandler(rend.Page): newuri = req.args["uri"][0] else: newuri = req.fields["uri"].value - # sanity checking - assert(is_dirnode_uri(newuri) or unpack_uri(newuri)) d = self._node.set_uri(name, newuri) def _done(res): return newuri -- 2.45.2