From: Brian Warner Date: Wed, 5 Nov 2008 01:00:22 +0000 (-0700) Subject: #531: implement an SFTP frontend. Mostly works, still lots of debug messages. Still... X-Git-Url: https://git.rkrishnan.org/architecture.txt?a=commitdiff_plain;h=9f908de9e2ba9e147272d1547236782c28a58e1e;p=tahoe-lafs%2Ftahoe-lafs.git #531: implement an SFTP frontend. Mostly works, still lots of debug messages. Still needs tests and auth-by-pubkey in accounts.file --- diff --git a/docs/sftp.txt b/docs/sftp.txt new file mode 100644 index 00000000..cb085823 --- /dev/null +++ b/docs/sftp.txt @@ -0,0 +1,74 @@ += Tahoe SFTP Frontend = + +All Tahoe client nodes can run a frontend SFTP server, allowing regular SFTP +clients to access the virtual filesystem. + +Since Tahoe does not use user accounts or passwords, the FTP server must be +configured with a way to translate a username (and either a password or +public key) into a root directory cap. Two mechanisms are provided. The first +is a simple flat file with one account per line. The second is an HTTP-based +login mechanism, backed by simple PHP script and a database. The latter form +is used by allmydata.com to provide secure access to customer rootcaps. + +The SFTP server must also be given a public/private host keypair. + +== Configuring a Keypair == + +First, generate a keypair for your server: + +% cd BASEDIR +% ssh-keygen -f private/ssh_host_rsa_key + +You will then use the following lines in the tahoe.cfg file: + + [sftpd] + sftp.host_pubkey_file = private/ssh_host_rsa_key.pub + sftp.host_privkey_file = private/ssh_host_rsa_key + +== Configuring an Account File == + +To configure the first form, create a file (probably in +BASEDIR/private/sftp.accounts) in which each non-comment/non-blank line is a +space-separated line of (USERNAME, PASSWORD/PUBKEY, ROOTCAP), like so: + +[TODO: the PUBKEY form is not yet supported] + + % cat BASEDIR/private/sftp.accounts + # This is a password file, (username, password/pubkey, rootcap) + alice password URI:DIR2:ioej8xmzrwilg772gzj4fhdg7a:wtiizszzz2rgmczv4wl6bqvbv33ag4kvbr6prz3u6w3geixa6m6a + bob sekrit URI:DIR2:6bdmeitystckbl9yqlw7g56f4e:serp5ioqxnh34mlbmzwvkp3odehsyrr7eytt5f64we3k9hhcrcja + carol ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv2xHRVBoXnwxHLzthRD1wOWtyZ08b8n9cMZfJ58CBdBwAYP2NVNXc0XjRvswm5hnnAO+jyWPVNpXJjm9XllzYhODSNtSN+TXuJlUjhzA/T+ZwdgsgSAeHuuMQBoWt4Qc9HV6rHCdAeMhcnyqm6Q0sRAsfA/wfwiIgbvE7+cWpFa2anB6WeAnvK8+dMN0nvnkPE7GNyf/WFR1Ffuh9ifKdRB6yDNp17bQAqA3OWSFjch6fGPhp94y4g2jmTHlEUTyVsilgGqvGOutOVYnmOMnFijugU1Vu33G39GGzXWla6+fXwTk/oiVPiCYD7A7WFKes3nqMg8iVN6a6sxujrhnHQ== warner@fluxx URI:DIR2:6bdmeitystckbl9yqlw7g56f4e:serp5ioqxnh34mlbmzwvkp3odehsyrr7eytt5f64we3k9hhcrcja + +Note that if the second word of the line is "ssh-rsa" or "ssh-dss", the rest +of the line is parsed differently, so users cannot have a password equal to +either of these strings. + +Then add the following lines to the BASEDIR/tahoe.cfg file: + + [sftpd] + enabled = true + sftp.port = 8022 + sftp.host_pubkey_file = private/ssh_host_rsa_key.pub + sftp.host_privkey_file = private/ssh_host_rsa_key + sftp.accounts.file = private/sftp.accounts + +The SFTP server will listen on the given port number. The sftp.accounts.file +pathname will be interpreted relative to the node's BASEDIR. + +== Configuring an Account Server == + +Determine the URL of the account server, say https://example.com/login . Then +add the following lines to BASEDIR/tahoe.cfg: + + [sftpd] + enabled = true + sftp.port = 8022 + sftp.host_pubkey_file = private/ssh_host_rsa_key.pub + sftp.host_privkey_file = private/ssh_host_rsa_key + sftp.accounts.url = https://example.com/login + +== Dependencies == + +The Tahoe SFTP server requires the Twisted "Conch" component, which itself +requires the pycrypto package (note that pycrypto is distinct from the +pycryptopp that Tahoe uses). diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 886a046b..418ba761 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -79,6 +79,7 @@ class Client(node.Node, pollmixin.PollMixin): self.init_key_gen(key_gen_furl) # ControlServer and Helper are attached after Tub startup self.init_ftp_server() + self.init_sftp_server() hotline_file = os.path.join(self.basedir, self.SUICIDE_PREVENTION_HOTLINE_FILE) @@ -270,6 +271,19 @@ class Client(node.Node, pollmixin.PollMixin): s = ftpd.FTPServer(self, accountfile, accounturl, ftp_portstr) s.setServiceParent(self) + def init_sftp_server(self): + if self.get_config("sftpd", "enabled", False, boolean=True): + accountfile = self.get_config("sftpd", "sftp.accounts.file", None) + accounturl = self.get_config("sftpd", "sftp.accounts.url", None) + sftp_portstr = self.get_config("sftpd", "sftp.port", "8022") + pubkey_file = self.get_config("sftpd", "sftp.host_pubkey_file") + privkey_file = self.get_config("sftpd", "sftp.host_privkey_file") + + from allmydata import sftpd + s = sftpd.SFTPServer(self, accountfile, accounturl, + sftp_portstr, pubkey_file, privkey_file) + s.setServiceParent(self) + def _check_hotline(self, hotline_file): if os.path.exists(hotline_file): mtime = os.stat(hotline_file)[stat.ST_MTIME] diff --git a/src/allmydata/sftpd.py b/src/allmydata/sftpd.py new file mode 100644 index 00000000..1b0a91f5 --- /dev/null +++ b/src/allmydata/sftpd.py @@ -0,0 +1,563 @@ + +import os +import tempfile +from zope.interface import implements +from twisted.python import components +from twisted.application import service, strports +from twisted.internet import defer +from twisted.internet.interfaces import IConsumer +from twisted.conch.ssh import factory, keys, session +from twisted.conch.interfaces import ISFTPServer, ISFTPFile, IConchUser +from twisted.conch.avatar import ConchUser +from twisted.conch.openssh_compat import primes +from twisted.conch import ls +from twisted.cred import error, portal, checkers, credentials +from twisted.web.client import getPage + +from allmydata.interfaces import IDirectoryNode, ExistingChildError, \ + NoSuchChildError +from allmydata.immutable.upload import FileHandle +from allmydata.util import base32 + +class MemoryConsumer: + implements(IConsumer) + def __init__(self): + self.chunks = [] + self.done = False + def registerProducer(self, p, streaming): + if streaming: + # call resumeProducing once to start things off + p.resumeProducing() + else: + while not self.done: + p.resumeProducing() + def write(self, data): + self.chunks.append(data) + def unregisterProducer(self): + self.done = True + +def download_to_data(n, offset=0, size=None): + d = n.read(MemoryConsumer(), offset, size) + d.addCallback(lambda mc: "".join(mc.chunks)) + return d + +class ReadFile: + implements(ISFTPFile) + def __init__(self, node): + self.node = node + def readChunk(self, offset, length): + d = download_to_data(self.node, offset, length) + def _got(data): + return data + d.addCallback(_got) + return d + def close(self): + pass + def getAttrs(self): + print "GETATTRS(file)" + raise NotImplementedError + def setAttrs(self, attrs): + print "SETATTRS(file)", attrs + raise NotImplementedError + +class WriteFile: + implements(ISFTPFile) + + def __init__(self, parent, childname, convergence): + self.parent = parent + self.childname = childname + self.convergence = convergence + self.f = tempfile.TemporaryFile() + def writeChunk(self, offset, data): + self.f.seek(offset) + self.f.write(data) + + def close(self): + u = FileHandle(self.f, self.convergence) + d = self.parent.add_file(self.childname, u) + return d + + def getAttrs(self): + print "GETATTRS(file)" + raise NotImplementedError + def setAttrs(self, attrs): + print "SETATTRS(file)", attrs + raise NotImplementedError + + +class NoParentError(Exception): + pass + +class PermissionError(Exception): + pass + +from twisted.conch.ssh.filetransfer import FileTransferServer, SFTPError, \ + FX_NO_SUCH_FILE, FX_FILE_ALREADY_EXISTS, FX_OP_UNSUPPORTED, \ + FX_PERMISSION_DENIED +from twisted.conch.ssh.filetransfer import FXF_READ, FXF_WRITE, FXF_APPEND, FXF_CREAT, FXF_TRUNC, FXF_EXCL + +class SFTPUser(ConchUser): + def __init__(self, client, rootnode, username, convergence): + ConchUser.__init__(self) + self.channelLookup["session"] = session.SSHSession + self.subsystemLookup["sftp"] = FileTransferServer + + self.client = client + self.root = rootnode + self.username = username + self.convergence = convergence + +class StoppableList: + def __init__(self, items): + self.items = items + def __iter__(self): + for i in self.items: + yield i + def close(self): + pass + +class FakeStat: + pass + +class SFTPHandler: + implements(ISFTPServer) + def __init__(self, user): + print "Creating SFTPHandler from", user + self.client = user.client + self.root = user.root + self.username = user.username + self.convergence = user.convergence + + def gotVersion(self, otherVersion, extData): + return {} + + def openFile(self, filename, flags, attrs): + f = "|".join([(flags & FXF_READ) and "FXF_READ" or "", + (flags & FXF_WRITE) and "FXF_WRITE" or "", + (flags & FXF_APPEND) and "FXF_APPEND" or "", + (flags & FXF_CREAT) and "FXF_CREAT" or "", + (flags & FXF_TRUNC) and "FXF_TRUNC" or "", + (flags & FXF_EXCL) and "FXF_EXCL" or "", + ]) + print "OPENFILE", filename, flags, f, attrs + # this is used for both reading and writing. + +# createPlease = False +# exclusive = False +# openFlags = 0 +# +# if flags & FXF_READ == FXF_READ and flags & FXF_WRITE == 0: +# openFlags = os.O_RDONLY +# if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == 0: +# createPlease = True +# openFlags = os.O_WRONLY +# if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == FXF_READ: +# createPlease = True +# openFlags = os.O_RDWR +# if flags & FXF_APPEND == FXF_APPEND: +# createPlease = True +# openFlags |= os.O_APPEND +# if flags & FXF_CREAT == FXF_CREAT: +# createPlease = True +# openFlags |= os.O_CREAT +# if flags & FXF_TRUNC == FXF_TRUNC: +# openFlags |= os.O_TRUNC +# if flags & FXF_EXCL == FXF_EXCL: +# exclusive = True + + # /usr/bin/sftp 'get' gives us FXF_READ, while 'put' on a new file + # gives FXF_WRITE,FXF_CREAT,FXF_TRUNC . I'm guessing that 'put' on an + # existing file gives the same. + + path = self._convert_sftp_path(filename) + + if flags & FXF_READ: + if flags & FXF_WRITE: + raise NotImplementedError + d = self._get_node_and_metadata_for_path(path) + d.addCallback(lambda (node,metadata): ReadFile(node)) + d.addErrback(self._convert_error) + return d + + if flags & FXF_WRITE: + if not (flags & FXF_CREAT) or not (flags & FXF_TRUNC): + raise NotImplementedError + if not path: + raise PermissionError("cannot STOR to root directory") + childname = path[-1] + d = self._get_root(path) + def _got_root((root, path)): + if not path: + raise PermissionError("cannot STOR to root directory") + return root.get_child_at_path(path[:-1]) + d.addCallback(_got_root) + def _got_parent(parent): + return WriteFile(parent, childname, self.convergence) + d.addCallback(_got_parent) + return d + raise NotImplementedError + + def removeFile(self, path): + print "REMOVEFILE", path + path = self._convert_sftp_path(path) + return self._remove_thing(path, must_be_file=True) + + def renameFile(self, oldpath, newpath): + print "RENAMEFILE", oldpath, newpath + fromPath = self._convert_sftp_path(oldpath) + toPath = self._convert_sftp_path(newpath) + # the target directory must already exist + d = self._get_parent(fromPath) + def _got_from_parent( (fromparent, childname) ): + d = self._get_parent(toPath) + d.addCallback(lambda (toparent, tochildname): + fromparent.move_child_to(childname, + toparent, tochildname, + overwrite=False)) + return d + d.addCallback(_got_from_parent) + d.addErrback(self._convert_error) + return d + + def makeDirectory(self, path, attrs): + print "MAKEDIRECTORY", path, attrs + # TODO: extract attrs["mtime"], use it to set the parent metadata. + # Maybe also copy attrs["ext_*"] . + path = self._convert_sftp_path(path) + d = self._get_root(path) + d.addCallback(lambda (root,path): + self._get_or_create_directories(root, path)) + return d + + def _get_or_create_directories(self, node, path): + if not IDirectoryNode.providedBy(node): + # unfortunately it is too late to provide the name of the + # blocking directory in the error message. + raise ExistingChildError("cannot create directory because there " + "is a file in the way") # close enough + if not path: + return defer.succeed(node) + d = node.get(path[0]) + def _maybe_create(f): + f.trap(NoSuchChildError) + return node.create_empty_directory(path[0]) + d.addErrback(_maybe_create) + d.addCallback(self._get_or_create_directories, path[1:]) + return d + + def removeDirectory(self, path): + print "REMOVEDIRECTORY", path + path = self._convert_sftp_path(path) + return self._remove_thing(path, must_be_directory=True) + + def _remove_thing(self, path, must_be_directory=False, must_be_file=False): + d = defer.maybeDeferred(self._get_parent, path) + def _convert_error(f): + f.trap(NoParentError) + raise PermissionError("cannot delete root directory") + d.addErrback(_convert_error) + def _got_parent( (parent, childname) ): + d = parent.get(childname) + def _got_child(child): + if must_be_directory and not IDirectoryNode.providedBy(child): + raise RuntimeError("rmdir called on a file") + if must_be_file and IDirectoryNode.providedBy(child): + raise RuntimeError("rmfile called on a directory") + return parent.delete(childname) + d.addCallback(_got_child) + d.addErrback(self._convert_error) + return d + d.addCallback(_got_parent) + return d + + + def openDirectory(self, path): + print "OPENDIRECTORY", path + path = self._convert_sftp_path(path) + d = self._get_node_and_metadata_for_path(path) + d.addCallback(lambda (dirnode,metadata): dirnode.list()) + def _render(children): + results = [] + for filename, (node, metadata) in children.iteritems(): + s = FakeStat() + if IDirectoryNode.providedBy(node): + s.st_mode = 040700 + s.st_size = 0 + else: + s.st_mode = 0100600 + s.st_size = node.get_size() + s.st_nlink = 1 + s.st_uid = 0 + s.st_gid = 0 + s.st_mtime = int(metadata.get("mtime", 0)) + longname = ls.lsLine(filename.encode("utf-8"), s) + attrs = self._populate_attrs(node, metadata) + results.append( (filename.encode("utf-8"), longname, attrs) ) + return StoppableList(results) + d.addCallback(_render) + return d + + def getAttrs(self, path, followLinks): + print "GETATTRS", path, followLinks + # from ftp.stat + d = self._get_node_and_metadata_for_path(self._convert_sftp_path(path)) + def _render((node,metadata)): + return self._populate_attrs(node, metadata) + d.addCallback(_render) + d.addErrback(self._convert_error) + def _done(res): + print " DONE", res + return res + d.addBoth(_done) + return d + + def _convert_sftp_path(self, pathstring): + assert pathstring[0] == "/" + pathstring = pathstring.strip("/") + if pathstring == "": + path = [] + else: + path = pathstring.split("/") + print "CONVERT", pathstring, path + path = [unicode(p) for p in path] + return path + + def _get_node_and_metadata_for_path(self, path): + d = self._get_root(path) + def _got_root((root,path)): + print "ROOT", root + print "PATH", path + if path: + return root.get_child_and_metadata_at_path(path) + else: + return (root,{}) + d.addCallback(_got_root) + return d + + def _get_root(self, path): + # return (root, remaining_path) + path = [unicode(p) for p in path] + if path and path[0] == "uri": + d = defer.maybeDeferred(self.client.create_node_from_uri, + str(path[1])) + d.addCallback(lambda root: (root, path[2:])) + else: + d = defer.succeed((self.root,path)) + return d + + def _populate_attrs(self, childnode, metadata): + attrs = {} + attrs["uid"] = 1000 + attrs["gid"] = 1000 + attrs["atime"] = 0 + attrs["mtime"] = int(metadata.get("mtime", 0)) + isdir = bool(IDirectoryNode.providedBy(childnode)) + if isdir: + attrs["size"] = 1 + # the permissions must have the extra bits (040000 or 0100000), + # otherwise the client will not call openDirectory + attrs["permissions"] = 040700 # S_IFDIR + else: + attrs["size"] = childnode.get_size() + attrs["permissions"] = 0100600 # S_IFREG + return attrs + + def _convert_error(self, f): + if f.check(NoSuchChildError): + childname = f.value.args[0].encode("utf-8") + raise SFTPError(FX_NO_SUCH_FILE, childname) + if f.check(ExistingChildError): + msg = f.value.args[0].encode("utf-8") + raise SFTPError(FX_FILE_ALREADY_EXISTS, msg) + if f.check(PermissionError): + raise SFTPError(FX_PERMISSION_DENIED, str(f.value)) + if f.check(NotImplementedError): + raise SFTPError(FX_OP_UNSUPPORTED, str(f.value)) + return f + + + def setAttrs(self, path, attrs): + print "SETATTRS", path, attrs + # ignored + return None + + def readLink(self, path): + print "READLINK", path + raise NotImplementedError + + def makeLink(self, linkPath, targetPath): + print "MAKELINK", linkPath, targetPath + raise NotImplementedError + + def extendedRequest(self, extendedName, extendedData): + print "EXTENDEDREQUEST", extendedName, extendedData + # client 'df' command requires 'statvfs@openssh.com' extension + raise NotImplementedError + def realPath(self, path): + print "REALPATH", path + if path == ".": + return "/" + return path + + + def _get_parent(self, path): + # fire with (parentnode, childname) + path = [unicode(p) for p in path] + if not path: + raise NoParentError + childname = path[-1] + d = self._get_root(path) + def _got_root((root, path)): + if not path: + raise NoParentError + return root.get_child_at_path(path[:-1]) + d.addCallback(_got_root) + def _got_parent(parent): + return (parent, childname) + d.addCallback(_got_parent) + return d + + +class FTPAvatarID: + def __init__(self, username, rootcap): + self.username = username + self.rootcap = rootcap + +class AccountFileChecker: + implements(checkers.ICredentialsChecker) + credentialInterfaces = (credentials.IUsernamePassword, + credentials.IUsernameHashedPassword) + def __init__(self, client, accountfile): + self.client = client + self.passwords = {} + self.pubkeys = {} + self.rootcaps = {} + for line in open(os.path.expanduser(accountfile), "r"): + line = line.strip() + if line.startswith("#") or not line: + continue + name, passwd, rest = line.split(None, 2) + if passwd in ("ssh-dss", "ssh-rsa"): + bits = rest.split() + keystring = " ".join(bits[-1]) + rootcap = bits[-1] + self.pubkeys[name] = keystring + else: + self.passwords[name] = passwd + rootcap = rest + self.rootcaps[name] = rootcap + + def _cbPasswordMatch(self, matched, username): + if matched: + return FTPAvatarID(username, self.rootcaps[username]) + raise error.UnauthorizedLogin + + def requestAvatarId(self, credentials): + if credentials.username in self.passwords: + d = defer.maybeDeferred(credentials.checkPassword, + self.passwords[credentials.username]) + d.addCallback(self._cbPasswordMatch, str(credentials.username)) + return d + return defer.fail(error.UnauthorizedLogin()) + +class AccountURLChecker: + implements(checkers.ICredentialsChecker) + credentialInterfaces = (credentials.IUsernamePassword,) + + def __init__(self, client, auth_url): + self.client = client + self.auth_url = auth_url + + def _cbPasswordMatch(self, rootcap, username): + return FTPAvatarID(username, rootcap) + + def post_form(self, username, password): + sepbase = base32.b2a(os.urandom(4)) + sep = "--" + sepbase + form = [] + form.append(sep) + fields = {"action": "authenticate", + "email": username, + "passwd": password, + } + for name, value in fields.iteritems(): + form.append('Content-Disposition: form-data; name="%s"' % name) + form.append('') + assert isinstance(value, str) + form.append(value) + form.append(sep) + form[-1] += "--" + body = "\r\n".join(form) + "\r\n" + headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase, + } + return getPage(self.auth_url, method="POST", + postdata=body, headers=headers, + followRedirect=True, timeout=30) + + def _parse_response(self, res): + rootcap = res.strip() + if rootcap == "0": + raise error.UnauthorizedLogin + return rootcap + + def requestAvatarId(self, credentials): + # construct a POST to the login form. While this could theoretically + # be done with something like the stdlib 'email' package, I can't + # figure out how, so we just slam together a form manually. + d = self.post_form(credentials.username, credentials.password) + d.addCallback(self._parse_response) + d.addCallback(self._cbPasswordMatch, str(credentials.username)) + return d + +# if you have an SFTPUser, and you want something that provides ISFTPServer, +# then you get SFTPHandler(user) +components.registerAdapter(SFTPHandler, SFTPUser, ISFTPServer) + +class Dispatcher: + implements(portal.IRealm) + def __init__(self, client): + self.client = client + + def requestAvatar(self, avatarID, mind, interface): + assert interface == IConchUser + rootnode = self.client.create_node_from_uri(avatarID.rootcap) + convergence = self.client.convergence + s = SFTPUser(self.client, rootnode, avatarID.username, convergence) + def logout(): pass + return (interface, s, logout) + +class SFTPServer(service.MultiService): + def __init__(self, client, accountfile, accounturl, + sftp_portstr, pubkey_file, privkey_file): + service.MultiService.__init__(self) + + if accountfile: + c = AccountFileChecker(self, accountfile) + elif accounturl: + c = AccountURLChecker(self, accounturl) + else: + # we could leave this anonymous, with just the /uri/CAP form + raise RuntimeError("must provide some translation") + + r = Dispatcher(client) + p = portal.Portal(r) + p.registerChecker(c) + + pubkey = keys.Key.fromFile(pubkey_file) + privkey = keys.Key.fromFile(privkey_file) + class SSHFactory(factory.SSHFactory): + publicKeys = {pubkey.sshType(): pubkey} + privateKeys = {privkey.sshType(): privkey} + def getPrimes(self): + try: + # if present, this enables diffie-hellman-group-exchange + return primes.parseModuliFile("/etc/ssh/moduli") + except IOError: + return None + + f = SSHFactory() + f.portal = p + + s = strports.service(sftp_portstr, f) + s.setServiceParent(self) +