import simplejson
-from zope.interface import implements
-from twisted.internet.interfaces import IConsumer
-from twisted.web import http, static, resource, server
+from twisted.web import http, static
from twisted.internet import defer
from nevow import url, rend
from nevow.inevow import IRequest
-from allmydata.interfaces import IDownloadTarget, ExistingChildError
-from allmydata.mutable.common import MODE_READ
+from allmydata.interfaces import ExistingChildError, SDMF_VERSION, MDMF_VERSION
+from allmydata.monitor import Monitor
from allmydata.immutable.upload import FileHandle
-from allmydata.util import log
-
-from allmydata.web.common import text_plain, WebError, IClient, RenderMixin, \
- boolean_of_arg, get_arg, should_create_intermediate_directories
-from allmydata.web.checker_results import CheckerResults
+from allmydata.mutable.publish import MutableFileHandle
+from allmydata.mutable.common import MODE_READ
+from allmydata.util import log, base32
+from allmydata.util.encodingutil import quote_output
+from allmydata.blacklist import FileProhibited, ProhibitedNode
+
+from allmydata.web.common import text_plain, WebError, RenderMixin, \
+ boolean_of_arg, get_arg, should_create_intermediate_directories, \
+ MyExceptionHandler, parse_replace_arg, parse_offset_arg, \
+ get_format, get_mutable_type
+from allmydata.web.check_results import CheckResults, \
+ CheckAndRepairResults, LiteralCheckResults
+from allmydata.web.info import MoreInfo
class ReplaceMeMixin:
-
- def replace_me_with_a_child(self, ctx, replace):
+ def replace_me_with_a_child(self, req, client, replace):
# a new file is being uploaded in our place.
- req = IRequest(ctx)
- client = IClient(ctx)
- mutable = boolean_of_arg(get_arg(req, "mutable", "false"))
- if mutable:
- req.content.seek(0)
- data = req.content.read()
- d = client.create_mutable_file(data)
+ file_format = get_format(req, "CHK")
+ mutable_type = get_mutable_type(file_format)
+ if mutable_type is not None:
+ data = MutableFileHandle(req.content)
+ d = client.create_mutable_file(data, version=mutable_type)
def _uploaded(newnode):
d2 = self.parentnode.set_node(self.name, newnode,
overwrite=replace)
return d2
d.addCallback(_uploaded)
else:
+ assert file_format == "CHK"
uploadable = FileHandle(req.content, convergence=client.convergence)
d = self.parentnode.add_file(self.name, uploadable,
overwrite=replace)
def _done(filenode):
log.msg("webish upload complete",
- facility="tahoe.webish", level=log.NOISY)
+ facility="tahoe.webish", level=log.NOISY, umid="TCjBGQ")
if self.node:
# we've replaced an existing file (or modified a mutable
# file), so the response code is 200
d.addCallback(_done)
return d
- def replace_me_with_a_childcap(self, ctx, replace):
- req = IRequest(ctx)
+ def replace_me_with_a_childcap(self, req, client, replace):
req.content.seek(0)
childcap = req.content.read()
- client = IClient(ctx)
- childnode = client.create_node_from_uri(childcap)
+ childnode = client.create_node_from_uri(childcap, None, name=self.name)
d = self.parentnode.set_node(self.name, childnode, overwrite=replace)
d.addCallback(lambda res: childnode.get_uri())
return d
- def _read_data_from_formpost(self, req):
- # SDMF: files are small, and we can only upload data, so we read
- # the whole file into memory before uploading.
- contents = req.fields["file"]
- contents.file.seek(0)
- data = contents.file.read()
- return data
- def replace_me_with_a_formpost(self, ctx, replace):
+ def replace_me_with_a_formpost(self, req, client, replace):
# create a new file, maybe mutable, maybe immutable
- req = IRequest(ctx)
- client = IClient(ctx)
- mutable = boolean_of_arg(get_arg(req, "mutable", "false"))
-
- if mutable:
- data = self._read_data_from_formpost(req)
- d = client.create_mutable_file(data)
+ file_format = get_format(req, "CHK")
+ contents = req.fields["file"]
+ if file_format in ("SDMF", "MDMF"):
+ mutable_type = get_mutable_type(file_format)
+ uploadable = MutableFileHandle(contents.file)
+ d = client.create_mutable_file(uploadable, version=mutable_type)
def _uploaded(newnode):
d2 = self.parentnode.set_node(self.name, newnode,
overwrite=replace)
return d2
d.addCallback(_uploaded)
return d
- # create an immutable file
- contents = req.fields["file"]
+
uploadable = FileHandle(contents.file, convergence=client.convergence)
d = self.parentnode.add_file(self.name, uploadable, overwrite=replace)
d.addCallback(lambda newnode: newnode.get_uri())
return d
+
class PlaceHolderNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
- def __init__(self, parentnode, name):
+ def __init__(self, client, parentnode, name):
rend.Page.__init__(self)
+ self.client = client
assert parentnode
self.parentnode = parentnode
self.name = name
def render_PUT(self, ctx):
req = IRequest(ctx)
t = get_arg(req, "t", "").strip()
- replace = boolean_of_arg(get_arg(req, "replace", "true"))
+ replace = parse_replace_arg(get_arg(req, "replace", "true"))
+
assert self.parentnode and self.name
+ if req.getHeader("content-range"):
+ raise WebError("Content-Range in PUT not yet supported",
+ http.NOT_IMPLEMENTED)
if not t:
- return self.replace_me_with_a_child(ctx, replace)
+ return self.replace_me_with_a_child(req, self.client, replace)
if t == "uri":
- return self.replace_me_with_a_childcap(ctx, replace)
+ return self.replace_me_with_a_childcap(req, self.client, replace)
raise WebError("PUT to a file: bad t=%s" % t)
# or POST /uri/path/file?t=upload, or
# POST /uri/path/dir?t=upload&name=foo . All have the same
# behavior, we just ignore any name= argument
- d = self.replace_me_with_a_formpost(ctx, replace)
+ d = self.replace_me_with_a_formpost(req, self.client, replace)
else:
# t=mkdir is handled in DirectoryNodeHandler._POST_mkdir, so
# there are no other t= values left to be handled by the
class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
- def __init__(self, node, parentnode=None, name=None):
+ def __init__(self, client, node, parentnode=None, name=None):
rend.Page.__init__(self)
+ self.client = client
assert node
self.node = node
self.parentnode = parentnode
def childFactory(self, ctx, name):
req = IRequest(ctx)
+ if isinstance(self.node, ProhibitedNode):
+ raise FileProhibited(self.node.reason)
if should_create_intermediate_directories(req):
- raise WebError("Cannot create directory '%s', because its "
- "parent is a file, not a directory" % name)
- raise WebError("Files have no children, certainly not named '%s'"
- % name)
+ raise WebError("Cannot create directory %s, because its "
+ "parent is a file, not a directory" % quote_output(name, encoding='utf-8'))
+ raise WebError("Files have no children, certainly not named %s"
+ % quote_output(name, encoding='utf-8'))
def render_GET(self, ctx):
req = IRequest(ctx)
t = get_arg(req, "t", "").strip()
if not t:
# just get the contents
+ # the filename arrives as part of the URL or in a form input
+ # element, and will be sent back in a Content-Disposition header.
+ # Different browsers use various character sets for this name,
+ # sometimes depending upon how language environment is
+ # configured. Firefox sends the equivalent of
+ # urllib.quote(name.encode("utf-8")), while IE7 sometimes does
+ # latin-1. Browsers cannot agree on how to interpret the name
+ # they see in the Content-Disposition header either, despite some
+ # 11-year old standards (RFC2231) that explain how to do it
+ # properly. So we assume that at least the browser will agree
+ # with itself, and echo back the same bytes that we were given.
filename = get_arg(req, "filename", self.name) or "unknown"
- save_to_file = boolean_of_arg(get_arg(req, "save", "False"))
- return FileDownloader(self.node, filename, save_to_file)
+ d = self.node.get_best_readable_version()
+ d.addCallback(lambda dn: FileDownloader(dn, filename))
+ return d
if t == "json":
- return FileJSONMetadata(ctx, self.node)
+ # We do this to make sure that fields like size and
+ # mutable-type (which depend on the file on the grid and not
+ # just on the cap) are filled in. The latter gets used in
+ # tests, in particular.
+ #
+ # TODO: Make it so that the servermap knows how to update in
+ # a mode specifically designed to fill in these fields, and
+ # then update it in that mode.
+ if self.node.is_mutable():
+ d = self.node.get_servermap(MODE_READ)
+ else:
+ d = defer.succeed(None)
+ if self.parentnode and self.name:
+ d.addCallback(lambda ignored:
+ self.parentnode.get_metadata_for(self.name))
+ else:
+ d.addCallback(lambda ignored: None)
+ d.addCallback(lambda md: FileJSONMetadata(ctx, self.node, md))
+ return d
+ if t == "info":
+ return MoreInfo(self.node)
if t == "uri":
return FileURI(ctx, self.node)
if t == "readonly-uri":
t = get_arg(req, "t", "").strip()
if t:
raise WebError("GET file: bad t=%s" % t)
- if self.node.is_mutable():
- # update the servermap to get the size of this file without
- # downloading the full contents.
- d = self.node.get_servermap(MODE_READ)
- def _got_servermap(smap):
- ver = smap.best_recoverable_version()
- if not ver:
- raise WebError("Unable to recover this file",
- http.NOT_FOUND)
- length = smap.size_of_version(ver)
- return length
- d.addCallback(_got_servermap)
- # otherwise, we can get the size from the URI
- else:
- d = defer.succeed(self.node.get_size())
- def _got_length(length):
- req.setHeader("content-length", length)
- return ""
- d.addCallback(_got_length)
+ filename = get_arg(req, "filename", self.name) or "unknown"
+ d = self.node.get_best_readable_version()
+ d.addCallback(lambda dn: FileDownloader(dn, filename))
return d
def render_PUT(self, ctx):
req = IRequest(ctx)
t = get_arg(req, "t", "").strip()
- replace = boolean_of_arg(get_arg(req, "replace", "true"))
+ replace = parse_replace_arg(get_arg(req, "replace", "true"))
+ offset = parse_offset_arg(get_arg(req, "offset", None))
+
if not t:
- if self.node.is_mutable():
- return self.replace_my_contents(ctx)
if not replace:
# this is the early trap: if someone else modifies the
# directory while we're uploading, the add_file(overwrite=)
# call in replace_me_with_a_child will do the late trap.
raise ExistingChildError()
- assert self.parentnode and self.name
- return self.replace_me_with_a_child(ctx, replace)
+
+ if self.node.is_mutable():
+ # Are we a readonly filenode? We shouldn't allow callers
+ # to try to replace us if we are.
+ if self.node.is_readonly():
+ raise WebError("PUT to a mutable file: replace or update"
+ " requested with read-only cap")
+ if offset is None:
+ return self.replace_my_contents(req)
+
+ if offset >= 0:
+ return self.update_my_contents(req, offset)
+
+ raise WebError("PUT to a mutable file: Invalid offset")
+
+ else:
+ if offset is not None:
+ raise WebError("PUT to a file: append operation invoked "
+ "on an immutable cap")
+
+ assert self.parentnode and self.name
+ return self.replace_me_with_a_child(req, self.client, replace)
+
if t == "uri":
if not replace:
raise ExistingChildError()
assert self.parentnode and self.name
- return self.replace_me_with_a_childcap(ctx, replace)
+ return self.replace_me_with_a_childcap(req, self.client, replace)
raise WebError("PUT to a file: bad t=%s" % t)
# POST /uri/path/dir?t=upload&name=foo . All have the same
# behavior, we just ignore any name= argument
if self.node.is_mutable():
- d = self.replace_my_contents_with_a_formpost(ctx)
+ d = self.replace_my_contents_with_a_formpost(req)
else:
if not replace:
raise ExistingChildError()
assert self.parentnode and self.name
- d = self.replace_me_with_a_formpost(ctx, replace)
+ d = self.replace_me_with_a_formpost(req, self.client, replace)
else:
raise WebError("POST to file: bad t=%s" % t)
d.addCallback(lambda res: url.URL.fromString(when_done))
return d
+ def _maybe_literal(self, res, Results_Class):
+ if res:
+ return Results_Class(self.client, res)
+ return LiteralCheckResults(self.client)
+
def _POST_check(self, req):
- d = self.node.check()
- d.addCallback(lambda res: CheckerResults(res))
+ verify = boolean_of_arg(get_arg(req, "verify", "false"))
+ repair = boolean_of_arg(get_arg(req, "repair", "false"))
+ add_lease = boolean_of_arg(get_arg(req, "add-lease", "false"))
+ if repair:
+ d = self.node.check_and_repair(Monitor(), verify, add_lease)
+ d.addCallback(self._maybe_literal, CheckAndRepairResults)
+ else:
+ d = self.node.check(Monitor(), verify, add_lease)
+ d.addCallback(self._maybe_literal, CheckResults)
return d
def render_DELETE(self, ctx):
d.addCallback(lambda res: self.node.get_uri())
return d
- def replace_my_contents(self, ctx):
- req = IRequest(ctx)
+ def replace_my_contents(self, req):
req.content.seek(0)
- new_contents = req.content.read()
+ new_contents = MutableFileHandle(req.content)
d = self.node.overwrite(new_contents)
d.addCallback(lambda res: self.node.get_uri())
return d
- def replace_my_contents_with_a_formpost(self, ctx):
+
+ def update_my_contents(self, req, offset):
+ req.content.seek(0)
+ added_contents = MutableFileHandle(req.content)
+
+ d = self.node.get_best_mutable_version()
+ d.addCallback(lambda mv:
+ mv.update(added_contents, offset))
+ d.addCallback(lambda ignored:
+ self.node.get_uri())
+ return d
+
+
+ def replace_my_contents_with_a_formpost(self, req):
# we have a mutable file. Get the data from the formpost, and replace
# the mutable file's contents with it.
- req = IRequest(ctx)
- new_contents = self._read_data_from_formpost(req)
+ new_contents = req.fields['file']
+ new_contents = MutableFileHandle(new_contents.file)
+
d = self.node.overwrite(new_contents)
d.addCallback(lambda res: self.node.get_uri())
return d
-class WebDownloadTarget:
- implements(IDownloadTarget, IConsumer)
- def __init__(self, req, content_type, content_encoding, save_to_filename):
- self._req = req
- self._content_type = content_type
- self._content_encoding = content_encoding
- self._opened = False
- self._producer = None
- self._save_to_filename = save_to_filename
-
- def registerProducer(self, producer, streaming):
- self._req.registerProducer(producer, streaming)
- def unregisterProducer(self):
- self._req.unregisterProducer()
-
- def open(self, size):
- self._opened = True
- self._req.setHeader("content-type", self._content_type)
- if self._content_encoding:
- self._req.setHeader("content-encoding", self._content_encoding)
- self._req.setHeader("content-length", str(size))
- if self._save_to_filename is not None:
- # tell the browser to save the file rather display it
- # TODO: indicate charset of filename= properly
- filename = self._save_to_filename.encode("utf-8")
- self._req.setHeader("content-disposition",
- 'attachment; filename="%s"'
- % filename)
-
- def write(self, data):
- self._req.write(data)
- def close(self):
- self._req.finish()
-
- def fail(self, why):
- if self._opened:
- # The content-type is already set, and the response code
- # has already been sent, so we can't provide a clean error
- # indication. We can emit text (which a browser might interpret
- # as something else), and if we sent a Size header, they might
- # notice that we've truncated the data. Keep the error message
- # small to improve the chances of having our error response be
- # shorter than the intended results.
- #
- # We don't have a lot of options, unfortunately.
- self._req.write("problem during download\n")
- else:
- # We haven't written anything yet, so we can provide a sensible
- # error message.
- msg = str(why.type)
- msg.replace("\n", "|")
- self._req.setResponseCode(http.GONE, msg)
- self._req.setHeader("content-type", "text/plain")
- # TODO: HTML-formatted exception?
- self._req.write(str(why))
- self._req.finish()
-
- def register_canceller(self, cb):
- pass
- def finish(self):
- pass
-
-class FileDownloader(resource.Resource):
- # since we override the rendering process (to let the tahoe Downloader
- # drive things), we must inherit from regular old twisted.web.resource
- # instead of nevow.rend.Page . Nevow will use adapters to wrap a
- # nevow.appserver.OldResourceAdapter around any
- # twisted.web.resource.IResource that it is given. TODO: it looks like
- # that wrapper would allow us to return a Deferred from render(), which
- # might could simplify the implementation of WebDownloadTarget.
-
- def __init__(self, filenode, filename, save_to_file):
- resource.Resource.__init__(self)
+class FileDownloader(rend.Page):
+ def __init__(self, filenode, filename):
+ rend.Page.__init__(self)
self.filenode = filenode
self.filename = filename
- self.save_to_file = save_to_file
- def render(self, req):
+
+ def parse_range_header(self, range):
+ # Parse a byte ranges according to RFC 2616 "14.35.1 Byte
+ # Ranges". Returns None if the range doesn't make sense so it
+ # can be ignored (per the spec). When successful, returns a
+ # list of (first,last) inclusive range tuples.
+
+ filesize = self.filenode.get_size()
+ assert isinstance(filesize, (int,long)), filesize
+
+ try:
+ # byte-ranges-specifier
+ units, rangeset = range.split('=', 1)
+ if units != 'bytes':
+ return None # nothing else supported
+
+ def parse_range(r):
+ first, last = r.split('-', 1)
+
+ if first is '':
+ # suffix-byte-range-spec
+ first = filesize - long(last)
+ last = filesize - 1
+ else:
+ # byte-range-spec
+
+ # first-byte-pos
+ first = long(first)
+
+ # last-byte-pos
+ if last is '':
+ last = filesize - 1
+ else:
+ last = long(last)
+
+ if last < first:
+ raise ValueError
+
+ return (first, last)
+
+ # byte-range-set
+ #
+ # Note: the spec uses "1#" for the list of ranges, which
+ # implicitly allows whitespace around the ',' separators,
+ # so strip it.
+ return [ parse_range(r.strip()) for r in rangeset.split(',') ]
+ except ValueError:
+ return None
+
+ def renderHTTP(self, ctx):
+ req = IRequest(ctx)
gte = static.getTypeAndEncoding
ctype, encoding = gte(self.filename,
static.File.contentTypes,
static.File.contentEncodings,
defaultType="text/plain")
- save_to_filename = None
- if self.save_to_file:
- save_to_filename = self.filename
- wdt = WebDownloadTarget(req, ctype, encoding, save_to_filename)
- d = self.filenode.download(wdt)
- # exceptions during download are handled by the WebDownloadTarget
- d.addErrback(lambda why: None)
- return server.NOT_DONE_YET
-
-def FileJSONMetadata(ctx, filenode):
- if filenode.is_readonly():
- rw_uri = None
- ro_uri = filenode.get_uri()
- else:
- rw_uri = filenode.get_uri()
- ro_uri = filenode.get_readonly_uri()
+ req.setHeader("content-type", ctype)
+ if encoding:
+ req.setHeader("content-encoding", encoding)
+
+ if boolean_of_arg(get_arg(req, "save", "False")):
+ # tell the browser to save the file rather display it we don't
+ # try to encode the filename, instead we echo back the exact same
+ # bytes we were given in the URL. See the comment in
+ # FileNodeHandler.render_GET for the sad details.
+ req.setHeader("content-disposition",
+ 'attachment; filename="%s"' % self.filename)
+
+ filesize = self.filenode.get_size()
+ assert isinstance(filesize, (int,long)), filesize
+ first, size = 0, None
+ contentsize = filesize
+ req.setHeader("accept-ranges", "bytes")
+ if not self.filenode.is_mutable():
+ # TODO: look more closely at Request.setETag and how it interacts
+ # with a conditional "if-etag-equals" request, I think this may
+ # need to occur after the setResponseCode below
+ si = self.filenode.get_storage_index()
+ if si:
+ req.setETag(base32.b2a(si))
+ # TODO: for mutable files, use the roothash. For LIT, hash the data.
+ # or maybe just use the URI for CHK and LIT.
+ rangeheader = req.getHeader('range')
+ if rangeheader:
+ ranges = self.parse_range_header(rangeheader)
+
+ # ranges = None means the header didn't parse, so ignore
+ # the header as if it didn't exist. If is more than one
+ # range, then just return the first for now, until we can
+ # generate multipart/byteranges.
+ if ranges is not None:
+ first, last = ranges[0]
+
+ if first >= filesize:
+ raise WebError('First beyond end of file',
+ http.REQUESTED_RANGE_NOT_SATISFIABLE)
+ else:
+ first = max(0, first)
+ last = min(filesize-1, last)
+
+ req.setResponseCode(http.PARTIAL_CONTENT)
+ req.setHeader('content-range',"bytes %s-%s/%s" %
+ (str(first), str(last),
+ str(filesize)))
+ contentsize = last - first + 1
+ size = contentsize
+
+ req.setHeader("content-length", str(contentsize))
+ if req.method == "HEAD":
+ return ""
+
+ # Twisted >=9.0 throws an error if we call req.finish() on a closed
+ # HTTP connection. It also has req.notifyFinish() to help avoid it.
+ finished = []
+ def _request_finished(ign):
+ finished.append(True)
+ if hasattr(req, "notifyFinish"):
+ req.notifyFinish().addBoth(_request_finished)
+
+ d = self.filenode.read(req, first, size)
+
+ def _finished(ign):
+ if not finished:
+ req.finish()
+ def _error(f):
+ lp = log.msg("error during GET", facility="tahoe.webish", failure=f,
+ level=log.UNUSUAL, umid="xSiF3w")
+ if finished:
+ log.msg("but it's too late to tell them", parent=lp,
+ level=log.UNUSUAL, umid="j1xIbw")
+ return
+ req._tahoe_request_had_error = f # for HTTP-style logging
+ if req.startedWriting:
+ # The content-type is already set, and the response code has
+ # already been sent, so we can't provide a clean error
+ # indication. We can emit text (which a browser might
+ # interpret as something else), and if we sent a Size header,
+ # they might notice that we've truncated the data. Keep the
+ # error message small to improve the chances of having our
+ # error response be shorter than the intended results.
+ #
+ # We don't have a lot of options, unfortunately.
+ req.write("problem during download\n")
+ req.finish()
+ else:
+ # We haven't written anything yet, so we can provide a
+ # sensible error message.
+ eh = MyExceptionHandler()
+ eh.renderHTTP_exception(ctx, f)
+ d.addCallbacks(_finished, _error)
+ return req.deferred
+
+
+def FileJSONMetadata(ctx, filenode, edge_metadata):
+ rw_uri = filenode.get_write_uri()
+ ro_uri = filenode.get_readonly_uri()
data = ("filenode", {})
data[1]['size'] = filenode.get_size()
if ro_uri:
data[1]['ro_uri'] = ro_uri
if rw_uri:
data[1]['rw_uri'] = rw_uri
+ verifycap = filenode.get_verify_cap()
+ if verifycap:
+ data[1]['verify_uri'] = verifycap.to_string()
data[1]['mutable'] = filenode.is_mutable()
- return text_plain(simplejson.dumps(data, indent=1), ctx)
+ if edge_metadata is not None:
+ data[1]['metadata'] = edge_metadata
+
+ if filenode.is_mutable():
+ mutable_type = filenode.get_version()
+ assert mutable_type in (SDMF_VERSION, MDMF_VERSION)
+ if mutable_type == MDMF_VERSION:
+ file_format = "MDMF"
+ else:
+ file_format = "SDMF"
+ else:
+ file_format = "CHK"
+ data[1]['format'] = file_format
+
+ return text_plain(simplejson.dumps(data, indent=1) + "\n", ctx)
def FileURI(ctx, filenode):
return text_plain(filenode.get_uri(), ctx)
class FileNodeDownloadHandler(FileNodeHandler):
def childFactory(self, ctx, name):
- return FileNodeDownloadHandler(self.node, name=name)
+ return FileNodeDownloadHandler(self.client, self.node, name=name)