From: david-sarah Date: Wed, 27 Jan 2010 07:03:09 +0000 (-0800) Subject: Miscellaneous documentation, test, and code formatting tweaks. X-Git-Tag: trac-4200~6 X-Git-Url: https://git.rkrishnan.org/components/%22news.html/architecture.txt?a=commitdiff_plain;h=56c00cb381d3cd8c61985242cebebfd0c331af66;p=tahoe-lafs%2Ftahoe-lafs.git Miscellaneous documentation, test, and code formatting tweaks. --- diff --git a/contrib/fuse/impl_c/blackmatch.py b/contrib/fuse/impl_c/blackmatch.py index c8d2ed61..c27b15f2 100644 --- a/contrib/fuse/impl_c/blackmatch.py +++ b/contrib/fuse/impl_c/blackmatch.py @@ -906,7 +906,7 @@ def stat_to_dict(statobj, fields=None): class TStat(fuse.Stat): # in fuse 0.2, these are set by fuse.Stat.__init__ - # in fuse 0.2-pre3 (hardy) they are not. badness unsues if they're missing + # in fuse 0.2-pre3 (hardy) they are not. badness ensues if they're missing st_mode = None st_ino = 0 st_dev = 0 @@ -1019,6 +1019,7 @@ class Directory(object): def get_uri(self): return self.rw_uri or self.ro_uri + # TODO: rename to 'is_writeable', or switch sense to 'is_readonly', for consistency with Tahoe code def writable(self): return self.rw_uri and self.rw_uri != self.ro_uri diff --git a/docs/frontends/webapi.txt b/docs/frontends/webapi.txt index 419cf42f..12435f4e 100644 --- a/docs/frontends/webapi.txt +++ b/docs/frontends/webapi.txt @@ -70,20 +70,20 @@ delete), because otherwise a regular web browser has no way to accomplish these tasks. In general, everything that can be done with a PUT or DELETE can also be done with a POST. -Tahoe's web API is designed for two different consumers. The first is a -program that needs to manipulate the virtual file system. Such programs are +Tahoe's web API is designed for two different kinds of consumer. The first is +a program that needs to manipulate the virtual file system. Such programs are expected to use the RESTful interface described above. The second is a human using a standard web browser to work with the filesystem. This user is given a series of HTML pages with links to download files, and forms that use POST actions to upload, rename, and delete files. When an error occurs, the HTTP response code will be set to an appropriate -400-series code (like 404 for an unknown childname, or 400 Gone when a file -is unrecoverable due to insufficient shares), and the HTTP response body will -usually contain a few lines of explanation as to the cause of the error and -possible responses. Unusual exceptions may result in a 500 Internal Server -Error as a catch-all, with a default response body will contain a -Nevow-generated HTML-ized representation of the Python exception stack trace +400-series code (like 404 Not Found for an unknown childname, or 400 Bad Request +when the parameters to a webapi operation are invalid), and the HTTP response +body will usually contain a few lines of explanation as to the cause of the +error and possible responses. Unusual exceptions may result in a +500 Internal Server Error as a catch-all, with a default response body containing +a Nevow-generated HTML-ized representation of the Python exception stack trace that caused the problem. CLI programs which want to copy the response body to stderr should provide an "Accept: text/plain" header to their requests to get a plain text stack trace instead. If the Accept header contains */*, or @@ -108,9 +108,9 @@ give read-only access to a directory. Finally there are also mutable file read- and write- caps, which start with "URI:SSK", and give access to mutable files. -(later versions of Tahoe will make these strings shorter, and will remove the +(Later versions of Tahoe will make these strings shorter, and will remove the unfortunate colons, which must be escaped when these caps are embedded in -URLs). +URLs.) To refer to any Tahoe object through the web API, you simply need to combine a prefix (which indicates the HTTP server to use) with the cap (which @@ -121,7 +121,7 @@ listening on this port: http://127.0.0.1:3456/uri/ + $CAP So, to access the directory named above (which happens to be the -publically-writable sample directory on the Tahoe test grid, described at +publically-writeable sample directory on the Tahoe test grid, described at http://allmydata.org/trac/tahoe/wiki/TestGrid), the URL would be: http://127.0.0.1:3456/uri/URI%3ADIR2%3Adjrdkfawoqihigoett4g6auz6a%3Ajx5mplfpwexnoqff7y5e4zjus4lidm76dcuarpct7cckorh2dpgq/ @@ -199,9 +199,9 @@ contain unicode filenames, and cannot contain binary strings that are not representable as such. All Tahoe operations that refer to existing files or directories must include -a suitable read- or write- cap in the URL: the wapi server won't add one +a suitable read- or write- cap in the URL: the webapi server won't add one for you. If you don't know the cap, you can't access the file. This allows -the security properties of Tahoe caps to be extended across the wapi +the security properties of Tahoe caps to be extended across the webapi interface. == Slow Operations, Progress, and Cancelling == @@ -275,7 +275,7 @@ If a retain-for= argument is not used, the default handle lifetimes are: since the operation completed) will remain valid for ten minutes. Many "slow" operations can begin to use unacceptable amounts of memory when -operation on large directory structures. The memory usage increases when the +operating on large directory structures. The memory usage increases when the ophandle is polled, as the results must be copied into a JSON string, sent over the wire, then parsed by a client. So, as an alternative, many "slow" operations have streaming equivalents. These equivalents do not use operation @@ -315,10 +315,10 @@ PUT /uri/$DIRCAP/[SUBDIRS../]FILENAME retrieve the same contents that were just uploaded. This will create any necessary intermediate subdirectories. - To use the /uri/$FILECAP form, $FILECAP be a write-cap for a mutable file. + To use the /uri/$FILECAP form, $FILECAP must be a write-cap for a mutable file. In the /uri/$DIRCAP/[SUBDIRS../]FILENAME form, if the target file is a - writable mutable file, that files contents will be overwritten in-place. If + writeable mutable file, that file's contents will be overwritten in-place. If it is a read-cap for a mutable file, an error will occur. If it is an immutable file, the old file will be discarded, and a new one will be put in its place. @@ -337,7 +337,7 @@ PUT /uri/$DIRCAP/[SUBDIRS../]FILENAME PUT /uri This uploads a file, and produces a file-cap for the contents, but does not - attach the file into the virtual drive. No directories will be modified by + attach the file into the filesystem. No directories will be modified by this operation. The file-cap is returned as the body of the HTTP response. If "mutable=true" is in the query arguments, the operation will create a @@ -351,7 +351,7 @@ PUT /uri?t=mkdir Create a new empty directory and return its write-cap as the HTTP response body. This does not make the newly created directory visible from the - virtual drive. The "PUT" operation is provided for backwards compatibility: + filesystem. The "PUT" operation is provided for backwards compatibility: new code should use POST. POST /uri?t=mkdir-with-children @@ -392,7 +392,7 @@ POST /uri?t=mkdir-with-children "linkcrtime": 1202777696.7564139, "linkmotime": 1202777696.7564139, } } } ] - } + } For forward-compatibility, a mutable directory can also contain caps in a format that is unknown to the webapi server. When such caps are retrieved @@ -522,7 +522,7 @@ POST /uri/$DIRCAP/[SUBDIRS../]?t=mkdir-with-children&name=NAME the immediate parent directory already has a a child named NAME. Note that the name= argument must be passed as a queryarg, because the POST - request body is used for the initial children JSON. + request body is used for the initial children JSON. POST /uri/$DIRCAP/[SUBDIRS../]?t=mkdir-immutable&name=NAME @@ -625,7 +625,7 @@ GET /uri/$DIRCAP/[SUBDIRS../]FILENAME?t=json Then the rw_uri field will be present in the information about a directory if and only if you have read-write access to that directory. The verify_uri - field will be presend if and only if the object has a verify-cap + field will be present if and only if the object has a verify-cap (non-distributed LIT files do not have verify-caps). ==== About the metadata ==== @@ -701,11 +701,11 @@ GET /uri/$DIRCAP/[SUBDIRS../]FILENAME?t=json link points. 4. Also, quite apart from Tahoe, you might be confused about the meaning of - the 'ctime' in unix local filesystems, which people sometimes think means - file creation time, but which actually means, in unix local filesystems, the + the 'ctime' in UNIX local filesystems, which people sometimes think means + file creation time, but which actually means, in UNIX local filesystems, the most recent time that the file contents or the file metadata (such as owner, permission bits, extended attributes, etc.) has changed. Note that although - 'ctime' does not mean file creation time in Unix, it does mean link creation + 'ctime' does not mean file creation time in UNIX, it does mean link creation time in Tahoe, unless the "tahoe backup" command has been used on that link, in which case it means something about the local filesystem file which corresponds to the Tahoe file which is pointed at by the link. It means @@ -713,7 +713,6 @@ GET /uri/$DIRCAP/[SUBDIRS../]FILENAME?t=json Windows) or file-contents-or-metadata-update-time of the local file (if "tahoe backup" was run on a different operating system). - === Attaching an existing File or Directory by its read- or write- cap === PUT /uri/$DIRCAP/[SUBDIRS../]CHILDNAME?t=uri @@ -737,10 +736,15 @@ PUT /uri/$DIRCAP/[SUBDIRS../]CHILDNAME?t=uri if there is already an object at the given location, rather than overwriting the existing object. To allow the operation to overwrite a file, but return an error when trying to overwrite a directory, use - "replace=only-files" (this behavior is closer to the traditional unix "mv" + "replace=only-files" (this behavior is closer to the traditional UNIX "mv" command). Note that "true", "t", and "1" are all synonyms for "True", and "false", "f", and "0" are synonyms for "False", and the parameter is case-insensitive. + + Note that this operation does not take its child cap in the form of + separate "rw_uri" and "ro_uri" fields. Therefore, it cannot accept a + child cap in a format unknown to the webapi server, because the server + is not able to attenuate an unknown write cap to a read cap. === Adding multiple files or directories to a parent directory at once === @@ -802,7 +806,7 @@ DELETE /uri/$DIRCAP/[SUBDIRS../]CHILDNAME The object will only become completely unreachable once 1: there are no reachable directories that reference it, and 2: nobody is holding a read- or write- cap to the object. (This behavior is very similar to the way - hardlinks and anonymous files work in traditional unix filesystems). + hardlinks and anonymous files work in traditional UNIX filesystems). This operation will not modify more than a single directory. Intermediate directories which were implicitly created by PUT or POST methods will *not* @@ -931,7 +935,7 @@ POST /uri/$DIRCAP/[SUBDIRS../]?t=mkdir&name=CHILDNAME POST /uri?t=upload This uploads a file, and produces a file-cap for the contents, but does not - attach the file into the virtual drive. No directories will be modified by + attach the file into the filesystem. No directories will be modified by this operation. The file must be provided as the "file" field of an HTML encoded form body, @@ -1677,7 +1681,7 @@ GET / (introducer status) == Static Files in /public_html == -The wapi server will take any request for a URL that starts with /static +The webapi server will take any request for a URL that starts with /static and serve it from a configurable directory which defaults to $BASEDIR/public_html . This is configured by setting the "[node]web.static" value in $BASEDIR/tahoe.cfg . If this is left at the default value of @@ -1685,10 +1689,10 @@ value in $BASEDIR/tahoe.cfg . If this is left at the default value of served with the contents of the file $BASEDIR/public_html/subdir/foo.html . This can be useful to serve a javascript application which provides a -prettier front-end to the rest of the Tahoe wapi. +prettier front-end to the rest of the Tahoe webapi. -== safety and security issues -- names vs. URIs == +== Safety and security issues -- names vs. URIs == Summary: use explicit file- and dir- caps whenever possible, to reduce the potential for surprises when the filesystem structure is changed. @@ -1774,7 +1778,7 @@ Coordination Directive. Tahoe nodes implement internal serialization to make sure that a single Tahoe node cannot conflict with itself. For example, it is safe to issue two -directory modification requests to a single tahoe node's wapi server at the +directory modification requests to a single tahoe node's webapi server at the same time, because the Tahoe node will internally delay one of them until after the other has finished being applied. (This feature was introduced in Tahoe-1.1; back with Tahoe-1.0 the web client was responsible for serializing diff --git a/relnotes.txt b/relnotes.txt index 6fdf6bb2..3ebce0cc 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,7 +1,7 @@ -ANNOUNCING Tahoe, the Lofty-Atmospheric Filesystem, v1.5 +ANNOUNCING Tahoe, the Lofty-Atmospheric Filesystem, v1.6 The Tahoe-LAFS team is pleased to announce the immediate -availability of version 1.5 of Tahoe, the Lofty Atmospheric +availability of version 1.6 of Tahoe, the Lofty Atmospheric File System. Tahoe-LAFS is the first cloud storage technology which offers @@ -29,15 +29,20 @@ and more. See the Related Projects page on the wiki [3]. COMPATIBILITY -Version 1.5 is fully compatible with the version 1 series of -Tahoe-LAFS. Files written by v1.5 clients can be read by -clients of all versions back to v1.0. v1.5 clients can read -files produced by clients of all versions since v1.0. v1.5 -servers can serve clients of all versions back to v1.0 and v1.5 +Version 1.6 is fully compatible with the version 1 series of +Tahoe-LAFS. Files written by v1.6 clients can be read by +clients of all versions back to v1.0. v1.6 clients can read +files produced by clients of all versions since v1.0. v1.6 +servers can serve clients of all versions back to v1.0 and v1.6 clients can use servers of all versions back to v1.0. -This is the sixth release in the version 1 series. The version -1 series of Tahoe-LAFS will be actively supported and +In addition, version 1.6 improves forward-compatibility with +planned future cap formats, allowing updates to a directory +containing both current and future caps, without loss of +information. + +This is the seventh major release in the version 1 series. The +version 1 series of Tahoe-LAFS will be actively supported and maintained for the forseeable future, and future versions of Tahoe-LAFS will retain the ability to read and write files compatible with Tahoe-LAFS v1. diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 506771af..2c1f0dd8 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -23,6 +23,11 @@ from allmydata.uri import LiteralFileURI, from_string, wrap_dirnode_cap from pycryptopp.cipher.aes import AES from allmydata.util.dictutil import AuxValueDict + +# TODO: {Deleter,MetadataSetter,Adder}.modify all start by unpacking the +# contents and end by repacking them. It might be better to apply them to +# the unpacked contents. + class Deleter: def __init__(self, node, name, must_exist=True): self.node = node diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index b00a7f8f..ad7197ba 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -426,6 +426,7 @@ class IURI(Interface): """Return True if the data can be modified by *somebody* (perhaps someone who has a more powerful URI than this one).""" + # TODO: rename to get_read_cap() def get_readonly(): """Return another IURI instance, which represents a read-only form of this one. If is_readonly() is True, this returns self.""" diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index 7d46b6e0..416459b6 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -700,11 +700,15 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase): self.PUT, base, "") return d + # TODO: version of this with a Unicode filename def test_GET_FILEURL_save(self): - d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true") - # TODO: look at the headers, expect a Content-Disposition: attachment - # header. - d.addCallback(self.failUnlessIsBarDotTxt) + d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true", + return_response=True) + def _got((res, statuscode, headers)): + content_disposition = headers["content-disposition"][0] + self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition) + self.failUnlessIsBarDotTxt(res) + d.addCallback(_got) return d def test_GET_FILEURL_missing(self): @@ -2258,7 +2262,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase): # Fetch the welcome page. d = self.GET("/") def _after_get_welcome_page(res): - MKDIR_BUTTON_RE=re.compile('
', re.I) + MKDIR_BUTTON_RE = re.compile( + '' + '' + '', + re.I) mo = MKDIR_BUTTON_RE.search(res) formaction = mo.group(1) formt = mo.group(2) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index b68acb08..a569b6fc 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -11,9 +11,16 @@ from allmydata.interfaces import IURI, IDirnodeURI, IFileURI, IImmutableFileURI, class BadURIError(CapConstraintError): pass -# the URI shall be an ascii representation of the file. It shall contain -# enough information to retrieve and validate the contents. It shall be -# expressed in a limited character set (namely [TODO]). +# The URI shall be an ASCII representation of a reference to the file/directory. +# It shall contain enough information to retrieve and validate the contents. +# It shall be expressed in a limited character set (currently base32 plus ':' and +# capital letters, but future URIs might use a larger charset). + +# TODO: +# - rename all of the *URI classes/interfaces to *Cap +# - make variable and method names consistently use _uri for an URI string, +# and _cap for a Cap object (decoded URI) +# - remove the human_encoding methods? BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits) BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits) @@ -22,30 +29,35 @@ SEP='(?::|%3A)' NUMBER='([0-9]+)' NUMBER_IGNORE='(?:[0-9]+)' -# URIs (soon to be renamed "caps") are always allowed to come with a leading +# "human-encoded" URIs are allowed to come with a leading # 'http://127.0.0.1:(8123|3456)/uri/' that will be ignored. +# Note that nothing in the Tahoe code currently uses the human encoding. OPTIONALHTTPLEAD=r'(?:https?://(?:[^:/]+)(?::%s)?/uri/)?' % NUMBER_IGNORE class _BaseURI: def __hash__(self): return self.to_string().__hash__() + def __eq__(self, them): if isinstance(them, _BaseURI): return self.to_string() == them.to_string() else: return False + def __ne__(self, them): if isinstance(them, _BaseURI): return self.to_string() != them.to_string() else: return True + def to_human_encoding(self): return 'http://127.0.0.1:3456/uri/'+self.to_string() def get_storage_index(self): return self.storage_index + class CHKFileURI(_BaseURI): implements(IURI, IImmutableFileURI)