]> git.rkrishnan.org Git - tahoe-lafs/tahoe-lafs.git/blobdiff - src/allmydata/test/test_sftp.py
SFTP: add a comment about a subtle interaction between OverwriteableFileConsumer...
[tahoe-lafs/tahoe-lafs.git] / src / allmydata / test / test_sftp.py
index 0d546ebca59eb4bae22d175f3eec760038d46c53..a45d71d0740431dceb8a3e6237b34d3f616b18fc 100644 (file)
@@ -1,11 +1,12 @@
 
-import re, struct
+import re, struct, traceback, time, calendar
 from stat import S_IFREG, S_IFDIR
 
 from twisted.trial import unittest
-from twisted.internet import defer
+from twisted.internet import defer, reactor
 from twisted.python.failure import Failure
 from twisted.internet.error import ProcessDone, ProcessTerminated
+from allmydata.util import deferredutil
 
 conch_interfaces = None
 sftp = None
@@ -23,41 +24,20 @@ if have_pycrypto:
     from twisted.conch.ssh import filetransfer as sftp
     from allmydata.frontends import sftpd
 
-import traceback
-
-"""
-import sys
-def trace_exceptions(frame, event, arg):
-    if event != 'exception':
-        return
-    co = frame.f_code
-    func_name = co.co_name
-    line_no = frame.f_lineno
-    filename = co.co_filename
-    exc_type, exc_value, exc_traceback = arg
-    print 'Tracing exception: %r %r on line %r of %r in %r' % \
-        (exc_type.__name__, exc_value, line_no, func_name, filename)
-
-def trace_calls(frame, event, arg):
-    if event != 'call':
-        return
-    return trace_exceptions
-
-sys.settrace(trace_calls)
-"""
-
-timeout = 240
-
 from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError
 from allmydata.mutable.common import NotWriteableError
 
 from allmydata.util.consumer import download_to_data
 from allmydata.immutable import upload
+from allmydata.mutable import publish
 from allmydata.test.no_network import GridTestMixin
 from allmydata.test.common import ShouldFailMixin
+from allmydata.test.common_util import ReallyEqualMixin
+
+timeout = 240
 
-class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
-    """This is a no-network unit test of the SFTPHandler class."""
+class Handler(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, unittest.TestCase):
+    """This is a no-network unit test of the SFTPUserHandler and the abstractions it uses."""
 
     if not have_pycrypto:
         skip = "SFTP support requires pycrypto, which is not installed"
@@ -71,19 +51,15 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             if isinstance(res, Failure):
                 res.trap(sftp.SFTPError)
                 self.failUnlessReallyEqual(res.value.code, expected_code,
-                                           "%s was supposed to raise SFTPError(%d), not SFTPError(%d): %s" %
+                                           "%s was supposed to raise SFTPError(%r), not SFTPError(%r): %s" %
                                            (which, expected_code, res.value.code, res))
             else:
                 print '@' + '@'.join(s)
-                self.fail("%s was supposed to raise SFTPError(%d), not get '%s'" %
+                self.fail("%s was supposed to raise SFTPError(%r), not get %r" %
                           (which, expected_code, res))
         d.addBoth(_done)
         return d
 
-    def failUnlessReallyEqual(self, a, b, msg=None):
-        self.failUnlessEqual(a, b, msg=msg)
-        self.failUnlessEqual(type(a), type(b), msg=msg)
-
     def _set_up(self, basedir, num_clients=1, num_servers=10):
         self.basedir = "sftp/" + basedir
         self.set_up_grid(num_clients=num_clients, num_servers=num_servers)
@@ -95,12 +71,14 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         def _created_root(node):
             self.root = node
             self.root_uri = node.get_uri()
+            sftpd._reload()
             self.handler = sftpd.SFTPUserHandler(self.client, self.root, self.username)
         d.addCallback(_created_root)
         return d
 
     def _set_up_tree(self):
-        d = self.client.create_mutable_file("mutable file contents")
+        u = publish.MutableData("mutable file contents")
+        d = self.client.create_mutable_file(u)
         d.addCallback(lambda node: self.root.set_node(u"mutable", node))
         def _created_mutable(n):
             self.mutable = n
@@ -164,7 +142,9 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             self.root.set_node(u"unknown", n)
         d.addCallback(_created_unknown)
 
-        d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
+        fall_of_the_Berlin_wall = calendar.timegm(time.strptime("1989-11-09 20:00:00 UTC", "%Y-%m-%d %H:%M:%S %Z"))
+        md = {'mtime': fall_of_the_Berlin_wall, 'tahoe': {'linkmotime': fall_of_the_Berlin_wall}}
+        d.addCallback(lambda ign: self.root.set_node(u"loop", self.root, metadata=md))
         return d
 
     def test_basic(self):
@@ -189,6 +169,8 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/bar"), [u"foo", u"bar"])
             self.failUnlessReallyEqual(self.handler._path_from_string("foo/bar//"), [u"foo", u"bar"])
             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/bar//"), [u"foo", u"bar"])
+            self.failUnlessReallyEqual(self.handler._path_from_string("foo/./bar"), [u"foo", u"bar"])
+            self.failUnlessReallyEqual(self.handler._path_from_string("./foo/./bar"), [u"foo", u"bar"])
             self.failUnlessReallyEqual(self.handler._path_from_string("foo/../bar"), [u"bar"])
             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/../bar"), [u"bar"])
             self.failUnlessReallyEqual(self.handler._path_from_string("../bar"), [u"bar"])
@@ -208,6 +190,8 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             self.failUnlessReallyEqual(self.handler.realPath("/foo/bar"), "/foo/bar")
             self.failUnlessReallyEqual(self.handler.realPath("foo/bar//"), "/foo/bar")
             self.failUnlessReallyEqual(self.handler.realPath("/foo/bar//"), "/foo/bar")
+            self.failUnlessReallyEqual(self.handler.realPath("foo/./bar"), "/foo/bar")
+            self.failUnlessReallyEqual(self.handler.realPath("./foo/./bar"), "/foo/bar")
             self.failUnlessReallyEqual(self.handler.realPath("foo/../bar"), "/bar")
             self.failUnlessReallyEqual(self.handler.realPath("/foo/../bar"), "/bar")
             self.failUnlessReallyEqual(self.handler.realPath("../bar"), "/bar")
@@ -225,7 +209,7 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
 
     def test_convert_error(self):
         self.failUnlessReallyEqual(sftpd._convert_error(None, "request"), None)
-        
+
         d = defer.succeed(None)
         d.addCallback(lambda ign:
             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "_convert_error SFTPError",
@@ -278,11 +262,15 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
            self.failUnlessReallyEqual(name, expected_name)
            self.failUnless(re.match(expected_text_re, text),
                            "%r does not match %r in\n%r" % (text, expected_text_re, actual_list))
-           # it is ok for there to be extra actual attributes
-           # TODO: check times
-           for e in expected_attrs:
-               self.failUnlessReallyEqual(attrs[e], expected_attrs[e],
-                                          "%r:%r is not %r in\n%r" % (e, attrs[e], expected_attrs[e], attrs))
+           self._compareAttributes(attrs, expected_attrs)
+
+    def _compareAttributes(self, attrs, expected_attrs):
+        # It is ok for there to be extra actual attributes.
+        # TODO: check times
+        for e in expected_attrs:
+            self.failUnless(e in attrs, "%r is not in\n%r" % (e, attrs))
+            self.failUnlessReallyEqual(attrs[e], expected_attrs[e],
+                                       "%r:%r is not %r in\n%r" % (e, attrs[e], expected_attrs[e], attrs))
 
     def test_openDirectory_and_attrs(self):
         d = self._set_up("openDirectory_and_attrs")
@@ -303,15 +291,17 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
 
         gross = u"gro\u00DF".encode("utf-8")
         expected_root = [
-            ('empty_lit_dir', r'drwxrwxrwx .* \? .* empty_lit_dir$', {'permissions': S_IFDIR | 0777}),
-            (gross,           r'-rw-rw-rw- .* 1010 .* '+gross+'$',   {'permissions': S_IFREG | 0666, 'size': 1010}),
-            ('loop',          r'drwxrwxrwx .* \? .* loop$',          {'permissions': S_IFDIR | 0777}),
-            ('mutable',       r'-rw-rw-rw- .* \? .* mutable$',       {'permissions': S_IFREG | 0666}),
-            ('readonly',      r'-r--r--r-- .* \? .* readonly$',      {'permissions': S_IFREG | 0444}),
-            ('small',         r'-rw-rw-rw- .* 10 .* small$',         {'permissions': S_IFREG | 0666, 'size': 10}),
-            ('small2',        r'-rw-rw-rw- .* 26 .* small2$',        {'permissions': S_IFREG | 0666, 'size': 26}),
-            ('tiny_lit_dir',  r'drwxrwxrwx .* \? .* tiny_lit_dir$',  {'permissions': S_IFDIR | 0777}),
-            ('unknown',       r'\?--------- .* \? .* unknown$',      {'permissions': 0}),
+            ('empty_lit_dir', r'dr-xr-xr-x .* 0 .* empty_lit_dir$',       {'permissions': S_IFDIR | 0555}),
+            (gross,           r'-rw-rw-rw- .* 1010 .* '+gross+'$',        {'permissions': S_IFREG | 0666, 'size': 1010}),
+            # The fall of the Berlin wall may have been on 9th or 10th November 1989 depending on the gateway's timezone.
+            #('loop',          r'drwxrwxrwx .* 0 Nov (09|10)  1989 loop$', {'permissions': S_IFDIR | 0777}),
+            ('loop',          r'drwxrwxrwx .* 0 .* loop$',                {'permissions': S_IFDIR | 0777}),
+            ('mutable',       r'-rw-rw-rw- .* 0 .* mutable$',             {'permissions': S_IFREG | 0666}),
+            ('readonly',      r'-r--r--r-- .* 0 .* readonly$',            {'permissions': S_IFREG | 0444}),
+            ('small',         r'-rw-rw-rw- .* 10 .* small$',              {'permissions': S_IFREG | 0666, 'size': 10}),
+            ('small2',        r'-rw-rw-rw- .* 26 .* small2$',             {'permissions': S_IFREG | 0666, 'size': 26}),
+            ('tiny_lit_dir',  r'dr-xr-xr-x .* 0 .* tiny_lit_dir$',        {'permissions': S_IFDIR | 0555}),
+            ('unknown',       r'\?--------- .* 0 .* unknown$',            {'permissions': 0}),
         ]
 
         d.addCallback(lambda ign: self.handler.openDirectory(""))
@@ -335,18 +325,20 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda res: self._compareDirLists(res, expected_tiny_lit))
 
         d.addCallback(lambda ign: self.handler.getAttrs("small", True))
-        def _check_attrs(attrs):
-            self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666)
-            self.failUnlessReallyEqual(attrs['size'], 10, repr(attrs))
-        d.addCallback(_check_attrs)
+        d.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 10}))
 
         d.addCallback(lambda ign: self.handler.setAttrs("small", {}))
         d.addCallback(lambda res: self.failUnlessReallyEqual(res, None))
 
+        d.addCallback(lambda ign: self.handler.getAttrs("small", True))
+        d.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 10}))
+
         d.addCallback(lambda ign:
             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "setAttrs size",
                                          self.handler.setAttrs, "small", {'size': 0}))
 
+        d.addCallback(lambda ign: self.failUnlessEqual(sftpd.all_heisenfiles, {}))
+        d.addCallback(lambda ign: self.failUnlessEqual(self.handler._heisenfiles, {}))
         return d
 
     def test_openFile_read(self):
@@ -354,20 +346,23 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda ign: self._set_up_tree())
 
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small 0",
+            self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small 0 bad",
                                          self.handler.openFile, "small", 0, {}))
 
         # attempting to open a non-existent file should fail
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nofile READ",
+            self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nofile READ nosuch",
                                          self.handler.openFile, "nofile", sftp.FXF_READ, {}))
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nodir/file READ",
+            self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nodir/file READ nosuch",
                                          self.handler.openFile, "nodir/file", sftp.FXF_READ, {}))
 
         d.addCallback(lambda ign:
             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown READ denied",
                                          self.handler.openFile, "unknown", sftp.FXF_READ, {}))
+        d.addCallback(lambda ign:
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown/file READ denied",
+                                         self.handler.openFile, "unknown/file", sftp.FXF_READ, {}))
         d.addCallback(lambda ign:
             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir READ denied",
                                          self.handler.openFile, "tiny_lit_dir", sftp.FXF_READ, {}))
@@ -411,13 +406,10 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
                                              rf.readChunk, 11, 1))
 
             d2.addCallback(lambda ign: rf.getAttrs())
-            def _check_attrs(attrs):
-                self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666)
-                self.failUnlessReallyEqual(attrs['size'], 10, repr(attrs))
-            d2.addCallback(_check_attrs)
+            d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 10}))
 
             d2.addCallback(lambda ign: self.handler.getAttrs("small", followLinks=0))
-            d2.addCallback(_check_attrs)
+            d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 10}))
 
             d2.addCallback(lambda ign:
                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "writeChunk on read-only handle denied",
@@ -429,10 +421,10 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             d2.addCallback(lambda ign: rf.close())
 
             d2.addCallback(lambda ign:
-                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "readChunk on closed file",
+                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "readChunk on closed file bad",
                                              rf.readChunk, 0, 1))
             d2.addCallback(lambda ign:
-                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "getAttrs on closed file",
+                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "getAttrs on closed file bad",
                                              rf.getAttrs))
 
             d2.addCallback(lambda ign: rf.close()) # should be no-op
@@ -466,13 +458,10 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
                                              rf.readChunk, 1011, 1))
 
             d2.addCallback(lambda ign: rf.getAttrs())
-            def _check_attrs(attrs):
-                self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666)
-                self.failUnlessReallyEqual(attrs['size'], 1010)
-            d2.addCallback(_check_attrs)
+            d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 1010}))
 
             d2.addCallback(lambda ign: self.handler.getAttrs(gross, followLinks=0))
-            d2.addCallback(_check_attrs)
+            d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 1010}))
 
             d2.addCallback(lambda ign:
                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "writeChunk on read-only handle denied",
@@ -530,6 +519,8 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             return d2
         d.addCallback(_read_short)
 
+        d.addCallback(lambda ign: self.failUnlessEqual(sftpd.all_heisenfiles, {}))
+        d.addCallback(lambda ign: self.failUnlessEqual(self.handler._heisenfiles, {}))
         return d
 
     def test_openFile_write(self):
@@ -538,88 +529,94 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
 
         # '' is an invalid filename
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile '' WRITE|CREAT|TRUNC",
+            self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile '' WRITE|CREAT|TRUNC nosuch",
                                          self.handler.openFile, "", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 
         # TRUNC is not valid without CREAT if the file does not already exist
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile newfile WRITE|TRUNC",
+            self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile newfile WRITE|TRUNC nosuch",
                                          self.handler.openFile, "newfile", sftp.FXF_WRITE | sftp.FXF_TRUNC, {}))
 
         # EXCL is not valid without CREAT
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small WRITE|EXCL",
+            self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small WRITE|EXCL bad",
                                          self.handler.openFile, "small", sftp.FXF_WRITE | sftp.FXF_EXCL, {}))
 
         # cannot write to an existing directory
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir WRITE",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir WRITE denied",
                                          self.handler.openFile, "tiny_lit_dir", sftp.FXF_WRITE, {}))
 
         # cannot write to an existing unknown
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown WRITE",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown WRITE denied",
                                          self.handler.openFile, "unknown", sftp.FXF_WRITE, {}))
 
+        # cannot create a child of an unknown
+        d.addCallback(lambda ign:
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown/newfile WRITE|CREAT denied",
+                                         self.handler.openFile, "unknown/newfile",
+                                         sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
+
         # cannot write to a new file in an immutable directory
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/newfile WRITE|CREAT|TRUNC",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/newfile WRITE|CREAT|TRUNC denied",
                                          self.handler.openFile, "tiny_lit_dir/newfile",
                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 
         # cannot write to an existing immutable file in an immutable directory (with or without CREAT and EXCL)
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE denied",
                                          self.handler.openFile, "tiny_lit_dir/short", sftp.FXF_WRITE, {}))
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE|CREAT",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE|CREAT denied",
                                          self.handler.openFile, "tiny_lit_dir/short",
                                          sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
 
         # cannot write to a mutable file via a readonly cap (by path or uri)
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly WRITE",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly WRITE denied",
                                          self.handler.openFile, "readonly", sftp.FXF_WRITE, {}))
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly uri WRITE",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly uri WRITE denied",
                                          self.handler.openFile, "uri/"+self.readonly_uri, sftp.FXF_WRITE, {}))
 
         # cannot create a file with the EXCL flag if it already exists
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile small WRITE|CREAT|EXCL",
+            self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile small WRITE|CREAT|EXCL failure",
                                          self.handler.openFile, "small",
                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile mutable WRITE|CREAT|EXCL",
+            self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile mutable WRITE|CREAT|EXCL failure",
                                          self.handler.openFile, "mutable",
                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile mutable uri WRITE|CREAT|EXCL",
+            self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile mutable uri WRITE|CREAT|EXCL failure",
                                          self.handler.openFile, "uri/"+self.mutable_uri,
                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile tiny_lit_dir/short WRITE|CREAT|EXCL",
+            self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile tiny_lit_dir/short WRITE|CREAT|EXCL failure",
                                          self.handler.openFile, "tiny_lit_dir/short",
                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 
         # cannot write to an immutable file if we don't have its parent (with or without CREAT, TRUNC, or EXCL)
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE denied",
                                          self.handler.openFile, "uri/"+self.small_uri, sftp.FXF_WRITE, {}))
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT denied",
                                          self.handler.openFile, "uri/"+self.small_uri,
                                          sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT|TRUNC",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT|TRUNC denied",
                                          self.handler.openFile, "uri/"+self.small_uri,
                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
         d.addCallback(lambda ign:
-            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT|EXCL",
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT|EXCL denied",
                                          self.handler.openFile, "uri/"+self.small_uri,
                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 
-        # test creating a new file with truncation
+        # test creating a new file with truncation and extension
         d.addCallback(lambda ign:
                       self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
         def _write(wf):
@@ -630,26 +627,31 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             d2.addCallback(lambda ign: wf.writeChunk(13, "abc"))
 
             d2.addCallback(lambda ign: wf.getAttrs())
-            def _check_attrs(attrs, expected_size):
-                self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666, repr(attrs))
-                self.failUnlessReallyEqual(attrs['size'], expected_size)
-            d2.addCallback(_check_attrs, 16)
+            d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 16}))
 
-            # The file does not actually exist as a Tahoe file at this point, but getAttrs should
-            # use the all_open_files dict to see that it has been opened for creation.
             d2.addCallback(lambda ign: self.handler.getAttrs("newfile", followLinks=0))
-            d2.addCallback(_check_attrs, 0)
+            d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 16}))
 
             d2.addCallback(lambda ign: wf.setAttrs({}))
 
             d2.addCallback(lambda ign:
-                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs with negative size",
+                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs with negative size bad",
                                              wf.setAttrs, {'size': -1}))
 
             d2.addCallback(lambda ign: wf.setAttrs({'size': 14}))
             d2.addCallback(lambda ign: wf.getAttrs())
             d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['size'], 14))
 
+            d2.addCallback(lambda ign: wf.setAttrs({'size': 14}))
+            d2.addCallback(lambda ign: wf.getAttrs())
+            d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['size'], 14))
+
+            d2.addCallback(lambda ign: wf.setAttrs({'size': 17}))
+            d2.addCallback(lambda ign: wf.getAttrs())
+            d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['size'], 17))
+            d2.addCallback(lambda ign: self.handler.getAttrs("newfile", followLinks=0))
+            d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['size'], 17))
+
             d2.addCallback(lambda ign:
                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "readChunk on write-only handle denied",
                                              wf.readChunk, 0, 1))
@@ -657,10 +659,10 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             d2.addCallback(lambda ign: wf.close())
 
             d2.addCallback(lambda ign:
-                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "writeChunk on closed file",
+                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "writeChunk on closed file bad",
                                              wf.writeChunk, 0, "a"))
             d2.addCallback(lambda ign:
-                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs on closed file",
+                self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs on closed file bad",
                                              wf.setAttrs, {'size': 0}))
 
             d2.addCallback(lambda ign: wf.close()) # should be no-op
@@ -668,7 +670,7 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(_write)
         d.addCallback(lambda ign: self.root.get(u"newfile"))
         d.addCallback(lambda node: download_to_data(node))
-        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123\x00a"))
+        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123\x00a\x00\x00\x00"))
 
         # test APPEND flag, and also replacing an existing file ("newfile" created by the previous test)
         d.addCallback(lambda ign:
@@ -677,12 +679,18 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         def _write_append(wf):
             d2 = wf.writeChunk(0, "0123456789")
             d2.addCallback(lambda ign: wf.writeChunk(8, "0123"))
+
+            d2.addCallback(lambda ign: wf.setAttrs({'size': 17}))
+            d2.addCallback(lambda ign: wf.getAttrs())
+            d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['size'], 17))
+
+            d2.addCallback(lambda ign: wf.writeChunk(0, "z"))
             d2.addCallback(lambda ign: wf.close())
             return d2
         d.addCallback(_write_append)
         d.addCallback(lambda ign: self.root.get(u"newfile"))
         d.addCallback(lambda node: download_to_data(node))
-        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234567890123"))
+        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234567890123\x00\x00\x00z"))
 
         # test WRITE | TRUNC without CREAT, when the file already exists
         # This is invalid according to section 6.3 of the SFTP spec, but required for interoperability,
@@ -698,6 +706,16 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda node: download_to_data(node))
         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234"))
 
+        # test WRITE | TRUNC with permissions: 0
+        d.addCallback(lambda ign:
+                      self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_TRUNC, {'permissions': 0}))
+        d.addCallback(_write_trunc)
+        d.addCallback(lambda ign: self.root.get(u"newfile"))
+        d.addCallback(lambda node: download_to_data(node))
+        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234"))
+        d.addCallback(lambda ign: self.root.get_metadata_for(u"newfile"))
+        d.addCallback(lambda metadata: self.failIf(metadata.get('no-write', False), metadata))
+
         # test EXCL flag
         d.addCallback(lambda ign:
                       self.handler.openFile("excl", sftp.FXF_WRITE | sftp.FXF_CREAT |
@@ -810,6 +828,8 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda node: download_to_data(node))
         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "abcde56789"))
 
+        d.addCallback(lambda ign: self.root.set_node(u"mutable2", self.mutable))
+
         # test writing to a mutable file
         d.addCallback(lambda ign:
                       self.handler.openFile("mutable", sftp.FXF_WRITE, {}))
@@ -821,16 +841,81 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.root.get(u"mutable"))
         def _check_same_file(node):
             self.failUnless(node.is_mutable())
+            self.failIf(node.is_readonly())
             self.failUnlessReallyEqual(node.get_uri(), self.mutable_uri)
             return node.download_best_version()
         d.addCallback(_check_same_file)
         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "mutable new! contents"))
 
+        # ... and with permissions, which should be ignored
+        d.addCallback(lambda ign:
+                      self.handler.openFile("mutable", sftp.FXF_WRITE, {'permissions': 0}))
+        d.addCallback(_write_mutable)
+        d.addCallback(lambda ign: self.root.get(u"mutable"))
+        d.addCallback(_check_same_file)
+        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "mutable new! contents"))
+
+        # ... and with a setAttrs call that diminishes the parent link to read-only, first by path
+        d.addCallback(lambda ign:
+                      self.handler.openFile("mutable", sftp.FXF_WRITE, {}))
+        def _write_mutable_setattr(wf):
+            d2 = wf.writeChunk(8, "read-only link from parent")
+
+            d2.addCallback(lambda ign: self.handler.setAttrs("mutable", {'permissions': 0444}))
+
+            d2.addCallback(lambda ign: self.root.get(u"mutable"))
+            d2.addCallback(lambda node: self.failUnless(node.is_readonly()))
+
+            d2.addCallback(lambda ign: wf.getAttrs())
+            d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666))
+            d2.addCallback(lambda ign: self.handler.getAttrs("mutable", followLinks=0))
+            d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0444))
+
+            d2.addCallback(lambda ign: wf.close())
+            return d2
+        d.addCallback(_write_mutable_setattr)
+        d.addCallback(lambda ign: self.root.get(u"mutable"))
+        def _check_readonly_file(node):
+            self.failUnless(node.is_mutable())
+            self.failUnless(node.is_readonly())
+            self.failUnlessReallyEqual(node.get_write_uri(), None)
+            self.failUnlessReallyEqual(node.get_storage_index(), self.mutable.get_storage_index())
+            return node.download_best_version()
+        d.addCallback(_check_readonly_file)
+        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "mutable read-only link from parent"))
+
+        # ... and then by handle
+        d.addCallback(lambda ign:
+                      self.handler.openFile("mutable2", sftp.FXF_WRITE, {}))
+        def _write_mutable2_setattr(wf):
+            d2 = wf.writeChunk(7, "2")
+
+            d2.addCallback(lambda ign: wf.setAttrs({'permissions': 0444, 'size': 8}))
+
+            # The link isn't made read-only until the file is closed.
+            d2.addCallback(lambda ign: self.root.get(u"mutable2"))
+            d2.addCallback(lambda node: self.failIf(node.is_readonly()))
+
+            d2.addCallback(lambda ign: wf.getAttrs())
+            d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0444))
+            d2.addCallback(lambda ign: self.handler.getAttrs("mutable2", followLinks=0))
+            d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666))
+
+            d2.addCallback(lambda ign: wf.close())
+            return d2
+        d.addCallback(_write_mutable2_setattr)
+        d.addCallback(lambda ign: self.root.get(u"mutable2"))
+        d.addCallback(_check_readonly_file)  # from above
+        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "mutable2"))
+
         # test READ | WRITE without CREAT or TRUNC
         d.addCallback(lambda ign:
                       self.handler.openFile("small", sftp.FXF_READ | sftp.FXF_WRITE, {}))
         def _read_write(rwf):
             d2 = rwf.writeChunk(8, "0123")
+            # test immediate read starting after the old end-of-file
+            d2.addCallback(lambda ign: rwf.readChunk(11, 1))
+            d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "3"))
             d2.addCallback(lambda ign: rwf.readChunk(0, 100))
             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123"))
             d2.addCallback(lambda ign: rwf.close())
@@ -840,6 +925,65 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda node: download_to_data(node))
         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123"))
 
+        # test WRITE and rename while still open
+        d.addCallback(lambda ign:
+                      self.handler.openFile("small", sftp.FXF_WRITE, {}))
+        def _write_rename(wf):
+            d2 = wf.writeChunk(0, "abcd")
+            d2.addCallback(lambda ign: self.handler.renameFile("small", "renamed"))
+            d2.addCallback(lambda ign: wf.writeChunk(4, "efgh"))
+            d2.addCallback(lambda ign: wf.close())
+            return d2
+        d.addCallback(_write_rename)
+        d.addCallback(lambda ign: self.root.get(u"renamed"))
+        d.addCallback(lambda node: download_to_data(node))
+        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "abcdefgh0123"))
+        d.addCallback(lambda ign:
+                      self.shouldFail(NoSuchChildError, "rename small while open", "small",
+                                      self.root.get, u"small"))
+
+        # test WRITE | CREAT | EXCL and rename while still open
+        d.addCallback(lambda ign:
+                      self.handler.openFile("newexcl", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
+        def _write_creat_excl_rename(wf):
+            d2 = wf.writeChunk(0, "abcd")
+            d2.addCallback(lambda ign: self.handler.renameFile("newexcl", "renamedexcl"))
+            d2.addCallback(lambda ign: wf.writeChunk(4, "efgh"))
+            d2.addCallback(lambda ign: wf.close())
+            return d2
+        d.addCallback(_write_creat_excl_rename)
+        d.addCallback(lambda ign: self.root.get(u"renamedexcl"))
+        d.addCallback(lambda node: download_to_data(node))
+        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "abcdefgh"))
+        d.addCallback(lambda ign:
+                      self.shouldFail(NoSuchChildError, "rename newexcl while open", "newexcl",
+                                      self.root.get, u"newexcl"))
+
+        # it should be possible to rename even before the open has completed
+        def _open_and_rename_race(ign):
+            slow_open = defer.Deferred()
+            reactor.callLater(1, slow_open.callback, None)
+            d2 = self.handler.openFile("new", sftp.FXF_WRITE | sftp.FXF_CREAT, {}, delay=slow_open)
+
+            # deliberate race between openFile and renameFile
+            d3 = self.handler.renameFile("new", "new2")
+            d3.addErrback(lambda err: self.fail("renameFile failed: %r" % (err,)))
+            return d2
+        d.addCallback(_open_and_rename_race)
+        def _write_rename_race(wf):
+            d2 = wf.writeChunk(0, "abcd")
+            d2.addCallback(lambda ign: wf.close())
+            return d2
+        d.addCallback(_write_rename_race)
+        d.addCallback(lambda ign: self.root.get(u"new2"))
+        d.addCallback(lambda node: download_to_data(node))
+        d.addCallback(lambda data: self.failUnlessReallyEqual(data, "abcd"))
+        d.addCallback(lambda ign:
+                      self.shouldFail(NoSuchChildError, "rename new while open", "new",
+                                      self.root.get, u"new"))
+
+        d.addCallback(lambda ign: self.failUnlessEqual(sftpd.all_heisenfiles, {}))
+        d.addCallback(lambda ign: self.failUnlessEqual(self.handler._heisenfiles, {}))
         return d
 
     def test_removeFile(self):
@@ -858,7 +1002,7 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda ign:
             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removefile ''",
                                          self.handler.removeFile, ""))
-            
+
         # removing a directory should fail
         d.addCallback(lambda ign:
             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "removeFile tiny_lit_dir",
@@ -881,7 +1025,7 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         # removing a link to an open file should not prevent it from being read
         d.addCallback(lambda ign: self.handler.openFile("small", sftp.FXF_READ, {}))
         def _remove_and_read_small(rf):
-            d2= self.handler.removeFile("small")
+            d2 = self.handler.removeFile("small")
             d2.addCallback(lambda ign:
                            self.shouldFail(NoSuchChildError, "removeFile small", "small",
                                            self.root.get, u"small"))
@@ -891,6 +1035,42 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             return d2
         d.addCallback(_remove_and_read_small)
 
+        # removing a link to a created file should prevent it from being created
+        d.addCallback(lambda ign: self.handler.openFile("tempfile", sftp.FXF_READ | sftp.FXF_WRITE |
+                                                                    sftp.FXF_CREAT, {}))
+        def _write_remove(rwf):
+            d2 = rwf.writeChunk(0, "0123456789")
+            d2.addCallback(lambda ign: self.handler.removeFile("tempfile"))
+            d2.addCallback(lambda ign: rwf.readChunk(0, 10))
+            d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
+            d2.addCallback(lambda ign: rwf.close())
+            return d2
+        d.addCallback(_write_remove)
+        d.addCallback(lambda ign:
+                      self.shouldFail(NoSuchChildError, "removeFile tempfile", "tempfile",
+                                      self.root.get, u"tempfile"))
+
+        # ... even if the link is renamed while open
+        d.addCallback(lambda ign: self.handler.openFile("tempfile2", sftp.FXF_READ | sftp.FXF_WRITE |
+                                                                     sftp.FXF_CREAT, {}))
+        def _write_rename_remove(rwf):
+            d2 = rwf.writeChunk(0, "0123456789")
+            d2.addCallback(lambda ign: self.handler.renameFile("tempfile2", "tempfile3"))
+            d2.addCallback(lambda ign: self.handler.removeFile("tempfile3"))
+            d2.addCallback(lambda ign: rwf.readChunk(0, 10))
+            d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
+            d2.addCallback(lambda ign: rwf.close())
+            return d2
+        d.addCallback(_write_rename_remove)
+        d.addCallback(lambda ign:
+                      self.shouldFail(NoSuchChildError, "removeFile tempfile2", "tempfile2",
+                                      self.root.get, u"tempfile2"))
+        d.addCallback(lambda ign:
+                      self.shouldFail(NoSuchChildError, "removeFile tempfile3", "tempfile3",
+                                      self.root.get, u"tempfile3"))
+
+        d.addCallback(lambda ign: self.failUnlessEqual(sftpd.all_heisenfiles, {}))
+        d.addCallback(lambda ign: self.failUnlessEqual(self.handler._heisenfiles, {}))
         return d
 
     def test_removeDirectory(self):
@@ -926,6 +1106,8 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
                       self.shouldFail(NoSuchChildError, "removeDirectory unknown", "unknown",
                                       self.root.get, u"unknown"))
 
+        d.addCallback(lambda ign: self.failUnlessEqual(sftpd.all_heisenfiles, {}))
+        d.addCallback(lambda ign: self.failUnlessEqual(self.handler._heisenfiles, {}))
         return d
 
     def test_renameFile(self):
@@ -971,6 +1153,20 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small unknown",
                                          self.handler.renameFile, "small", "unknown"))
 
+        # renaming a file onto a heisenfile should fail, even if the open hasn't completed
+        def _rename_onto_heisenfile_race(wf):
+            slow_open = defer.Deferred()
+            reactor.callLater(1, slow_open.callback, None)
+
+            d2 = self.handler.openFile("heisenfile", sftp.FXF_WRITE | sftp.FXF_CREAT, {}, delay=slow_open)
+
+            # deliberate race between openFile and renameFile
+            d3 = self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small heisenfile",
+                                              self.handler.renameFile, "small", "heisenfile")
+            d2.addCallback(lambda wf: wf.close())
+            return deferredutil.gatherResults([d2, d3])
+        d.addCallback(_rename_onto_heisenfile_race)
+
         # renaming a file to a correct path should succeed
         d.addCallback(lambda ign: self.handler.renameFile("small", "new_small"))
         d.addCallback(lambda ign: self.root.get(u"new_small"))
@@ -992,13 +1188,25 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.root.get(u"new_unknown"))
         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.unknown_uri))
 
+        d.addCallback(lambda ign: self.failUnlessEqual(sftpd.all_heisenfiles, {}))
+        d.addCallback(lambda ign: self.failUnlessEqual(self.handler._heisenfiles, {}))
         return d
 
     def test_renameFile_posix(self):
         def _renameFile(fromPathstring, toPathstring):
             extData = (struct.pack('>L', len(fromPathstring)) + fromPathstring +
                        struct.pack('>L', len(toPathstring))   + toPathstring)
-            return self.handler.extendedRequest('extposix-rename@openssh.com', extData)
+
+            d2 = self.handler.extendedRequest('posix-rename@openssh.com', extData)
+            def _check(res):
+                res.trap(sftp.SFTPError)
+                if res.value.code == sftp.FX_OK:
+                    return None
+                return res
+            d2.addCallbacks(lambda res: self.fail("posix-rename request was supposed to "
+                                                  "raise an SFTPError, not get '%r'" % (res,)),
+                            _check)
+            return d2
 
         d = self._set_up("renameFile_posix")
         d.addCallback(lambda ign: self._set_up_tree())
@@ -1066,12 +1274,14 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.root.get(u"new_unknown"))
         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.unknown_uri))
 
+        d.addCallback(lambda ign: self.failUnlessEqual(sftpd.all_heisenfiles, {}))
+        d.addCallback(lambda ign: self.failUnlessEqual(self.handler._heisenfiles, {}))
         return d
 
     def test_makeDirectory(self):
         d = self._set_up("makeDirectory")
         d.addCallback(lambda ign: self._set_up_tree())
-            
+
         # making a directory at a correct path should succeed
         d.addCallback(lambda ign: self.handler.makeDirectory("newdir", {'ext_foo': 'bar', 'ctime': 42}))
 
@@ -1110,54 +1320,84 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
         d.addCallback(lambda ign:
             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "makeDirectory small",
                                          self.handler.makeDirectory, "small", {}))
+
+        # directories cannot be created read-only via SFTP
+        d.addCallback(lambda ign:
+            self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "makeDirectory newdir2 permissions:0444 denied",
+                                         self.handler.makeDirectory, "newdir2",
+                                         {'permissions': 0444}))
+
+        d.addCallback(lambda ign: self.failUnlessEqual(sftpd.all_heisenfiles, {}))
+        d.addCallback(lambda ign: self.failUnlessEqual(self.handler._heisenfiles, {}))
         return d
 
     def test_execCommand_and_openShell(self):
-        class FakeProtocol:
+        class MockProtocol:
             def __init__(self):
                 self.output = ""
+                self.error = ""
                 self.reason = None
+
             def write(self, data):
+                return self.outReceived(data)
+
+            def outReceived(self, data):
                 self.output += data
                 return defer.succeed(None)
+
+            def errReceived(self, data):
+                self.error += data
+                return defer.succeed(None)
+
             def processEnded(self, reason):
                 self.reason = reason
                 return defer.succeed(None)
 
+        def _lines_end_in_crlf(s):
+            return s.replace('\r\n', '').find('\n') == -1 and s.endswith('\r\n')
+
         d = self._set_up("execCommand_and_openShell")
 
         d.addCallback(lambda ign: conch_interfaces.ISession(self.handler))
         def _exec_df(session):
-            protocol = FakeProtocol()
+            protocol = MockProtocol()
             d2 = session.execCommand(protocol, "df -P -k /")
             d2.addCallback(lambda ign: self.failUnlessIn("1024-blocks", protocol.output))
+            d2.addCallback(lambda ign: self.failUnless(_lines_end_in_crlf(protocol.output), protocol.output))
+            d2.addCallback(lambda ign: self.failUnlessEqual(protocol.error, ""))
             d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessDone)))
             d2.addCallback(lambda ign: session.eofReceived())
             d2.addCallback(lambda ign: session.closed())
             return d2
         d.addCallback(_exec_df)
 
+        def _check_unsupported(protocol):
+            d2 = defer.succeed(None)
+            d2.addCallback(lambda ign: self.failUnlessEqual(protocol.output, ""))
+            d2.addCallback(lambda ign: self.failUnlessIn("only the SFTP protocol", protocol.error))
+            d2.addCallback(lambda ign: self.failUnless(_lines_end_in_crlf(protocol.error), protocol.error))
+            d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessTerminated)))
+            d2.addCallback(lambda ign: self.failUnlessEqual(protocol.reason.value.exitCode, 1))
+            return d2
+
         d.addCallback(lambda ign: conch_interfaces.ISession(self.handler))
         def _exec_error(session):
-            protocol = FakeProtocol()
+            protocol = MockProtocol()
             d2 = session.execCommand(protocol, "error")
-            d2.addCallback(lambda ign: self.failUnlessEqual("", protocol.output))
-            d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessTerminated)))
-            d2.addCallback(lambda ign: self.failUnlessEqual(protocol.reason.value.exitCode, 1))
+            d2.addCallback(lambda ign: session.windowChanged(None))
+            d2.addCallback(lambda ign: _check_unsupported(protocol))
             d2.addCallback(lambda ign: session.closed())
             return d2
         d.addCallback(_exec_error)
 
         d.addCallback(lambda ign: conch_interfaces.ISession(self.handler))
         def _openShell(session):
-            protocol = FakeProtocol()
+            protocol = MockProtocol()
             d2 = session.openShell(protocol)
-            d2.addCallback(lambda ign: self.failUnlessIn("only SFTP", protocol.output))
-            d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessTerminated)))
-            d2.addCallback(lambda ign: self.failUnlessEqual(protocol.reason.value.exitCode, 1))
+            d2.addCallback(lambda ign: _check_unsupported(protocol))
             d2.addCallback(lambda ign: session.closed())
             return d2
-        d.addCallback(_exec_error)
+        d.addCallback(_openShell)
 
         return d
 
@@ -1174,4 +1414,14 @@ class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "extendedRequest foo bar",
                                          self.handler.extendedRequest, "foo", "bar"))
 
+        d.addCallback(lambda ign:
+            self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "extendedRequest posix-rename@openssh.com invalid 1",
+                                         self.handler.extendedRequest, 'posix-rename@openssh.com', ''))
+        d.addCallback(lambda ign:
+            self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "extendedRequest posix-rename@openssh.com invalid 2",
+                                         self.handler.extendedRequest, 'posix-rename@openssh.com', '\x00\x00\x00\x01'))
+        d.addCallback(lambda ign:
+            self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "extendedRequest posix-rename@openssh.com invalid 3",
+                                         self.handler.extendedRequest, 'posix-rename@openssh.com', '\x00\x00\x00\x01_\x00\x00\x00\x01'))
+
         return d