src/allmydata/test/test_sftp.py

   1
   2 import re, struct
   3 from stat import S_IFREG, S_IFDIR
   4
   5 from twisted.trial import unittest
   6 from twisted.internet import defer
   7 from twisted.python.failure import Failure
   8 from twisted.internet.error import ProcessDone, ProcessTerminated
   9
  10 conch_interfaces = None
  11 sftp = None
  12 sftpd = None
  13 have_pycrypto = False
  14 try:
  15     from Crypto import Util
  16     Util  # hush pyflakes
  17     have_pycrypto = True
  18 except ImportError:
  19     pass
  20
  21 if have_pycrypto:
  22     from twisted.conch import interfaces as conch_interfaces
  23     from twisted.conch.ssh import filetransfer as sftp
  24     from allmydata.frontends import sftpd
  25
  26 import traceback
  27
  28 """
  29 import sys
  30 def trace_exceptions(frame, event, arg):
  31     if event != 'exception':
  32         return
  33     co = frame.f_code
  34     func_name = co.co_name
  35     line_no = frame.f_lineno
  36     filename = co.co_filename
  37     exc_type, exc_value, exc_traceback = arg
  38     print 'Tracing exception: %r %r on line %r of %r in %r' % \
  39         (exc_type.__name__, exc_value, line_no, func_name, filename)
  40
  41 def trace_calls(frame, event, arg):
  42     if event != 'call':
  43         return
  44     return trace_exceptions
  45
  46 sys.settrace(trace_calls)
  47 """
  48
  49 timeout = 60
  50
  51 from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError
  52 from allmydata.mutable.common import NotWriteableError
  53
  54 from allmydata.util.consumer import download_to_data
  55 from allmydata.immutable import upload
  56 from allmydata.test.no_network import GridTestMixin
  57 from allmydata.test.common import ShouldFailMixin
  58
  59 class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
  60     """This is a no-network unit test of the SFTPHandler class."""
  61
  62     if not have_pycrypto:
  63         skip = "SFTP support requires pycrypto, which is not installed"
  64
  65     def shouldFailWithSFTPError(self, expected_code, which, callable, *args, **kwargs):
  66         assert isinstance(expected_code, int), repr(expected_code)
  67         assert isinstance(which, str), repr(which)
  68         s = traceback.format_stack()
  69         d = defer.maybeDeferred(callable, *args, **kwargs)
  70         def _done(res):
  71             if isinstance(res, Failure):
  72                 res.trap(sftp.SFTPError)
  73                 self.failUnlessReallyEqual(res.value.code, expected_code,
  74                                            "%s was supposed to raise SFTPError(%d), not SFTPError(%d): %s" %
  75                                            (which, expected_code, res.value.code, res))
  76             else:
  77                 print '@' + '@'.join(s)
  78                 self.fail("%s was supposed to raise SFTPError(%d), not get '%s'" %
  79                           (which, expected_code, res))
  80         d.addBoth(_done)
  81         return d
  82
  83     def failUnlessReallyEqual(self, a, b, msg=None):
  84         self.failUnlessEqual(a, b, msg=msg)
  85         self.failUnlessEqual(type(a), type(b), msg=msg)
  86
  87     def _set_up(self, basedir, num_clients=1, num_servers=10):
  88         self.basedir = "sftp/" + basedir
  89         self.set_up_grid(num_clients=num_clients, num_servers=num_servers)
  90
  91         self.client = self.g.clients[0]
  92         self.username = "alice"
  93
  94         d = self.client.create_dirnode()
  95         def _created_root(node):
  96             self.root = node
  97             self.root_uri = node.get_uri()
  98             self.handler = sftpd.SFTPUserHandler(self.client, self.root, self.username)
  99         d.addCallback(_created_root)
 100         return d
 101
 102     def _set_up_tree(self):
 103         d = self.client.create_mutable_file("mutable file contents")
 104         d.addCallback(lambda node: self.root.set_node(u"mutable", node))
 105         def _created_mutable(n):
 106             self.mutable = n
 107             self.mutable_uri = n.get_uri()
 108         d.addCallback(_created_mutable)
 109
 110         d.addCallback(lambda ign:
 111                       self.root._create_and_validate_node(None, self.mutable.get_readonly_uri(), name=u"readonly"))
 112         d.addCallback(lambda node: self.root.set_node(u"readonly", node))
 113         def _created_readonly(n):
 114             self.readonly = n
 115             self.readonly_uri = n.get_uri()
 116         d.addCallback(_created_readonly)
 117
 118         gross = upload.Data("0123456789" * 101, None)
 119         d.addCallback(lambda ign: self.root.add_file(u"gro\u00DF", gross))
 120         def _created_gross(n):
 121             self.gross = n
 122             self.gross_uri = n.get_uri()
 123         d.addCallback(_created_gross)
 124
 125         small = upload.Data("0123456789", None)
 126         d.addCallback(lambda ign: self.root.add_file(u"small", small))
 127         def _created_small(n):
 128             self.small = n
 129             self.small_uri = n.get_uri()
 130         d.addCallback(_created_small)
 131
 132         small2 = upload.Data("Small enough for a LIT too", None)
 133         d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
 134         def _created_small2(n):
 135             self.small2 = n
 136             self.small2_uri = n.get_uri()
 137         d.addCallback(_created_small2)
 138
 139         empty_litdir_uri = "URI:DIR2-LIT:"
 140
 141         # contains one child which is itself also LIT:
 142         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm"
 143
 144         unknown_uri = "x-tahoe-crazy://I_am_from_the_future."
 145
 146         d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"empty_lit_dir"))
 147         def _created_empty_lit_dir(n):
 148             self.empty_lit_dir = n
 149             self.empty_lit_dir_uri = n.get_uri()
 150             self.root.set_node(u"empty_lit_dir", n)
 151         d.addCallback(_created_empty_lit_dir)
 152
 153         d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"tiny_lit_dir"))
 154         def _created_tiny_lit_dir(n):
 155             self.tiny_lit_dir = n
 156             self.tiny_lit_dir_uri = n.get_uri()
 157             self.root.set_node(u"tiny_lit_dir", n)
 158         d.addCallback(_created_tiny_lit_dir)
 159
 160         d.addCallback(lambda ign: self.root._create_and_validate_node(None, unknown_uri, name=u"unknown"))
 161         def _created_unknown(n):
 162             self.unknown = n
 163             self.unknown_uri = n.get_uri()
 164             self.root.set_node(u"unknown", n)
 165         d.addCallback(_created_unknown)
 166
 167         d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
 168         return d
 169
 170     def test_basic(self):
 171         d = self._set_up("basic")
 172         def _check(ign):
 173             # Test operations that have no side-effects, and don't need the tree.
 174
 175             version = self.handler.gotVersion(3, {})
 176             self.failUnless(isinstance(version, dict))
 177
 178             self.failUnlessReallyEqual(self.handler._path_from_string(""), [])
 179             self.failUnlessReallyEqual(self.handler._path_from_string("/"), [])
 180             self.failUnlessReallyEqual(self.handler._path_from_string("."), [])
 181             self.failUnlessReallyEqual(self.handler._path_from_string("//"), [])
 182             self.failUnlessReallyEqual(self.handler._path_from_string("/."), [])
 183             self.failUnlessReallyEqual(self.handler._path_from_string("/./"), [])
 184             self.failUnlessReallyEqual(self.handler._path_from_string("foo"), [u"foo"])
 185             self.failUnlessReallyEqual(self.handler._path_from_string("/foo"), [u"foo"])
 186             self.failUnlessReallyEqual(self.handler._path_from_string("foo/"), [u"foo"])
 187             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/"), [u"foo"])
 188             self.failUnlessReallyEqual(self.handler._path_from_string("foo/bar"), [u"foo", u"bar"])
 189             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/bar"), [u"foo", u"bar"])
 190             self.failUnlessReallyEqual(self.handler._path_from_string("foo/bar//"), [u"foo", u"bar"])
 191             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/bar//"), [u"foo", u"bar"])
 192             self.failUnlessReallyEqual(self.handler._path_from_string("foo/../bar"), [u"bar"])
 193             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/../bar"), [u"bar"])
 194             self.failUnlessReallyEqual(self.handler._path_from_string("../bar"), [u"bar"])
 195             self.failUnlessReallyEqual(self.handler._path_from_string("/../bar"), [u"bar"])
 196
 197             self.failUnlessReallyEqual(self.handler.realPath(""), "/")
 198             self.failUnlessReallyEqual(self.handler.realPath("/"), "/")
 199             self.failUnlessReallyEqual(self.handler.realPath("."), "/")
 200             self.failUnlessReallyEqual(self.handler.realPath("//"), "/")
 201             self.failUnlessReallyEqual(self.handler.realPath("/."), "/")
 202             self.failUnlessReallyEqual(self.handler.realPath("/./"), "/")
 203             self.failUnlessReallyEqual(self.handler.realPath("foo"), "/foo")
 204             self.failUnlessReallyEqual(self.handler.realPath("/foo"), "/foo")
 205             self.failUnlessReallyEqual(self.handler.realPath("foo/"), "/foo")
 206             self.failUnlessReallyEqual(self.handler.realPath("/foo/"), "/foo")
 207             self.failUnlessReallyEqual(self.handler.realPath("foo/bar"), "/foo/bar")
 208             self.failUnlessReallyEqual(self.handler.realPath("/foo/bar"), "/foo/bar")
 209             self.failUnlessReallyEqual(self.handler.realPath("foo/bar//"), "/foo/bar")
 210             self.failUnlessReallyEqual(self.handler.realPath("/foo/bar//"), "/foo/bar")
 211             self.failUnlessReallyEqual(self.handler.realPath("foo/../bar"), "/bar")
 212             self.failUnlessReallyEqual(self.handler.realPath("/foo/../bar"), "/bar")
 213             self.failUnlessReallyEqual(self.handler.realPath("../bar"), "/bar")
 214             self.failUnlessReallyEqual(self.handler.realPath("/../bar"), "/bar")
 215         d.addCallback(_check)
 216
 217         d.addCallback(lambda ign:
 218             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "_path_from_string invalid UTF-8",
 219                                          self.handler._path_from_string, "\xFF"))
 220         d.addCallback(lambda ign:
 221             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "realPath invalid UTF-8",
 222                                          self.handler.realPath, "\xFF"))
 223
 224         return d
 225
 226     def test_convert_error(self):
 227         self.failUnlessReallyEqual(sftpd._convert_error(None, "request"), None)
 228
 229         d = defer.succeed(None)
 230         d.addCallback(lambda ign:
 231             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "_convert_error SFTPError",
 232                                          sftpd._convert_error, Failure(sftp.SFTPError(sftp.FX_FAILURE, "foo")), "request"))
 233         d.addCallback(lambda ign:
 234             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "_convert_error NoSuchChildError",
 235                                          sftpd._convert_error, Failure(NoSuchChildError("foo")), "request"))
 236         d.addCallback(lambda ign:
 237             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "_convert_error ExistingChildError",
 238                                          sftpd._convert_error, Failure(ExistingChildError("foo")), "request"))
 239         d.addCallback(lambda ign:
 240             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "_convert_error NotWriteableError",
 241                                          sftpd._convert_error, Failure(NotWriteableError("foo")), "request"))
 242         d.addCallback(lambda ign:
 243             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "_convert_error NotImplementedError",
 244                                          sftpd._convert_error, Failure(NotImplementedError("foo")), "request"))
 245         d.addCallback(lambda ign:
 246             self.shouldFailWithSFTPError(sftp.FX_EOF, "_convert_error EOFError",
 247                                          sftpd._convert_error, Failure(EOFError("foo")), "request"))
 248         d.addCallback(lambda ign:
 249             self.shouldFailWithSFTPError(sftp.FX_EOF, "_convert_error defer.FirstError",
 250                                          sftpd._convert_error, Failure(defer.FirstError(
 251                                                                  Failure(sftp.SFTPError(sftp.FX_EOF, "foo")), 0)), "request"))
 252         d.addCallback(lambda ign:
 253             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "_convert_error AssertionError",
 254                                          sftpd._convert_error, Failure(AssertionError("foo")), "request"))
 255
 256         return d
 257
 258     def test_not_implemented(self):
 259         d = self._set_up("not_implemented")
 260
 261         d.addCallback(lambda ign:
 262             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "readLink link",
 263                                          self.handler.readLink, "link"))
 264         d.addCallback(lambda ign:
 265             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "makeLink link file",
 266                                          self.handler.makeLink, "link", "file"))
 267
 268         return d
 269
 270     def _compareDirLists(self, actual, expected):
 271        actual_list = sorted(actual)
 272        expected_list = sorted(expected)
 273        self.failUnlessReallyEqual(len(actual_list), len(expected_list),
 274                             "%r is wrong length, expecting %r" % (actual_list, expected_list))
 275        for (a, b) in zip(actual_list, expected_list):
 276            (name, text, attrs) = a
 277            (expected_name, expected_text_re, expected_attrs) = b
 278            self.failUnlessReallyEqual(name, expected_name)
 279            self.failUnless(re.match(expected_text_re, text),
 280                            "%r does not match %r in\n%r" % (text, expected_text_re, actual_list))
 281            # it is ok for there to be extra actual attributes
 282            # TODO: check times
 283            for e in expected_attrs:
 284                self.failUnlessReallyEqual(attrs[e], expected_attrs[e],
 285                                           "%r:%r is not %r in\n%r" % (e, attrs[e], expected_attrs[e], attrs))
 286
 287     def test_openDirectory_and_attrs(self):
 288         d = self._set_up("openDirectory_and_attrs")
 289         d.addCallback(lambda ign: self._set_up_tree())
 290
 291         d.addCallback(lambda ign:
 292             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openDirectory small",
 293                                          self.handler.openDirectory, "small"))
 294         d.addCallback(lambda ign:
 295             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openDirectory unknown",
 296                                          self.handler.openDirectory, "unknown"))
 297         d.addCallback(lambda ign:
 298             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openDirectory nodir",
 299                                          self.handler.openDirectory, "nodir"))
 300         d.addCallback(lambda ign:
 301             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openDirectory nodir/nodir",
 302                                          self.handler.openDirectory, "nodir/nodir"))
 303
 304         gross = u"gro\u00DF".encode("utf-8")
 305         expected_root = [
 306             ('empty_lit_dir', r'drwxrwxrwx .* \? .* empty_lit_dir$', {'permissions': S_IFDIR | 0777}),
 307             (gross,           r'-rw-rw-rw- .* 1010 .* '+gross+'$',   {'permissions': S_IFREG | 0666, 'size': 1010}),
 308             ('loop',          r'drwxrwxrwx .* \? .* loop$',          {'permissions': S_IFDIR | 0777}),
 309             ('mutable',       r'-rw-rw-rw- .* \? .* mutable$',       {'permissions': S_IFREG | 0666}),
 310             ('readonly',      r'-r--r--r-- .* \? .* readonly$',      {'permissions': S_IFREG | 0444}),
 311             ('small',         r'-rw-rw-rw- .* 10 .* small$',         {'permissions': S_IFREG | 0666, 'size': 10}),
 312             ('small2',        r'-rw-rw-rw- .* 26 .* small2$',        {'permissions': S_IFREG | 0666, 'size': 26}),
 313             ('tiny_lit_dir',  r'drwxrwxrwx .* \? .* tiny_lit_dir$',  {'permissions': S_IFDIR | 0777}),
 314             ('unknown',       r'\?--------- .* \? .* unknown$',      {'permissions': 0}),
 315         ]
 316
 317         d.addCallback(lambda ign: self.handler.openDirectory(""))
 318         d.addCallback(lambda res: self._compareDirLists(res, expected_root))
 319
 320         d.addCallback(lambda ign: self.handler.openDirectory("loop"))
 321         d.addCallback(lambda res: self._compareDirLists(res, expected_root))
 322
 323         d.addCallback(lambda ign: self.handler.openDirectory("loop/loop"))
 324         d.addCallback(lambda res: self._compareDirLists(res, expected_root))
 325
 326         d.addCallback(lambda ign: self.handler.openDirectory("empty_lit_dir"))
 327         d.addCallback(lambda res: self._compareDirLists(res, []))
 328
 329         # The UTC epoch may either be in Jan 1 1970 or Dec 31 1969 depending on the gateway's timezone.
 330         expected_tiny_lit = [
 331             ('short', r'-r--r--r-- .* 8 (Jan 01  1970|Dec 31  1969) short$', {'permissions': S_IFREG | 0444, 'size': 8}),
 332         ]
 333
 334         d.addCallback(lambda ign: self.handler.openDirectory("tiny_lit_dir"))
 335         d.addCallback(lambda res: self._compareDirLists(res, expected_tiny_lit))
 336
 337         d.addCallback(lambda ign: self.handler.getAttrs("small", True))
 338         def _check_attrs(attrs):
 339             self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666)
 340             self.failUnlessReallyEqual(attrs['size'], 10, repr(attrs))
 341         d.addCallback(_check_attrs)
 342
 343         d.addCallback(lambda ign: self.handler.setAttrs("small", {}))
 344         d.addCallback(lambda res: self.failUnlessReallyEqual(res, None))
 345
 346         d.addCallback(lambda ign:
 347             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "setAttrs size",
 348                                          self.handler.setAttrs, "small", {'size': 0}))
 349
 350         return d
 351
 352     def test_openFile_read(self):
 353         d = self._set_up("openFile_read")
 354         d.addCallback(lambda ign: self._set_up_tree())
 355
 356         d.addCallback(lambda ign:
 357             self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small 0",
 358                                          self.handler.openFile, "small", 0, {}))
 359
 360         # attempting to open a non-existent file should fail
 361         d.addCallback(lambda ign:
 362             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nofile READ",
 363                                          self.handler.openFile, "nofile", sftp.FXF_READ, {}))
 364         d.addCallback(lambda ign:
 365             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nodir/file READ",
 366                                          self.handler.openFile, "nodir/file", sftp.FXF_READ, {}))
 367
 368         d.addCallback(lambda ign:
 369             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown READ denied",
 370                                          self.handler.openFile, "unknown", sftp.FXF_READ, {}))
 371         d.addCallback(lambda ign:
 372             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir READ denied",
 373                                          self.handler.openFile, "tiny_lit_dir", sftp.FXF_READ, {}))
 374         d.addCallback(lambda ign:
 375             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown uri READ denied",
 376                                          self.handler.openFile, "uri/"+self.unknown_uri, sftp.FXF_READ, {}))
 377         d.addCallback(lambda ign:
 378             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir uri READ denied",
 379                                          self.handler.openFile, "uri/"+self.tiny_lit_dir_uri, sftp.FXF_READ, {}))
 380         # FIXME: should be FX_NO_SUCH_FILE?
 381         d.addCallback(lambda ign:
 382             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile noexist uri READ denied",
 383                                          self.handler.openFile, "uri/URI:noexist", sftp.FXF_READ, {}))
 384         d.addCallback(lambda ign:
 385             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile invalid UTF-8 uri READ denied",
 386                                          self.handler.openFile, "uri/URI:\xFF", sftp.FXF_READ, {}))
 387
 388         # reading an existing file should succeed
 389         d.addCallback(lambda ign: self.handler.openFile("small", sftp.FXF_READ, {}))
 390         def _read_small(rf):
 391             d2 = rf.readChunk(0, 10)
 392             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 393
 394             d2.addCallback(lambda ign: rf.readChunk(2, 6))
 395             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "234567"))
 396
 397             d2.addCallback(lambda ign: rf.readChunk(1, 0))
 398             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 399
 400             d2.addCallback(lambda ign: rf.readChunk(8, 4))  # read that starts before EOF is OK
 401             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "89"))
 402
 403             d2.addCallback(lambda ign:
 404                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF (0-byte)",
 405                                              rf.readChunk, 10, 0))
 406             d2.addCallback(lambda ign:
 407                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF",
 408                                              rf.readChunk, 10, 1))
 409             d2.addCallback(lambda ign:
 410                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting after EOF",
 411                                              rf.readChunk, 11, 1))
 412
 413             d2.addCallback(lambda ign: rf.getAttrs())
 414             def _check_attrs(attrs):
 415                 self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666)
 416                 self.failUnlessReallyEqual(attrs['size'], 10, repr(attrs))
 417             d2.addCallback(_check_attrs)
 418
 419             d2.addCallback(lambda ign: self.handler.getAttrs("small", followLinks=0))
 420             d2.addCallback(_check_attrs)
 421
 422             d2.addCallback(lambda ign:
 423                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "writeChunk on read-only handle denied",
 424                                              rf.writeChunk, 0, "a"))
 425             d2.addCallback(lambda ign:
 426                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "setAttrs on read-only handle denied",
 427                                              rf.setAttrs, {}))
 428
 429             d2.addCallback(lambda ign: rf.close())
 430
 431             d2.addCallback(lambda ign:
 432                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "readChunk on closed file",
 433                                              rf.readChunk, 0, 1))
 434             d2.addCallback(lambda ign:
 435                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "getAttrs on closed file",
 436                                              rf.getAttrs))
 437
 438             d2.addCallback(lambda ign: rf.close()) # should be no-op
 439             return d2
 440         d.addCallback(_read_small)
 441
 442         # repeat for a large file
 443         gross = u"gro\u00DF".encode("utf-8")
 444         d.addCallback(lambda ign: self.handler.openFile(gross, sftp.FXF_READ, {}))
 445         def _read_gross(rf):
 446             d2 = rf.readChunk(0, 10)
 447             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 448
 449             d2.addCallback(lambda ign: rf.readChunk(2, 6))
 450             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "234567"))
 451
 452             d2.addCallback(lambda ign: rf.readChunk(1, 0))
 453             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 454
 455             d2.addCallback(lambda ign: rf.readChunk(1008, 4))  # read that starts before EOF is OK
 456             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "89"))
 457
 458             d2.addCallback(lambda ign:
 459                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF (0-byte)",
 460                                              rf.readChunk, 1010, 0))
 461             d2.addCallback(lambda ign:
 462                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF",
 463                                              rf.readChunk, 1010, 1))
 464             d2.addCallback(lambda ign:
 465                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting after EOF",
 466                                              rf.readChunk, 1011, 1))
 467
 468             d2.addCallback(lambda ign: rf.getAttrs())
 469             def _check_attrs(attrs):
 470                 self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666)
 471                 self.failUnlessReallyEqual(attrs['size'], 1010)
 472             d2.addCallback(_check_attrs)
 473
 474             d2.addCallback(lambda ign: self.handler.getAttrs(gross, followLinks=0))
 475             d2.addCallback(_check_attrs)
 476
 477             d2.addCallback(lambda ign:
 478                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "writeChunk on read-only handle denied",
 479                                              rf.writeChunk, 0, "a"))
 480             d2.addCallback(lambda ign:
 481                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "setAttrs on read-only handle denied",
 482                                              rf.setAttrs, {}))
 483
 484             d2.addCallback(lambda ign: rf.close())
 485
 486             d2.addCallback(lambda ign:
 487                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "readChunk on closed file",
 488                                              rf.readChunk, 0, 1))
 489             d2.addCallback(lambda ign:
 490                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "getAttrs on closed file",
 491                                              rf.getAttrs))
 492
 493             d2.addCallback(lambda ign: rf.close()) # should be no-op
 494             return d2
 495         d.addCallback(_read_gross)
 496
 497         # reading an existing small file via uri/ should succeed
 498         d.addCallback(lambda ign: self.handler.openFile("uri/"+self.small_uri, sftp.FXF_READ, {}))
 499         def _read_small_uri(rf):
 500             d2 = rf.readChunk(0, 10)
 501             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 502             d2.addCallback(lambda ign: rf.close())
 503             return d2
 504         d.addCallback(_read_small_uri)
 505
 506         # repeat for a large file
 507         d.addCallback(lambda ign: self.handler.openFile("uri/"+self.gross_uri, sftp.FXF_READ, {}))
 508         def _read_gross_uri(rf):
 509             d2 = rf.readChunk(0, 10)
 510             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 511             d2.addCallback(lambda ign: rf.close())
 512             return d2
 513         d.addCallback(_read_gross_uri)
 514
 515         # repeat for a mutable file
 516         d.addCallback(lambda ign: self.handler.openFile("uri/"+self.mutable_uri, sftp.FXF_READ, {}))
 517         def _read_mutable_uri(rf):
 518             d2 = rf.readChunk(0, 100)
 519             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "mutable file contents"))
 520             d2.addCallback(lambda ign: rf.close())
 521             return d2
 522         d.addCallback(_read_mutable_uri)
 523
 524         # repeat for a file within a directory referenced by URI
 525         d.addCallback(lambda ign: self.handler.openFile("uri/"+self.tiny_lit_dir_uri+"/short", sftp.FXF_READ, {}))
 526         def _read_short(rf):
 527             d2 = rf.readChunk(0, 100)
 528             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "The end."))
 529             d2.addCallback(lambda ign: rf.close())
 530             return d2
 531         d.addCallback(_read_short)
 532
 533         return d
 534
 535     def test_openFile_write(self):
 536         d = self._set_up("openFile_write")
 537         d.addCallback(lambda ign: self._set_up_tree())
 538
 539         # '' is an invalid filename
 540         d.addCallback(lambda ign:
 541             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile '' WRITE|CREAT|TRUNC",
 542                                          self.handler.openFile, "", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 543
 544         # TRUNC is not valid without CREAT if the file does not already exist
 545         d.addCallback(lambda ign:
 546             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile newfile WRITE|TRUNC",
 547                                          self.handler.openFile, "newfile", sftp.FXF_WRITE | sftp.FXF_TRUNC, {}))
 548
 549         # EXCL is not valid without CREAT
 550         d.addCallback(lambda ign:
 551             self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small WRITE|EXCL",
 552                                          self.handler.openFile, "small", sftp.FXF_WRITE | sftp.FXF_EXCL, {}))
 553
 554         # cannot write to an existing directory
 555         d.addCallback(lambda ign:
 556             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir WRITE",
 557                                          self.handler.openFile, "tiny_lit_dir", sftp.FXF_WRITE, {}))
 558
 559         # cannot write to an existing unknown
 560         d.addCallback(lambda ign:
 561             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown WRITE",
 562                                          self.handler.openFile, "unknown", sftp.FXF_WRITE, {}))
 563
 564         # cannot write to a new file in an immutable directory
 565         d.addCallback(lambda ign:
 566             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/newfile WRITE|CREAT|TRUNC",
 567                                          self.handler.openFile, "tiny_lit_dir/newfile",
 568                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 569
 570         # cannot write to an existing immutable file in an immutable directory (with or without CREAT and EXCL)
 571         d.addCallback(lambda ign:
 572             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE",
 573                                          self.handler.openFile, "tiny_lit_dir/short", sftp.FXF_WRITE, {}))
 574         d.addCallback(lambda ign:
 575             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE|CREAT",
 576                                          self.handler.openFile, "tiny_lit_dir/short",
 577                                          sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
 578
 579         # cannot write to a mutable file via a readonly cap (by path or uri)
 580         d.addCallback(lambda ign:
 581             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly WRITE",
 582                                          self.handler.openFile, "readonly", sftp.FXF_WRITE, {}))
 583         d.addCallback(lambda ign:
 584             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly uri WRITE",
 585                                          self.handler.openFile, "uri/"+self.readonly_uri, sftp.FXF_WRITE, {}))
 586
 587         # cannot create a file with the EXCL flag if it already exists
 588         d.addCallback(lambda ign:
 589             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile small WRITE|CREAT|EXCL",
 590                                          self.handler.openFile, "small",
 591                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 592         d.addCallback(lambda ign:
 593             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile mutable WRITE|CREAT|EXCL",
 594                                          self.handler.openFile, "mutable",
 595                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 596         d.addCallback(lambda ign:
 597             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile mutable uri WRITE|CREAT|EXCL",
 598                                          self.handler.openFile, "uri/"+self.mutable_uri,
 599                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 600         d.addCallback(lambda ign:
 601             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile tiny_lit_dir/short WRITE|CREAT|EXCL",
 602                                          self.handler.openFile, "tiny_lit_dir/short",
 603                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 604
 605         # cannot write to an immutable file if we don't have its parent (with or without CREAT, TRUNC, or EXCL)
 606         d.addCallback(lambda ign:
 607             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE",
 608                                          self.handler.openFile, "uri/"+self.small_uri, sftp.FXF_WRITE, {}))
 609         d.addCallback(lambda ign:
 610             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT",
 611                                          self.handler.openFile, "uri/"+self.small_uri,
 612                                          sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
 613         d.addCallback(lambda ign:
 614             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT|TRUNC",
 615                                          self.handler.openFile, "uri/"+self.small_uri,
 616                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 617         d.addCallback(lambda ign:
 618             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT|EXCL",
 619                                          self.handler.openFile, "uri/"+self.small_uri,
 620                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 621
 622         # test creating a new file with truncation
 623         d.addCallback(lambda ign:
 624                       self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 625         def _write(wf):
 626             d2 = wf.writeChunk(0, "0123456789")
 627             d2.addCallback(lambda res: self.failUnlessReallyEqual(res, None))
 628
 629             d2.addCallback(lambda ign: wf.writeChunk(8, "0123"))
 630             d2.addCallback(lambda ign: wf.writeChunk(13, "abc"))
 631
 632             d2.addCallback(lambda ign: wf.getAttrs())
 633             def _check_attrs(attrs, expected_size):
 634                 self.failUnlessReallyEqual(attrs['permissions'], S_IFREG | 0666, repr(attrs))
 635                 self.failUnlessReallyEqual(attrs['size'], expected_size)
 636             d2.addCallback(_check_attrs, 16)
 637
 638             # The file does not actually exist as a Tahoe file at this point, but getAttrs should
 639             # use the all_open_files dict to see that it has been opened for creation.
 640             d2.addCallback(lambda ign: self.handler.getAttrs("newfile", followLinks=0))
 641             d2.addCallback(_check_attrs, 0)
 642
 643             d2.addCallback(lambda ign: wf.setAttrs({}))
 644
 645             d2.addCallback(lambda ign:
 646                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs with negative size",
 647                                              wf.setAttrs, {'size': -1}))
 648
 649             d2.addCallback(lambda ign: wf.setAttrs({'size': 14}))
 650             d2.addCallback(lambda ign: wf.getAttrs())
 651             d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['size'], 14))
 652
 653             d2.addCallback(lambda ign:
 654                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "readChunk on write-only handle denied",
 655                                              wf.readChunk, 0, 1))
 656
 657             d2.addCallback(lambda ign: wf.close())
 658
 659             d2.addCallback(lambda ign:
 660                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "writeChunk on closed file",
 661                                              wf.writeChunk, 0, "a"))
 662             d2.addCallback(lambda ign:
 663                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs on closed file",
 664                                              wf.setAttrs, {'size': 0}))
 665
 666             d2.addCallback(lambda ign: wf.close()) # should be no-op
 667             return d2
 668         d.addCallback(_write)
 669         d.addCallback(lambda ign: self.root.get(u"newfile"))
 670         d.addCallback(lambda node: download_to_data(node))
 671         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123\x00a"))
 672
 673         # test APPEND flag, and also replacing an existing file ("newfile" created by the previous test)
 674         d.addCallback(lambda ign:
 675                       self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_CREAT |
 676                                                        sftp.FXF_TRUNC | sftp.FXF_APPEND, {}))
 677         def _write_append(wf):
 678             d2 = wf.writeChunk(0, "0123456789")
 679             d2.addCallback(lambda ign: wf.writeChunk(8, "0123"))
 680             d2.addCallback(lambda ign: wf.close())
 681             return d2
 682         d.addCallback(_write_append)
 683         d.addCallback(lambda ign: self.root.get(u"newfile"))
 684         d.addCallback(lambda node: download_to_data(node))
 685         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234567890123"))
 686
 687         # test WRITE | TRUNC without CREAT, when the file already exists
 688         # This is invalid according to section 6.3 of the SFTP spec, but required for interoperability,
 689         # since POSIX does allow O_WRONLY | O_TRUNC.
 690         d.addCallback(lambda ign:
 691                       self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_TRUNC, {}))
 692         def _write_trunc(wf):
 693             d2 = wf.writeChunk(0, "01234")
 694             d2.addCallback(lambda ign: wf.close())
 695             return d2
 696         d.addCallback(_write_trunc)
 697         d.addCallback(lambda ign: self.root.get(u"newfile"))
 698         d.addCallback(lambda node: download_to_data(node))
 699         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234"))
 700
 701         # test EXCL flag
 702         d.addCallback(lambda ign:
 703                       self.handler.openFile("excl", sftp.FXF_WRITE | sftp.FXF_CREAT |
 704                                                     sftp.FXF_TRUNC | sftp.FXF_EXCL, {}))
 705         def _write_excl(wf):
 706             d2 = self.root.get(u"excl")
 707             d2.addCallback(lambda node: download_to_data(node))
 708             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 709
 710             d2.addCallback(lambda ign: wf.writeChunk(0, "0123456789"))
 711             d2.addCallback(lambda ign: wf.close())
 712             return d2
 713         d.addCallback(_write_excl)
 714         d.addCallback(lambda ign: self.root.get(u"excl"))
 715         d.addCallback(lambda node: download_to_data(node))
 716         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 717
 718         # test that writing a zero-length file with EXCL only updates the directory once
 719         d.addCallback(lambda ign:
 720                       self.handler.openFile("zerolength", sftp.FXF_WRITE | sftp.FXF_CREAT |
 721                                                           sftp.FXF_EXCL, {}))
 722         def _write_excl_zerolength(wf):
 723             d2 = self.root.get(u"zerolength")
 724             d2.addCallback(lambda node: download_to_data(node))
 725             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 726
 727             # FIXME: no API to get the best version number exists (fix as part of #993)
 728             """
 729             d2.addCallback(lambda ign: self.root.get_best_version_number())
 730             def _check_version(version):
 731                 d3 = wf.close()
 732                 d3.addCallback(lambda ign: self.root.get_best_version_number())
 733                 d3.addCallback(lambda new_version: self.failUnlessReallyEqual(new_version, version))
 734                 return d3
 735             d2.addCallback(_check_version)
 736             """
 737             d2.addCallback(lambda ign: wf.close())
 738             return d2
 739         d.addCallback(_write_excl_zerolength)
 740         d.addCallback(lambda ign: self.root.get(u"zerolength"))
 741         d.addCallback(lambda node: download_to_data(node))
 742         d.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 743
 744         # test WRITE | CREAT | EXCL | APPEND
 745         d.addCallback(lambda ign:
 746                       self.handler.openFile("exclappend", sftp.FXF_WRITE | sftp.FXF_CREAT |
 747                                                           sftp.FXF_EXCL | sftp.FXF_APPEND, {}))
 748         def _write_excl_append(wf):
 749             d2 = self.root.get(u"exclappend")
 750             d2.addCallback(lambda node: download_to_data(node))
 751             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 752
 753             d2.addCallback(lambda ign: wf.writeChunk(10, "0123456789"))
 754             d2.addCallback(lambda ign: wf.writeChunk(5, "01234"))
 755             d2.addCallback(lambda ign: wf.close())
 756             return d2
 757         d.addCallback(_write_excl_append)
 758         d.addCallback(lambda ign: self.root.get(u"exclappend"))
 759         d.addCallback(lambda node: download_to_data(node))
 760         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345678901234"))
 761
 762         # test WRITE | CREAT | APPEND when the file does not already exist
 763         d.addCallback(lambda ign:
 764                       self.handler.openFile("creatappend", sftp.FXF_WRITE | sftp.FXF_CREAT |
 765                                                            sftp.FXF_APPEND, {}))
 766         def _write_creat_append_new(wf):
 767             d2 = wf.writeChunk(10, "0123456789")
 768             d2.addCallback(lambda ign: wf.writeChunk(5, "01234"))
 769             d2.addCallback(lambda ign: wf.close())
 770             return d2
 771         d.addCallback(_write_creat_append_new)
 772         d.addCallback(lambda ign: self.root.get(u"creatappend"))
 773         d.addCallback(lambda node: download_to_data(node))
 774         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345678901234"))
 775
 776         # ... and when it does exist
 777         d.addCallback(lambda ign:
 778                       self.handler.openFile("creatappend", sftp.FXF_WRITE | sftp.FXF_CREAT |
 779                                                            sftp.FXF_APPEND, {}))
 780         def _write_creat_append_existing(wf):
 781             d2 = wf.writeChunk(5, "01234")
 782             d2.addCallback(lambda ign: wf.close())
 783             return d2
 784         d.addCallback(_write_creat_append_existing)
 785         d.addCallback(lambda ign: self.root.get(u"creatappend"))
 786         d.addCallback(lambda node: download_to_data(node))
 787         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234567890123401234"))
 788
 789         # test WRITE | CREAT without TRUNC, when the file does not already exist
 790         d.addCallback(lambda ign:
 791                       self.handler.openFile("newfile2", sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
 792         def _write_creat_new(wf):
 793             d2 =  wf.writeChunk(0, "0123456789")
 794             d2.addCallback(lambda ign: wf.close())
 795             return d2
 796         d.addCallback(_write_creat_new)
 797         d.addCallback(lambda ign: self.root.get(u"newfile2"))
 798         d.addCallback(lambda node: download_to_data(node))
 799         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 800
 801         # ... and when it does exist
 802         d.addCallback(lambda ign:
 803                       self.handler.openFile("newfile2", sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
 804         def _write_creat_existing(wf):
 805             d2 =  wf.writeChunk(0, "abcde")
 806             d2.addCallback(lambda ign: wf.close())
 807             return d2
 808         d.addCallback(_write_creat_existing)
 809         d.addCallback(lambda ign: self.root.get(u"newfile2"))
 810         d.addCallback(lambda node: download_to_data(node))
 811         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "abcde56789"))
 812
 813         # test writing to a mutable file
 814         d.addCallback(lambda ign:
 815                       self.handler.openFile("mutable", sftp.FXF_WRITE, {}))
 816         def _write_mutable(wf):
 817             d2 = wf.writeChunk(8, "new!")
 818             d2.addCallback(lambda ign: wf.close())
 819             return d2
 820         d.addCallback(_write_mutable)
 821         d.addCallback(lambda ign: self.root.get(u"mutable"))
 822         def _check_same_file(node):
 823             self.failUnless(node.is_mutable())
 824             self.failUnlessReallyEqual(node.get_uri(), self.mutable_uri)
 825             return node.download_best_version()
 826         d.addCallback(_check_same_file)
 827         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "mutable new! contents"))
 828
 829         # test READ | WRITE without CREAT or TRUNC
 830         d.addCallback(lambda ign:
 831                       self.handler.openFile("small", sftp.FXF_READ | sftp.FXF_WRITE, {}))
 832         def _read_write(rwf):
 833             d2 = rwf.writeChunk(8, "0123")
 834             d2.addCallback(lambda ign: rwf.readChunk(0, 100))
 835             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123"))
 836             d2.addCallback(lambda ign: rwf.close())
 837             return d2
 838         d.addCallback(_read_write)
 839         d.addCallback(lambda ign: self.root.get(u"small"))
 840         d.addCallback(lambda node: download_to_data(node))
 841         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123"))
 842
 843         return d
 844
 845     def test_removeFile(self):
 846         d = self._set_up("removeFile")
 847         d.addCallback(lambda ign: self._set_up_tree())
 848
 849         d.addCallback(lambda ign:
 850             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeFile nofile",
 851                                          self.handler.removeFile, "nofile"))
 852         d.addCallback(lambda ign:
 853             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeFile nofile",
 854                                          self.handler.removeFile, "nofile"))
 855         d.addCallback(lambda ign:
 856             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeFile nodir/file",
 857                                          self.handler.removeFile, "nodir/file"))
 858         d.addCallback(lambda ign:
 859             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removefile ''",
 860                                          self.handler.removeFile, ""))
 861
 862         # removing a directory should fail
 863         d.addCallback(lambda ign:
 864             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "removeFile tiny_lit_dir",
 865                                          self.handler.removeFile, "tiny_lit_dir"))
 866
 867         # removing a file should succeed
 868         d.addCallback(lambda ign: self.root.get(u"gro\u00DF"))
 869         d.addCallback(lambda ign: self.handler.removeFile(u"gro\u00DF".encode('utf-8')))
 870         d.addCallback(lambda ign:
 871                       self.shouldFail(NoSuchChildError, "removeFile gross", "gro\\xdf",
 872                                       self.root.get, u"gro\u00DF"))
 873
 874         # removing an unknown should succeed
 875         d.addCallback(lambda ign: self.root.get(u"unknown"))
 876         d.addCallback(lambda ign: self.handler.removeFile("unknown"))
 877         d.addCallback(lambda ign:
 878                       self.shouldFail(NoSuchChildError, "removeFile unknown", "unknown",
 879                                       self.root.get, u"unknown"))
 880
 881         # removing a link to an open file should not prevent it from being read
 882         d.addCallback(lambda ign: self.handler.openFile("small", sftp.FXF_READ, {}))
 883         def _remove_and_read_small(rf):
 884             d2= self.handler.removeFile("small")
 885             d2.addCallback(lambda ign:
 886                            self.shouldFail(NoSuchChildError, "removeFile small", "small",
 887                                            self.root.get, u"small"))
 888             d2.addCallback(lambda ign: rf.readChunk(0, 10))
 889             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 890             d2.addCallback(lambda ign: rf.close())
 891             return d2
 892         d.addCallback(_remove_and_read_small)
 893
 894         return d
 895
 896     def test_removeDirectory(self):
 897         d = self._set_up("removeDirectory")
 898         d.addCallback(lambda ign: self._set_up_tree())
 899
 900         d.addCallback(lambda ign:
 901             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeDirectory nodir",
 902                                          self.handler.removeDirectory, "nodir"))
 903         d.addCallback(lambda ign:
 904             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeDirectory nodir/nodir",
 905                                          self.handler.removeDirectory, "nodir/nodir"))
 906         d.addCallback(lambda ign:
 907             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeDirectory ''",
 908                                          self.handler.removeDirectory, ""))
 909
 910         # removing a file should fail
 911         d.addCallback(lambda ign:
 912             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "removeDirectory gross",
 913                                          self.handler.removeDirectory, u"gro\u00DF".encode('utf-8')))
 914
 915         # removing a directory should succeed
 916         d.addCallback(lambda ign: self.root.get(u"tiny_lit_dir"))
 917         d.addCallback(lambda ign: self.handler.removeDirectory("tiny_lit_dir"))
 918         d.addCallback(lambda ign:
 919                       self.shouldFail(NoSuchChildError, "removeDirectory tiny_lit_dir", "tiny_lit_dir",
 920                                       self.root.get, u"tiny_lit_dir"))
 921
 922         # removing an unknown should succeed
 923         d.addCallback(lambda ign: self.root.get(u"unknown"))
 924         d.addCallback(lambda ign: self.handler.removeDirectory("unknown"))
 925         d.addCallback(lambda err:
 926                       self.shouldFail(NoSuchChildError, "removeDirectory unknown", "unknown",
 927                                       self.root.get, u"unknown"))
 928
 929         return d
 930
 931     def test_renameFile(self):
 932         d = self._set_up("renameFile")
 933         d.addCallback(lambda ign: self._set_up_tree())
 934
 935         # renaming a non-existent file should fail
 936         d.addCallback(lambda ign:
 937             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile nofile newfile",
 938                                          self.handler.renameFile, "nofile", "newfile"))
 939         d.addCallback(lambda ign:
 940             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile '' newfile",
 941                                          self.handler.renameFile, "", "newfile"))
 942
 943         # renaming a file to a non-existent path should fail
 944         d.addCallback(lambda ign:
 945             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small nodir/small",
 946                                          self.handler.renameFile, "small", "nodir/small"))
 947
 948         # renaming a file to an invalid UTF-8 name should fail
 949         d.addCallback(lambda ign:
 950             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small invalid",
 951                                          self.handler.renameFile, "small", "\xFF"))
 952
 953         # renaming a file to or from an URI should fail
 954         d.addCallback(lambda ign:
 955             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small from uri",
 956                                          self.handler.renameFile, "uri/"+self.small_uri, "new"))
 957         d.addCallback(lambda ign:
 958             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small to uri",
 959                                          self.handler.renameFile, "small", "uri/fake_uri"))
 960
 961         # renaming a file onto an existing file, directory or unknown should fail
 962         # The SFTP spec isn't clear about what error should be returned, but sshfs depends on
 963         # it being FX_PERMISSION_DENIED.
 964         d.addCallback(lambda ign:
 965             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small small2",
 966                                          self.handler.renameFile, "small", "small2"))
 967         d.addCallback(lambda ign:
 968             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small tiny_lit_dir",
 969                                          self.handler.renameFile, "small", "tiny_lit_dir"))
 970         d.addCallback(lambda ign:
 971             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small unknown",
 972                                          self.handler.renameFile, "small", "unknown"))
 973
 974         # renaming a file to a correct path should succeed
 975         d.addCallback(lambda ign: self.handler.renameFile("small", "new_small"))
 976         d.addCallback(lambda ign: self.root.get(u"new_small"))
 977         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
 978
 979         # renaming a file into a subdirectory should succeed (also tests Unicode names)
 980         d.addCallback(lambda ign: self.handler.renameFile(u"gro\u00DF".encode('utf-8'),
 981                                                           u"loop/neue_gro\u00DF".encode('utf-8')))
 982         d.addCallback(lambda ign: self.root.get(u"neue_gro\u00DF"))
 983         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.gross_uri))
 984
 985         # renaming a directory to a correct path should succeed
 986         d.addCallback(lambda ign: self.handler.renameFile("tiny_lit_dir", "new_tiny_lit_dir"))
 987         d.addCallback(lambda ign: self.root.get(u"new_tiny_lit_dir"))
 988         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.tiny_lit_dir_uri))
 989
 990         # renaming an unknown to a correct path should succeed
 991         d.addCallback(lambda ign: self.handler.renameFile("unknown", "new_unknown"))
 992         d.addCallback(lambda ign: self.root.get(u"new_unknown"))
 993         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.unknown_uri))
 994
 995         return d
 996
 997     def test_renameFile_posix(self):
 998         def _renameFile(fromPathstring, toPathstring):
 999             extData = (struct.pack('>L', len(fromPathstring)) + fromPathstring +
1000                        struct.pack('>L', len(toPathstring))   + toPathstring)
1001             return self.handler.extendedRequest('extposix-rename@openssh.com', extData)
1002
1003         d = self._set_up("renameFile_posix")
1004         d.addCallback(lambda ign: self._set_up_tree())
1005
1006         d.addCallback(lambda ign: self.root.set_node(u"loop2", self.root))
1007         d.addCallback(lambda ign: self.root.set_node(u"unknown2", self.unknown))
1008
1009         # POSIX-renaming a non-existent file should fail
1010         d.addCallback(lambda ign:
1011             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix nofile newfile",
1012                                          _renameFile, "nofile", "newfile"))
1013         d.addCallback(lambda ign:
1014             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix '' newfile",
1015                                          _renameFile, "", "newfile"))
1016
1017         # POSIX-renaming a file to a non-existent path should fail
1018         d.addCallback(lambda ign:
1019             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix small nodir/small",
1020                                          _renameFile, "small", "nodir/small"))
1021
1022         # POSIX-renaming a file to an invalid UTF-8 name should fail
1023         d.addCallback(lambda ign:
1024             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix small invalid",
1025                                          _renameFile, "small", "\xFF"))
1026
1027         # POSIX-renaming a file to or from an URI should fail
1028         d.addCallback(lambda ign:
1029             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix small from uri",
1030                                          _renameFile, "uri/"+self.small_uri, "new"))
1031         d.addCallback(lambda ign:
1032             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix small to uri",
1033                                          _renameFile, "small", "uri/fake_uri"))
1034
1035         # POSIX-renaming a file onto an existing file, directory or unknown should succeed
1036         d.addCallback(lambda ign: _renameFile("small", "small2"))
1037         d.addCallback(lambda ign: self.root.get(u"small2"))
1038         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
1039
1040         d.addCallback(lambda ign: _renameFile("small2", "loop2"))
1041         d.addCallback(lambda ign: self.root.get(u"loop2"))
1042         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
1043
1044         d.addCallback(lambda ign: _renameFile("loop2", "unknown2"))
1045         d.addCallback(lambda ign: self.root.get(u"unknown2"))
1046         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
1047
1048         # POSIX-renaming a file to a correct new path should succeed
1049         d.addCallback(lambda ign: _renameFile("unknown2", "new_small"))
1050         d.addCallback(lambda ign: self.root.get(u"new_small"))
1051         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
1052
1053         # POSIX-renaming a file into a subdirectory should succeed (also tests Unicode names)
1054         d.addCallback(lambda ign: _renameFile(u"gro\u00DF".encode('utf-8'),
1055                                               u"loop/neue_gro\u00DF".encode('utf-8')))
1056         d.addCallback(lambda ign: self.root.get(u"neue_gro\u00DF"))
1057         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.gross_uri))
1058
1059         # POSIX-renaming a directory to a correct path should succeed
1060         d.addCallback(lambda ign: _renameFile("tiny_lit_dir", "new_tiny_lit_dir"))
1061         d.addCallback(lambda ign: self.root.get(u"new_tiny_lit_dir"))
1062         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.tiny_lit_dir_uri))
1063
1064         # POSIX-renaming an unknown to a correct path should succeed
1065         d.addCallback(lambda ign: _renameFile("unknown", "new_unknown"))
1066         d.addCallback(lambda ign: self.root.get(u"new_unknown"))
1067         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.unknown_uri))
1068
1069         return d
1070
1071     def test_makeDirectory(self):
1072         d = self._set_up("makeDirectory")
1073         d.addCallback(lambda ign: self._set_up_tree())
1074
1075         # making a directory at a correct path should succeed
1076         d.addCallback(lambda ign: self.handler.makeDirectory("newdir", {'ext_foo': 'bar', 'ctime': 42}))
1077
1078         d.addCallback(lambda ign: self.root.get_child_and_metadata(u"newdir"))
1079         def _got( (child, metadata) ):
1080             self.failUnless(IDirectoryNode.providedBy(child))
1081             self.failUnless(child.is_mutable())
1082             # FIXME
1083             #self.failUnless('ctime' in metadata, metadata)
1084             #self.failUnlessReallyEqual(metadata['ctime'], 42)
1085             #self.failUnless('ext_foo' in metadata, metadata)
1086             #self.failUnlessReallyEqual(metadata['ext_foo'], 'bar')
1087             # TODO: child should be empty
1088         d.addCallback(_got)
1089
1090         # making intermediate directories should also succeed
1091         d.addCallback(lambda ign: self.handler.makeDirectory("newparent/newchild", {}))
1092
1093         d.addCallback(lambda ign: self.root.get(u"newparent"))
1094         def _got_newparent(newparent):
1095             self.failUnless(IDirectoryNode.providedBy(newparent))
1096             self.failUnless(newparent.is_mutable())
1097             return newparent.get(u"newchild")
1098         d.addCallback(_got_newparent)
1099
1100         def _got_newchild(newchild):
1101             self.failUnless(IDirectoryNode.providedBy(newchild))
1102             self.failUnless(newchild.is_mutable())
1103         d.addCallback(_got_newchild)
1104
1105         d.addCallback(lambda ign:
1106             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "makeDirectory invalid UTF-8",
1107                                          self.handler.makeDirectory, "\xFF", {}))
1108
1109         # should fail because there is an existing file "small"
1110         d.addCallback(lambda ign:
1111             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "makeDirectory small",
1112                                          self.handler.makeDirectory, "small", {}))
1113         return d
1114
1115     def test_execCommand_and_openShell(self):
1116         class FakeProtocol:
1117             def __init__(self):
1118                 self.output = ""
1119                 self.reason = None
1120             def write(self, data):
1121                 self.output += data
1122                 return defer.succeed(None)
1123             def processEnded(self, reason):
1124                 self.reason = reason
1125                 return defer.succeed(None)
1126
1127         d = self._set_up("execCommand_and_openShell")
1128
1129         d.addCallback(lambda ign: conch_interfaces.ISession(self.handler))
1130         def _exec_df(session):
1131             protocol = FakeProtocol()
1132             d2 = session.execCommand(protocol, "df -P -k /")
1133             d2.addCallback(lambda ign: self.failUnlessIn("1024-blocks", protocol.output))
1134             d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessDone)))
1135             d2.addCallback(lambda ign: session.eofReceived())
1136             d2.addCallback(lambda ign: session.closed())
1137             return d2
1138         d.addCallback(_exec_df)
1139
1140         d.addCallback(lambda ign: conch_interfaces.ISession(self.handler))
1141         def _exec_error(session):
1142             protocol = FakeProtocol()
1143             d2 = session.execCommand(protocol, "error")
1144             d2.addCallback(lambda ign: self.failUnlessEqual("", protocol.output))
1145             d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessTerminated)))
1146             d2.addCallback(lambda ign: self.failUnlessEqual(protocol.reason.value.exitCode, 1))
1147             d2.addCallback(lambda ign: session.closed())
1148             return d2
1149         d.addCallback(_exec_error)
1150
1151         d.addCallback(lambda ign: conch_interfaces.ISession(self.handler))
1152         def _openShell(session):
1153             protocol = FakeProtocol()
1154             d2 = session.openShell(protocol)
1155             d2.addCallback(lambda ign: self.failUnlessIn("only SFTP", protocol.output))
1156             d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessTerminated)))
1157             d2.addCallback(lambda ign: self.failUnlessEqual(protocol.reason.value.exitCode, 1))
1158             d2.addCallback(lambda ign: session.closed())
1159             return d2
1160         d.addCallback(_exec_error)
1161
1162         return d
1163
1164     def test_extendedRequest(self):
1165         d = self._set_up("extendedRequest")
1166
1167         d.addCallback(lambda ign: self.handler.extendedRequest("statvfs@openssh.com", "/"))
1168         def _check(res):
1169             self.failUnless(isinstance(res, str))
1170             self.failUnlessEqual(len(res), 8*11)
1171         d.addCallback(_check)
1172
1173         d.addCallback(lambda ign:
1174             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "extendedRequest foo bar",
1175                                          self.handler.extendedRequest, "foo", "bar"))
1176
1177         return d