src/allmydata/test/test_sftp.py

   1
   2 import re, struct, traceback, gc, time, calendar
   3 from stat import S_IFREG, S_IFDIR
   4
   5 from twisted.trial import unittest
   6 from twisted.internet import defer
   7 from twisted.python.failure import Failure
   8 from twisted.internet.error import ProcessDone, ProcessTerminated
   9
  10 conch_interfaces = None
  11 sftp = None
  12 sftpd = None
  13 have_pycrypto = False
  14 try:
  15     from Crypto import Util
  16     Util  # hush pyflakes
  17     have_pycrypto = True
  18 except ImportError:
  19     pass
  20
  21 if have_pycrypto:
  22     from twisted.conch import interfaces as conch_interfaces
  23     from twisted.conch.ssh import filetransfer as sftp
  24     from allmydata.frontends import sftpd
  25
  26 from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError
  27 from allmydata.mutable.common import NotWriteableError
  28
  29 from allmydata.util.consumer import download_to_data
  30 from allmydata.immutable import upload
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common import ShouldFailMixin
  33
  34 timeout = 240
  35
  36 class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase):
  37     """This is a no-network unit test of the SFTPUserHandler and the abstractions it uses."""
  38
  39     if not have_pycrypto:
  40         skip = "SFTP support requires pycrypto, which is not installed"
  41
  42     def shouldFailWithSFTPError(self, expected_code, which, callable, *args, **kwargs):
  43         assert isinstance(expected_code, int), repr(expected_code)
  44         assert isinstance(which, str), repr(which)
  45         s = traceback.format_stack()
  46         d = defer.maybeDeferred(callable, *args, **kwargs)
  47         def _done(res):
  48             if isinstance(res, Failure):
  49                 res.trap(sftp.SFTPError)
  50                 self.failUnlessReallyEqual(res.value.code, expected_code,
  51                                            "%s was supposed to raise SFTPError(%d), not SFTPError(%d): %s" %
  52                                            (which, expected_code, res.value.code, res))
  53             else:
  54                 print '@' + '@'.join(s)
  55                 self.fail("%s was supposed to raise SFTPError(%d), not get %r" %
  56                           (which, expected_code, res))
  57         d.addBoth(_done)
  58         return d
  59
  60     def failUnlessReallyEqual(self, a, b, msg=None):
  61         self.failUnlessEqual(a, b, msg=msg)
  62         self.failUnlessEqual(type(a), type(b), msg=msg)
  63
  64     def _set_up(self, basedir, num_clients=1, num_servers=10):
  65         self.basedir = "sftp/" + basedir
  66         self.set_up_grid(num_clients=num_clients, num_servers=num_servers)
  67
  68         self.client = self.g.clients[0]
  69         self.username = "alice"
  70
  71         d = self.client.create_dirnode()
  72         def _created_root(node):
  73             self.root = node
  74             self.root_uri = node.get_uri()
  75             self.handler = sftpd.SFTPUserHandler(self.client, self.root, self.username)
  76         d.addCallback(_created_root)
  77         return d
  78
  79     def _set_up_tree(self):
  80         d = self.client.create_mutable_file("mutable file contents")
  81         d.addCallback(lambda node: self.root.set_node(u"mutable", node))
  82         def _created_mutable(n):
  83             self.mutable = n
  84             self.mutable_uri = n.get_uri()
  85         d.addCallback(_created_mutable)
  86
  87         d.addCallback(lambda ign:
  88                       self.root._create_and_validate_node(None, self.mutable.get_readonly_uri(), name=u"readonly"))
  89         d.addCallback(lambda node: self.root.set_node(u"readonly", node))
  90         def _created_readonly(n):
  91             self.readonly = n
  92             self.readonly_uri = n.get_uri()
  93         d.addCallback(_created_readonly)
  94
  95         gross = upload.Data("0123456789" * 101, None)
  96         d.addCallback(lambda ign: self.root.add_file(u"gro\u00DF", gross))
  97         def _created_gross(n):
  98             self.gross = n
  99             self.gross_uri = n.get_uri()
 100         d.addCallback(_created_gross)
 101
 102         small = upload.Data("0123456789", None)
 103         d.addCallback(lambda ign: self.root.add_file(u"small", small))
 104         def _created_small(n):
 105             self.small = n
 106             self.small_uri = n.get_uri()
 107         d.addCallback(_created_small)
 108
 109         small2 = upload.Data("Small enough for a LIT too", None)
 110         d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
 111         def _created_small2(n):
 112             self.small2 = n
 113             self.small2_uri = n.get_uri()
 114         d.addCallback(_created_small2)
 115
 116         empty_litdir_uri = "URI:DIR2-LIT:"
 117
 118         # contains one child which is itself also LIT:
 119         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm"
 120
 121         unknown_uri = "x-tahoe-crazy://I_am_from_the_future."
 122
 123         d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"empty_lit_dir"))
 124         def _created_empty_lit_dir(n):
 125             self.empty_lit_dir = n
 126             self.empty_lit_dir_uri = n.get_uri()
 127             self.root.set_node(u"empty_lit_dir", n)
 128         d.addCallback(_created_empty_lit_dir)
 129
 130         d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"tiny_lit_dir"))
 131         def _created_tiny_lit_dir(n):
 132             self.tiny_lit_dir = n
 133             self.tiny_lit_dir_uri = n.get_uri()
 134             self.root.set_node(u"tiny_lit_dir", n)
 135         d.addCallback(_created_tiny_lit_dir)
 136
 137         d.addCallback(lambda ign: self.root._create_and_validate_node(None, unknown_uri, name=u"unknown"))
 138         def _created_unknown(n):
 139             self.unknown = n
 140             self.unknown_uri = n.get_uri()
 141             self.root.set_node(u"unknown", n)
 142         d.addCallback(_created_unknown)
 143
 144         fall_of_the_Berlin_wall = calendar.timegm(time.strptime("1989-11-09 20:00:00 UTC", "%Y-%m-%d %H:%M:%S %Z"))
 145         md = {'mtime': fall_of_the_Berlin_wall, 'tahoe': {'linkmotime': fall_of_the_Berlin_wall}}
 146         d.addCallback(lambda ign: self.root.set_node(u"loop", self.root, metadata=md))
 147         return d
 148
 149     def test_basic(self):
 150         d = self._set_up("basic")
 151         def _check(ign):
 152             # Test operations that have no side-effects, and don't need the tree.
 153
 154             version = self.handler.gotVersion(3, {})
 155             self.failUnless(isinstance(version, dict))
 156
 157             self.failUnlessReallyEqual(self.handler._path_from_string(""), [])
 158             self.failUnlessReallyEqual(self.handler._path_from_string("/"), [])
 159             self.failUnlessReallyEqual(self.handler._path_from_string("."), [])
 160             self.failUnlessReallyEqual(self.handler._path_from_string("//"), [])
 161             self.failUnlessReallyEqual(self.handler._path_from_string("/."), [])
 162             self.failUnlessReallyEqual(self.handler._path_from_string("/./"), [])
 163             self.failUnlessReallyEqual(self.handler._path_from_string("foo"), [u"foo"])
 164             self.failUnlessReallyEqual(self.handler._path_from_string("/foo"), [u"foo"])
 165             self.failUnlessReallyEqual(self.handler._path_from_string("foo/"), [u"foo"])
 166             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/"), [u"foo"])
 167             self.failUnlessReallyEqual(self.handler._path_from_string("foo/bar"), [u"foo", u"bar"])
 168             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/bar"), [u"foo", u"bar"])
 169             self.failUnlessReallyEqual(self.handler._path_from_string("foo/bar//"), [u"foo", u"bar"])
 170             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/bar//"), [u"foo", u"bar"])
 171             self.failUnlessReallyEqual(self.handler._path_from_string("foo/../bar"), [u"bar"])
 172             self.failUnlessReallyEqual(self.handler._path_from_string("/foo/../bar"), [u"bar"])
 173             self.failUnlessReallyEqual(self.handler._path_from_string("../bar"), [u"bar"])
 174             self.failUnlessReallyEqual(self.handler._path_from_string("/../bar"), [u"bar"])
 175
 176             self.failUnlessReallyEqual(self.handler.realPath(""), "/")
 177             self.failUnlessReallyEqual(self.handler.realPath("/"), "/")
 178             self.failUnlessReallyEqual(self.handler.realPath("."), "/")
 179             self.failUnlessReallyEqual(self.handler.realPath("//"), "/")
 180             self.failUnlessReallyEqual(self.handler.realPath("/."), "/")
 181             self.failUnlessReallyEqual(self.handler.realPath("/./"), "/")
 182             self.failUnlessReallyEqual(self.handler.realPath("foo"), "/foo")
 183             self.failUnlessReallyEqual(self.handler.realPath("/foo"), "/foo")
 184             self.failUnlessReallyEqual(self.handler.realPath("foo/"), "/foo")
 185             self.failUnlessReallyEqual(self.handler.realPath("/foo/"), "/foo")
 186             self.failUnlessReallyEqual(self.handler.realPath("foo/bar"), "/foo/bar")
 187             self.failUnlessReallyEqual(self.handler.realPath("/foo/bar"), "/foo/bar")
 188             self.failUnlessReallyEqual(self.handler.realPath("foo/bar//"), "/foo/bar")
 189             self.failUnlessReallyEqual(self.handler.realPath("/foo/bar//"), "/foo/bar")
 190             self.failUnlessReallyEqual(self.handler.realPath("foo/../bar"), "/bar")
 191             self.failUnlessReallyEqual(self.handler.realPath("/foo/../bar"), "/bar")
 192             self.failUnlessReallyEqual(self.handler.realPath("../bar"), "/bar")
 193             self.failUnlessReallyEqual(self.handler.realPath("/../bar"), "/bar")
 194         d.addCallback(_check)
 195
 196         d.addCallback(lambda ign:
 197             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "_path_from_string invalid UTF-8",
 198                                          self.handler._path_from_string, "\xFF"))
 199         d.addCallback(lambda ign:
 200             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "realPath invalid UTF-8",
 201                                          self.handler.realPath, "\xFF"))
 202
 203         return d
 204
 205     def test_convert_error(self):
 206         self.failUnlessReallyEqual(sftpd._convert_error(None, "request"), None)
 207
 208         d = defer.succeed(None)
 209         d.addCallback(lambda ign:
 210             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "_convert_error SFTPError",
 211                                          sftpd._convert_error, Failure(sftp.SFTPError(sftp.FX_FAILURE, "foo")), "request"))
 212         d.addCallback(lambda ign:
 213             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "_convert_error NoSuchChildError",
 214                                          sftpd._convert_error, Failure(NoSuchChildError("foo")), "request"))
 215         d.addCallback(lambda ign:
 216             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "_convert_error ExistingChildError",
 217                                          sftpd._convert_error, Failure(ExistingChildError("foo")), "request"))
 218         d.addCallback(lambda ign:
 219             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "_convert_error NotWriteableError",
 220                                          sftpd._convert_error, Failure(NotWriteableError("foo")), "request"))
 221         d.addCallback(lambda ign:
 222             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "_convert_error NotImplementedError",
 223                                          sftpd._convert_error, Failure(NotImplementedError("foo")), "request"))
 224         d.addCallback(lambda ign:
 225             self.shouldFailWithSFTPError(sftp.FX_EOF, "_convert_error EOFError",
 226                                          sftpd._convert_error, Failure(EOFError("foo")), "request"))
 227         d.addCallback(lambda ign:
 228             self.shouldFailWithSFTPError(sftp.FX_EOF, "_convert_error defer.FirstError",
 229                                          sftpd._convert_error, Failure(defer.FirstError(
 230                                                                  Failure(sftp.SFTPError(sftp.FX_EOF, "foo")), 0)), "request"))
 231         d.addCallback(lambda ign:
 232             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "_convert_error AssertionError",
 233                                          sftpd._convert_error, Failure(AssertionError("foo")), "request"))
 234
 235         return d
 236
 237     def test_not_implemented(self):
 238         d = self._set_up("not_implemented")
 239
 240         d.addCallback(lambda ign:
 241             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "readLink link",
 242                                          self.handler.readLink, "link"))
 243         d.addCallback(lambda ign:
 244             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "makeLink link file",
 245                                          self.handler.makeLink, "link", "file"))
 246
 247         return d
 248
 249     def _compareDirLists(self, actual, expected):
 250        actual_list = sorted(actual)
 251        expected_list = sorted(expected)
 252        self.failUnlessReallyEqual(len(actual_list), len(expected_list),
 253                             "%r is wrong length, expecting %r" % (actual_list, expected_list))
 254        for (a, b) in zip(actual_list, expected_list):
 255            (name, text, attrs) = a
 256            (expected_name, expected_text_re, expected_attrs) = b
 257            self.failUnlessReallyEqual(name, expected_name)
 258            self.failUnless(re.match(expected_text_re, text),
 259                            "%r does not match %r in\n%r" % (text, expected_text_re, actual_list))
 260            self._compareAttributes(attrs, expected_attrs)
 261
 262     def _compareAttributes(self, attrs, expected_attrs):
 263         # It is ok for there to be extra actual attributes.
 264         # TODO: check times
 265         for e in expected_attrs:
 266             self.failUnless(e in attrs, "%r is not in\n%r" % (e, attrs))
 267             self.failUnlessReallyEqual(attrs[e], expected_attrs[e],
 268                                        "%r:%r is not %r in\n%r" % (e, attrs[e], expected_attrs[e], attrs))
 269
 270     def test_openDirectory_and_attrs(self):
 271         d = self._set_up("openDirectory_and_attrs")
 272         d.addCallback(lambda ign: self._set_up_tree())
 273
 274         d.addCallback(lambda ign:
 275             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openDirectory small",
 276                                          self.handler.openDirectory, "small"))
 277         d.addCallback(lambda ign:
 278             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openDirectory unknown",
 279                                          self.handler.openDirectory, "unknown"))
 280         d.addCallback(lambda ign:
 281             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openDirectory nodir",
 282                                          self.handler.openDirectory, "nodir"))
 283         d.addCallback(lambda ign:
 284             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openDirectory nodir/nodir",
 285                                          self.handler.openDirectory, "nodir/nodir"))
 286
 287         gross = u"gro\u00DF".encode("utf-8")
 288         expected_root = [
 289             ('empty_lit_dir', r'dr-xr-xr-x .* \? .* empty_lit_dir$',      {'permissions': S_IFDIR | 0555}),
 290             (gross,           r'-rw-rw-rw- .* 1010 .* '+gross+'$',        {'permissions': S_IFREG | 0666, 'size': 1010}),
 291             # The fall of the Berlin wall may have been on 9th or 10th November 1989 depending on the gateway's timezone.
 292             #('loop',          r'drwxrwxrwx .* \? Nov (09|10)  1989 loop$', {'permissions': S_IFDIR | 0777}),
 293             ('loop',          r'drwxrwxrwx .* \? .* loop$',               {'permissions': S_IFDIR | 0777}),
 294             ('mutable',       r'-rw-rw-rw- .* \? .* mutable$',            {'permissions': S_IFREG | 0666}),
 295             ('readonly',      r'-r--r--r-- .* \? .* readonly$',           {'permissions': S_IFREG | 0444}),
 296             ('small',         r'-rw-rw-rw- .* 10 .* small$',              {'permissions': S_IFREG | 0666, 'size': 10}),
 297             ('small2',        r'-rw-rw-rw- .* 26 .* small2$',             {'permissions': S_IFREG | 0666, 'size': 26}),
 298             ('tiny_lit_dir',  r'dr-xr-xr-x .* \? .* tiny_lit_dir$',       {'permissions': S_IFDIR | 0555}),
 299             ('unknown',       r'\?--------- .* \? .* unknown$',           {'permissions': 0}),
 300         ]
 301
 302         d.addCallback(lambda ign: self.handler.openDirectory(""))
 303         d.addCallback(lambda res: self._compareDirLists(res, expected_root))
 304
 305         d.addCallback(lambda ign: self.handler.openDirectory("loop"))
 306         d.addCallback(lambda res: self._compareDirLists(res, expected_root))
 307
 308         d.addCallback(lambda ign: self.handler.openDirectory("loop/loop"))
 309         d.addCallback(lambda res: self._compareDirLists(res, expected_root))
 310
 311         d.addCallback(lambda ign: self.handler.openDirectory("empty_lit_dir"))
 312         d.addCallback(lambda res: self._compareDirLists(res, []))
 313
 314         # The UTC epoch may either be in Jan 1 1970 or Dec 31 1969 depending on the gateway's timezone.
 315         expected_tiny_lit = [
 316             ('short', r'-r--r--r-- .* 8 (Jan 01  1970|Dec 31  1969) short$', {'permissions': S_IFREG | 0444, 'size': 8}),
 317         ]
 318
 319         d.addCallback(lambda ign: self.handler.openDirectory("tiny_lit_dir"))
 320         d.addCallback(lambda res: self._compareDirLists(res, expected_tiny_lit))
 321
 322         d.addCallback(lambda ign: self.handler.getAttrs("small", True))
 323         d.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 10}))
 324
 325         d.addCallback(lambda ign: self.handler.setAttrs("small", {}))
 326         d.addCallback(lambda res: self.failUnlessReallyEqual(res, None))
 327
 328         d.addCallback(lambda ign: self.handler.getAttrs("small", True))
 329         d.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 10}))
 330
 331         d.addCallback(lambda ign:
 332             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "setAttrs size",
 333                                          self.handler.setAttrs, "small", {'size': 0}))
 334
 335         return d
 336
 337     def test_openFile_read(self):
 338         d = self._set_up("openFile_read")
 339         d.addCallback(lambda ign: self._set_up_tree())
 340
 341         d.addCallback(lambda ign:
 342             self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small 0",
 343                                          self.handler.openFile, "small", 0, {}))
 344
 345         # attempting to open a non-existent file should fail
 346         d.addCallback(lambda ign:
 347             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nofile READ",
 348                                          self.handler.openFile, "nofile", sftp.FXF_READ, {}))
 349         d.addCallback(lambda ign:
 350             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nodir/file READ",
 351                                          self.handler.openFile, "nodir/file", sftp.FXF_READ, {}))
 352
 353         d.addCallback(lambda ign:
 354             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown READ denied",
 355                                          self.handler.openFile, "unknown", sftp.FXF_READ, {}))
 356         d.addCallback(lambda ign:
 357             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir READ denied",
 358                                          self.handler.openFile, "tiny_lit_dir", sftp.FXF_READ, {}))
 359         d.addCallback(lambda ign:
 360             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown uri READ denied",
 361                                          self.handler.openFile, "uri/"+self.unknown_uri, sftp.FXF_READ, {}))
 362         d.addCallback(lambda ign:
 363             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir uri READ denied",
 364                                          self.handler.openFile, "uri/"+self.tiny_lit_dir_uri, sftp.FXF_READ, {}))
 365         # FIXME: should be FX_NO_SUCH_FILE?
 366         d.addCallback(lambda ign:
 367             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile noexist uri READ denied",
 368                                          self.handler.openFile, "uri/URI:noexist", sftp.FXF_READ, {}))
 369         d.addCallback(lambda ign:
 370             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile invalid UTF-8 uri READ denied",
 371                                          self.handler.openFile, "uri/URI:\xFF", sftp.FXF_READ, {}))
 372
 373         # reading an existing file should succeed
 374         d.addCallback(lambda ign: self.handler.openFile("small", sftp.FXF_READ, {}))
 375         def _read_small(rf):
 376             d2 = rf.readChunk(0, 10)
 377             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 378
 379             d2.addCallback(lambda ign: rf.readChunk(2, 6))
 380             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "234567"))
 381
 382             d2.addCallback(lambda ign: rf.readChunk(1, 0))
 383             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 384
 385             d2.addCallback(lambda ign: rf.readChunk(8, 4))  # read that starts before EOF is OK
 386             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "89"))
 387
 388             d2.addCallback(lambda ign:
 389                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF (0-byte)",
 390                                              rf.readChunk, 10, 0))
 391             d2.addCallback(lambda ign:
 392                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF",
 393                                              rf.readChunk, 10, 1))
 394             d2.addCallback(lambda ign:
 395                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting after EOF",
 396                                              rf.readChunk, 11, 1))
 397
 398             d2.addCallback(lambda ign: rf.getAttrs())
 399             d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 10}))
 400
 401             d2.addCallback(lambda ign: self.handler.getAttrs("small", followLinks=0))
 402             d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 10}))
 403
 404             d2.addCallback(lambda ign:
 405                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "writeChunk on read-only handle denied",
 406                                              rf.writeChunk, 0, "a"))
 407             d2.addCallback(lambda ign:
 408                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "setAttrs on read-only handle denied",
 409                                              rf.setAttrs, {}))
 410
 411             d2.addCallback(lambda ign: rf.close())
 412
 413             d2.addCallback(lambda ign:
 414                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "readChunk on closed file",
 415                                              rf.readChunk, 0, 1))
 416             d2.addCallback(lambda ign:
 417                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "getAttrs on closed file",
 418                                              rf.getAttrs))
 419
 420             d2.addCallback(lambda ign: rf.close()) # should be no-op
 421             return d2
 422         d.addCallback(_read_small)
 423
 424         # repeat for a large file
 425         gross = u"gro\u00DF".encode("utf-8")
 426         d.addCallback(lambda ign: self.handler.openFile(gross, sftp.FXF_READ, {}))
 427         def _read_gross(rf):
 428             d2 = rf.readChunk(0, 10)
 429             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 430
 431             d2.addCallback(lambda ign: rf.readChunk(2, 6))
 432             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "234567"))
 433
 434             d2.addCallback(lambda ign: rf.readChunk(1, 0))
 435             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 436
 437             d2.addCallback(lambda ign: rf.readChunk(1008, 4))  # read that starts before EOF is OK
 438             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "89"))
 439
 440             d2.addCallback(lambda ign:
 441                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF (0-byte)",
 442                                              rf.readChunk, 1010, 0))
 443             d2.addCallback(lambda ign:
 444                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF",
 445                                              rf.readChunk, 1010, 1))
 446             d2.addCallback(lambda ign:
 447                 self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting after EOF",
 448                                              rf.readChunk, 1011, 1))
 449
 450             d2.addCallback(lambda ign: rf.getAttrs())
 451             d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 1010}))
 452
 453             d2.addCallback(lambda ign: self.handler.getAttrs(gross, followLinks=0))
 454             d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 1010}))
 455
 456             d2.addCallback(lambda ign:
 457                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "writeChunk on read-only handle denied",
 458                                              rf.writeChunk, 0, "a"))
 459             d2.addCallback(lambda ign:
 460                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "setAttrs on read-only handle denied",
 461                                              rf.setAttrs, {}))
 462
 463             d2.addCallback(lambda ign: rf.close())
 464
 465             d2.addCallback(lambda ign:
 466                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "readChunk on closed file",
 467                                              rf.readChunk, 0, 1))
 468             d2.addCallback(lambda ign:
 469                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "getAttrs on closed file",
 470                                              rf.getAttrs))
 471
 472             d2.addCallback(lambda ign: rf.close()) # should be no-op
 473             return d2
 474         d.addCallback(_read_gross)
 475
 476         # reading an existing small file via uri/ should succeed
 477         d.addCallback(lambda ign: self.handler.openFile("uri/"+self.small_uri, sftp.FXF_READ, {}))
 478         def _read_small_uri(rf):
 479             d2 = rf.readChunk(0, 10)
 480             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 481             d2.addCallback(lambda ign: rf.close())
 482             return d2
 483         d.addCallback(_read_small_uri)
 484
 485         # repeat for a large file
 486         d.addCallback(lambda ign: self.handler.openFile("uri/"+self.gross_uri, sftp.FXF_READ, {}))
 487         def _read_gross_uri(rf):
 488             d2 = rf.readChunk(0, 10)
 489             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 490             d2.addCallback(lambda ign: rf.close())
 491             return d2
 492         d.addCallback(_read_gross_uri)
 493
 494         # repeat for a mutable file
 495         d.addCallback(lambda ign: self.handler.openFile("uri/"+self.mutable_uri, sftp.FXF_READ, {}))
 496         def _read_mutable_uri(rf):
 497             d2 = rf.readChunk(0, 100)
 498             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "mutable file contents"))
 499             d2.addCallback(lambda ign: rf.close())
 500             return d2
 501         d.addCallback(_read_mutable_uri)
 502
 503         # repeat for a file within a directory referenced by URI
 504         d.addCallback(lambda ign: self.handler.openFile("uri/"+self.tiny_lit_dir_uri+"/short", sftp.FXF_READ, {}))
 505         def _read_short(rf):
 506             d2 = rf.readChunk(0, 100)
 507             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "The end."))
 508             d2.addCallback(lambda ign: rf.close())
 509             return d2
 510         d.addCallback(_read_short)
 511
 512         return d
 513
 514     def test_openFile_write(self):
 515         d = self._set_up("openFile_write")
 516         d.addCallback(lambda ign: self._set_up_tree())
 517
 518         # '' is an invalid filename
 519         d.addCallback(lambda ign:
 520             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile '' WRITE|CREAT|TRUNC",
 521                                          self.handler.openFile, "", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 522
 523         # TRUNC is not valid without CREAT if the file does not already exist
 524         d.addCallback(lambda ign:
 525             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile newfile WRITE|TRUNC",
 526                                          self.handler.openFile, "newfile", sftp.FXF_WRITE | sftp.FXF_TRUNC, {}))
 527
 528         # EXCL is not valid without CREAT
 529         d.addCallback(lambda ign:
 530             self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small WRITE|EXCL",
 531                                          self.handler.openFile, "small", sftp.FXF_WRITE | sftp.FXF_EXCL, {}))
 532
 533         # cannot write to an existing directory
 534         d.addCallback(lambda ign:
 535             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir WRITE",
 536                                          self.handler.openFile, "tiny_lit_dir", sftp.FXF_WRITE, {}))
 537
 538         # cannot write to an existing unknown
 539         d.addCallback(lambda ign:
 540             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown WRITE",
 541                                          self.handler.openFile, "unknown", sftp.FXF_WRITE, {}))
 542
 543         # cannot write to a new file in an immutable directory
 544         d.addCallback(lambda ign:
 545             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/newfile WRITE|CREAT|TRUNC",
 546                                          self.handler.openFile, "tiny_lit_dir/newfile",
 547                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 548
 549         # cannot write to an existing immutable file in an immutable directory (with or without CREAT and EXCL)
 550         d.addCallback(lambda ign:
 551             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE",
 552                                          self.handler.openFile, "tiny_lit_dir/short", sftp.FXF_WRITE, {}))
 553         d.addCallback(lambda ign:
 554             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE|CREAT",
 555                                          self.handler.openFile, "tiny_lit_dir/short",
 556                                          sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
 557
 558         # cannot write to a mutable file via a readonly cap (by path or uri)
 559         d.addCallback(lambda ign:
 560             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly WRITE",
 561                                          self.handler.openFile, "readonly", sftp.FXF_WRITE, {}))
 562         d.addCallback(lambda ign:
 563             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly uri WRITE",
 564                                          self.handler.openFile, "uri/"+self.readonly_uri, sftp.FXF_WRITE, {}))
 565
 566         # cannot create a file with the EXCL flag if it already exists
 567         d.addCallback(lambda ign:
 568             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile small WRITE|CREAT|EXCL",
 569                                          self.handler.openFile, "small",
 570                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 571         d.addCallback(lambda ign:
 572             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile mutable WRITE|CREAT|EXCL",
 573                                          self.handler.openFile, "mutable",
 574                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 575         d.addCallback(lambda ign:
 576             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile mutable uri WRITE|CREAT|EXCL",
 577                                          self.handler.openFile, "uri/"+self.mutable_uri,
 578                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 579         d.addCallback(lambda ign:
 580             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "openFile tiny_lit_dir/short WRITE|CREAT|EXCL",
 581                                          self.handler.openFile, "tiny_lit_dir/short",
 582                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 583
 584         # cannot write to an immutable file if we don't have its parent (with or without CREAT, TRUNC, or EXCL)
 585         d.addCallback(lambda ign:
 586             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE",
 587                                          self.handler.openFile, "uri/"+self.small_uri, sftp.FXF_WRITE, {}))
 588         d.addCallback(lambda ign:
 589             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT",
 590                                          self.handler.openFile, "uri/"+self.small_uri,
 591                                          sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
 592         d.addCallback(lambda ign:
 593             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT|TRUNC",
 594                                          self.handler.openFile, "uri/"+self.small_uri,
 595                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 596         d.addCallback(lambda ign:
 597             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT|EXCL",
 598                                          self.handler.openFile, "uri/"+self.small_uri,
 599                                          sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 600
 601         # test creating a new file with truncation
 602         d.addCallback(lambda ign:
 603                       self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {}))
 604         def _write(wf):
 605             d2 = wf.writeChunk(0, "0123456789")
 606             d2.addCallback(lambda res: self.failUnlessReallyEqual(res, None))
 607
 608             d2.addCallback(lambda ign: wf.writeChunk(8, "0123"))
 609             d2.addCallback(lambda ign: wf.writeChunk(13, "abc"))
 610
 611             d2.addCallback(lambda ign: wf.getAttrs())
 612             d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 16}))
 613
 614             d2.addCallback(lambda ign: self.handler.getAttrs("newfile", followLinks=0))
 615             d2.addCallback(lambda attrs: self._compareAttributes(attrs, {'permissions': S_IFREG | 0666, 'size': 16}))
 616
 617             d2.addCallback(lambda ign: wf.setAttrs({}))
 618
 619             d2.addCallback(lambda ign:
 620                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs with negative size",
 621                                              wf.setAttrs, {'size': -1}))
 622
 623             d2.addCallback(lambda ign: wf.setAttrs({'size': 14}))
 624             d2.addCallback(lambda ign: wf.getAttrs())
 625             d2.addCallback(lambda attrs: self.failUnlessReallyEqual(attrs['size'], 14))
 626
 627             d2.addCallback(lambda ign:
 628                 self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "readChunk on write-only handle denied",
 629                                              wf.readChunk, 0, 1))
 630
 631             d2.addCallback(lambda ign: wf.close())
 632
 633             d2.addCallback(lambda ign:
 634                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "writeChunk on closed file",
 635                                              wf.writeChunk, 0, "a"))
 636             d2.addCallback(lambda ign:
 637                 self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs on closed file",
 638                                              wf.setAttrs, {'size': 0}))
 639
 640             d2.addCallback(lambda ign: wf.close()) # should be no-op
 641             return d2
 642         d.addCallback(_write)
 643         d.addCallback(lambda ign: self.root.get(u"newfile"))
 644         d.addCallback(lambda node: download_to_data(node))
 645         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123\x00a"))
 646
 647         # test APPEND flag, and also replacing an existing file ("newfile" created by the previous test)
 648         d.addCallback(lambda ign:
 649                       self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_CREAT |
 650                                                        sftp.FXF_TRUNC | sftp.FXF_APPEND, {}))
 651         def _write_append(wf):
 652             d2 = wf.writeChunk(0, "0123456789")
 653             d2.addCallback(lambda ign: wf.writeChunk(8, "0123"))
 654             d2.addCallback(lambda ign: wf.close())
 655             return d2
 656         d.addCallback(_write_append)
 657         d.addCallback(lambda ign: self.root.get(u"newfile"))
 658         d.addCallback(lambda node: download_to_data(node))
 659         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234567890123"))
 660
 661         # test WRITE | TRUNC without CREAT, when the file already exists
 662         # This is invalid according to section 6.3 of the SFTP spec, but required for interoperability,
 663         # since POSIX does allow O_WRONLY | O_TRUNC.
 664         d.addCallback(lambda ign:
 665                       self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_TRUNC, {}))
 666         def _write_trunc(wf):
 667             d2 = wf.writeChunk(0, "01234")
 668             d2.addCallback(lambda ign: wf.close())
 669             return d2
 670         d.addCallback(_write_trunc)
 671         d.addCallback(lambda ign: self.root.get(u"newfile"))
 672         d.addCallback(lambda node: download_to_data(node))
 673         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234"))
 674
 675         # test EXCL flag
 676         d.addCallback(lambda ign:
 677                       self.handler.openFile("excl", sftp.FXF_WRITE | sftp.FXF_CREAT |
 678                                                     sftp.FXF_TRUNC | sftp.FXF_EXCL, {}))
 679         def _write_excl(wf):
 680             d2 = self.root.get(u"excl")
 681             d2.addCallback(lambda node: download_to_data(node))
 682             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 683
 684             d2.addCallback(lambda ign: wf.writeChunk(0, "0123456789"))
 685             d2.addCallback(lambda ign: wf.close())
 686             return d2
 687         d.addCallback(_write_excl)
 688         d.addCallback(lambda ign: self.root.get(u"excl"))
 689         d.addCallback(lambda node: download_to_data(node))
 690         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 691
 692         # test that writing a zero-length file with EXCL only updates the directory once
 693         d.addCallback(lambda ign:
 694                       self.handler.openFile("zerolength", sftp.FXF_WRITE | sftp.FXF_CREAT |
 695                                                           sftp.FXF_EXCL, {}))
 696         def _write_excl_zerolength(wf):
 697             d2 = self.root.get(u"zerolength")
 698             d2.addCallback(lambda node: download_to_data(node))
 699             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 700
 701             # FIXME: no API to get the best version number exists (fix as part of #993)
 702             """
 703             d2.addCallback(lambda ign: self.root.get_best_version_number())
 704             def _check_version(version):
 705                 d3 = wf.close()
 706                 d3.addCallback(lambda ign: self.root.get_best_version_number())
 707                 d3.addCallback(lambda new_version: self.failUnlessReallyEqual(new_version, version))
 708                 return d3
 709             d2.addCallback(_check_version)
 710             """
 711             d2.addCallback(lambda ign: wf.close())
 712             return d2
 713         d.addCallback(_write_excl_zerolength)
 714         d.addCallback(lambda ign: self.root.get(u"zerolength"))
 715         d.addCallback(lambda node: download_to_data(node))
 716         d.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 717
 718         # test WRITE | CREAT | EXCL | APPEND
 719         d.addCallback(lambda ign:
 720                       self.handler.openFile("exclappend", sftp.FXF_WRITE | sftp.FXF_CREAT |
 721                                                           sftp.FXF_EXCL | sftp.FXF_APPEND, {}))
 722         def _write_excl_append(wf):
 723             d2 = self.root.get(u"exclappend")
 724             d2.addCallback(lambda node: download_to_data(node))
 725             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, ""))
 726
 727             d2.addCallback(lambda ign: wf.writeChunk(10, "0123456789"))
 728             d2.addCallback(lambda ign: wf.writeChunk(5, "01234"))
 729             d2.addCallback(lambda ign: wf.close())
 730             return d2
 731         d.addCallback(_write_excl_append)
 732         d.addCallback(lambda ign: self.root.get(u"exclappend"))
 733         d.addCallback(lambda node: download_to_data(node))
 734         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345678901234"))
 735
 736         # test WRITE | CREAT | APPEND when the file does not already exist
 737         d.addCallback(lambda ign:
 738                       self.handler.openFile("creatappend", sftp.FXF_WRITE | sftp.FXF_CREAT |
 739                                                            sftp.FXF_APPEND, {}))
 740         def _write_creat_append_new(wf):
 741             d2 = wf.writeChunk(10, "0123456789")
 742             d2.addCallback(lambda ign: wf.writeChunk(5, "01234"))
 743             d2.addCallback(lambda ign: wf.close())
 744             return d2
 745         d.addCallback(_write_creat_append_new)
 746         d.addCallback(lambda ign: self.root.get(u"creatappend"))
 747         d.addCallback(lambda node: download_to_data(node))
 748         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345678901234"))
 749
 750         # ... and when it does exist
 751         d.addCallback(lambda ign:
 752                       self.handler.openFile("creatappend", sftp.FXF_WRITE | sftp.FXF_CREAT |
 753                                                            sftp.FXF_APPEND, {}))
 754         def _write_creat_append_existing(wf):
 755             d2 = wf.writeChunk(5, "01234")
 756             d2.addCallback(lambda ign: wf.close())
 757             return d2
 758         d.addCallback(_write_creat_append_existing)
 759         d.addCallback(lambda ign: self.root.get(u"creatappend"))
 760         d.addCallback(lambda node: download_to_data(node))
 761         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "01234567890123401234"))
 762
 763         # test WRITE | CREAT without TRUNC, when the file does not already exist
 764         d.addCallback(lambda ign:
 765                       self.handler.openFile("newfile2", sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
 766         def _write_creat_new(wf):
 767             d2 =  wf.writeChunk(0, "0123456789")
 768             d2.addCallback(lambda ign: wf.close())
 769             return d2
 770         d.addCallback(_write_creat_new)
 771         d.addCallback(lambda ign: self.root.get(u"newfile2"))
 772         d.addCallback(lambda node: download_to_data(node))
 773         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 774
 775         # ... and when it does exist
 776         d.addCallback(lambda ign:
 777                       self.handler.openFile("newfile2", sftp.FXF_WRITE | sftp.FXF_CREAT, {}))
 778         def _write_creat_existing(wf):
 779             d2 =  wf.writeChunk(0, "abcde")
 780             d2.addCallback(lambda ign: wf.close())
 781             return d2
 782         d.addCallback(_write_creat_existing)
 783         d.addCallback(lambda ign: self.root.get(u"newfile2"))
 784         d.addCallback(lambda node: download_to_data(node))
 785         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "abcde56789"))
 786
 787         # test writing to a mutable file
 788         d.addCallback(lambda ign:
 789                       self.handler.openFile("mutable", sftp.FXF_WRITE, {}))
 790         def _write_mutable(wf):
 791             d2 = wf.writeChunk(8, "new!")
 792             d2.addCallback(lambda ign: wf.close())
 793             return d2
 794         d.addCallback(_write_mutable)
 795         d.addCallback(lambda ign: self.root.get(u"mutable"))
 796         def _check_same_file(node):
 797             self.failUnless(node.is_mutable())
 798             self.failUnlessReallyEqual(node.get_uri(), self.mutable_uri)
 799             return node.download_best_version()
 800         d.addCallback(_check_same_file)
 801         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "mutable new! contents"))
 802
 803         # test READ | WRITE without CREAT or TRUNC
 804         d.addCallback(lambda ign:
 805                       self.handler.openFile("small", sftp.FXF_READ | sftp.FXF_WRITE, {}))
 806         def _read_write(rwf):
 807             d2 = rwf.writeChunk(8, "0123")
 808             d2.addCallback(lambda ign: rwf.readChunk(0, 100))
 809             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123"))
 810             d2.addCallback(lambda ign: rwf.close())
 811             return d2
 812         d.addCallback(_read_write)
 813         d.addCallback(lambda ign: self.root.get(u"small"))
 814         d.addCallback(lambda node: download_to_data(node))
 815         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "012345670123"))
 816
 817         # test WRITE and rename while still open
 818         d.addCallback(lambda ign:
 819                       self.handler.openFile("small", sftp.FXF_WRITE, {}))
 820         def _write_rename(wf):
 821             d2 = wf.writeChunk(0, "abcd")
 822             d2.addCallback(lambda ign: self.handler.renameFile("small", "renamed"))
 823             d2.addCallback(lambda ign: wf.writeChunk(4, "efgh"))
 824             d2.addCallback(lambda ign: wf.close())
 825             return d2
 826         d.addCallback(_write_rename)
 827         d.addCallback(lambda ign: self.root.get(u"renamed"))
 828         d.addCallback(lambda node: download_to_data(node))
 829         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "abcdefgh0123"))
 830         d.addCallback(lambda ign:
 831                       self.shouldFail(NoSuchChildError, "rename small while open", "small",
 832                                       self.root.get, u"small"))
 833
 834         # test WRITE | CREAT | EXCL and rename while still open
 835         d.addCallback(lambda ign:
 836                       self.handler.openFile("newexcl", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {}))
 837         def _write_creat_excl_rename(wf):
 838             d2 = wf.writeChunk(0, "abcd")
 839             d2.addCallback(lambda ign: self.handler.renameFile("newexcl", "renamedexcl"))
 840             d2.addCallback(lambda ign: wf.writeChunk(4, "efgh"))
 841             d2.addCallback(lambda ign: wf.close())
 842             return d2
 843         d.addCallback(_write_creat_excl_rename)
 844         d.addCallback(lambda ign: self.root.get(u"renamedexcl"))
 845         d.addCallback(lambda node: download_to_data(node))
 846         d.addCallback(lambda data: self.failUnlessReallyEqual(data, "abcdefgh"))
 847         d.addCallback(lambda ign:
 848                       self.shouldFail(NoSuchChildError, "rename newexcl while open", "newexcl",
 849                                       self.root.get, u"newexcl"))
 850
 851         return d
 852
 853     def test_removeFile(self):
 854         d = self._set_up("removeFile")
 855         d.addCallback(lambda ign: self._set_up_tree())
 856
 857         d.addCallback(lambda ign:
 858             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeFile nofile",
 859                                          self.handler.removeFile, "nofile"))
 860         d.addCallback(lambda ign:
 861             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeFile nofile",
 862                                          self.handler.removeFile, "nofile"))
 863         d.addCallback(lambda ign:
 864             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeFile nodir/file",
 865                                          self.handler.removeFile, "nodir/file"))
 866         d.addCallback(lambda ign:
 867             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removefile ''",
 868                                          self.handler.removeFile, ""))
 869
 870         # removing a directory should fail
 871         d.addCallback(lambda ign:
 872             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "removeFile tiny_lit_dir",
 873                                          self.handler.removeFile, "tiny_lit_dir"))
 874
 875         # removing a file should succeed
 876         d.addCallback(lambda ign: self.root.get(u"gro\u00DF"))
 877         d.addCallback(lambda ign: self.handler.removeFile(u"gro\u00DF".encode('utf-8')))
 878         d.addCallback(lambda ign:
 879                       self.shouldFail(NoSuchChildError, "removeFile gross", "gro\\xdf",
 880                                       self.root.get, u"gro\u00DF"))
 881
 882         # removing an unknown should succeed
 883         d.addCallback(lambda ign: self.root.get(u"unknown"))
 884         d.addCallback(lambda ign: self.handler.removeFile("unknown"))
 885         d.addCallback(lambda ign:
 886                       self.shouldFail(NoSuchChildError, "removeFile unknown", "unknown",
 887                                       self.root.get, u"unknown"))
 888
 889         # removing a link to an open file should not prevent it from being read
 890         d.addCallback(lambda ign: self.handler.openFile("small", sftp.FXF_READ, {}))
 891         def _remove_and_read_small(rf):
 892             d2 = self.handler.removeFile("small")
 893             d2.addCallback(lambda ign:
 894                            self.shouldFail(NoSuchChildError, "removeFile small", "small",
 895                                            self.root.get, u"small"))
 896             d2.addCallback(lambda ign: rf.readChunk(0, 10))
 897             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 898             d2.addCallback(lambda ign: rf.close())
 899             return d2
 900         d.addCallback(_remove_and_read_small)
 901
 902         # removing a link to a created file should prevent it from being created
 903         d.addCallback(lambda ign: self.handler.openFile("tempfile", sftp.FXF_READ | sftp.FXF_WRITE |
 904                                                                     sftp.FXF_CREAT, {}))
 905         def _write_remove(rwf):
 906             d2 = rwf.writeChunk(0, "0123456789")
 907             d2.addCallback(lambda ign: self.handler.removeFile("tempfile"))
 908             d2.addCallback(lambda ign: rwf.readChunk(0, 10))
 909             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 910             d2.addCallback(lambda ign: rwf.close())
 911             return d2
 912         d.addCallback(_write_remove)
 913         d.addCallback(lambda ign:
 914                       self.shouldFail(NoSuchChildError, "removeFile tempfile", "tempfile",
 915                                       self.root.get, u"tempfile"))
 916
 917         # ... even if the link is renamed while open
 918         d.addCallback(lambda ign: self.handler.openFile("tempfile2", sftp.FXF_READ | sftp.FXF_WRITE |
 919                                                                      sftp.FXF_CREAT, {}))
 920         def _write_rename_remove(rwf):
 921             d2 = rwf.writeChunk(0, "0123456789")
 922             d2.addCallback(lambda ign: self.handler.renameFile("tempfile2", "tempfile3"))
 923             d2.addCallback(lambda ign: self.handler.removeFile("tempfile3"))
 924             d2.addCallback(lambda ign: rwf.readChunk(0, 10))
 925             d2.addCallback(lambda data: self.failUnlessReallyEqual(data, "0123456789"))
 926             d2.addCallback(lambda ign: rwf.close())
 927             return d2
 928         d.addCallback(_write_rename_remove)
 929         d.addCallback(lambda ign:
 930                       self.shouldFail(NoSuchChildError, "removeFile tempfile2", "tempfile2",
 931                                       self.root.get, u"tempfile2"))
 932         d.addCallback(lambda ign:
 933                       self.shouldFail(NoSuchChildError, "removeFile tempfile3", "tempfile3",
 934                                       self.root.get, u"tempfile3"))
 935
 936         return d
 937
 938     def test_removeDirectory(self):
 939         d = self._set_up("removeDirectory")
 940         d.addCallback(lambda ign: self._set_up_tree())
 941
 942         d.addCallback(lambda ign:
 943             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeDirectory nodir",
 944                                          self.handler.removeDirectory, "nodir"))
 945         d.addCallback(lambda ign:
 946             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeDirectory nodir/nodir",
 947                                          self.handler.removeDirectory, "nodir/nodir"))
 948         d.addCallback(lambda ign:
 949             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeDirectory ''",
 950                                          self.handler.removeDirectory, ""))
 951
 952         # removing a file should fail
 953         d.addCallback(lambda ign:
 954             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "removeDirectory gross",
 955                                          self.handler.removeDirectory, u"gro\u00DF".encode('utf-8')))
 956
 957         # removing a directory should succeed
 958         d.addCallback(lambda ign: self.root.get(u"tiny_lit_dir"))
 959         d.addCallback(lambda ign: self.handler.removeDirectory("tiny_lit_dir"))
 960         d.addCallback(lambda ign:
 961                       self.shouldFail(NoSuchChildError, "removeDirectory tiny_lit_dir", "tiny_lit_dir",
 962                                       self.root.get, u"tiny_lit_dir"))
 963
 964         # removing an unknown should succeed
 965         d.addCallback(lambda ign: self.root.get(u"unknown"))
 966         d.addCallback(lambda ign: self.handler.removeDirectory("unknown"))
 967         d.addCallback(lambda err:
 968                       self.shouldFail(NoSuchChildError, "removeDirectory unknown", "unknown",
 969                                       self.root.get, u"unknown"))
 970
 971         return d
 972
 973     def test_renameFile(self):
 974         d = self._set_up("renameFile")
 975         d.addCallback(lambda ign: self._set_up_tree())
 976
 977         # renaming a non-existent file should fail
 978         d.addCallback(lambda ign:
 979             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile nofile newfile",
 980                                          self.handler.renameFile, "nofile", "newfile"))
 981         d.addCallback(lambda ign:
 982             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile '' newfile",
 983                                          self.handler.renameFile, "", "newfile"))
 984
 985         # renaming a file to a non-existent path should fail
 986         d.addCallback(lambda ign:
 987             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small nodir/small",
 988                                          self.handler.renameFile, "small", "nodir/small"))
 989
 990         # renaming a file to an invalid UTF-8 name should fail
 991         d.addCallback(lambda ign:
 992             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small invalid",
 993                                          self.handler.renameFile, "small", "\xFF"))
 994
 995         # renaming a file to or from an URI should fail
 996         d.addCallback(lambda ign:
 997             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small from uri",
 998                                          self.handler.renameFile, "uri/"+self.small_uri, "new"))
 999         d.addCallback(lambda ign:
1000             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small to uri",
1001                                          self.handler.renameFile, "small", "uri/fake_uri"))
1002
1003         # renaming a file onto an existing file, directory or unknown should fail
1004         # The SFTP spec isn't clear about what error should be returned, but sshfs depends on
1005         # it being FX_PERMISSION_DENIED.
1006         d.addCallback(lambda ign:
1007             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small small2",
1008                                          self.handler.renameFile, "small", "small2"))
1009         d.addCallback(lambda ign:
1010             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small tiny_lit_dir",
1011                                          self.handler.renameFile, "small", "tiny_lit_dir"))
1012         d.addCallback(lambda ign:
1013             self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small unknown",
1014                                          self.handler.renameFile, "small", "unknown"))
1015
1016         # renaming a file to a correct path should succeed
1017         d.addCallback(lambda ign: self.handler.renameFile("small", "new_small"))
1018         d.addCallback(lambda ign: self.root.get(u"new_small"))
1019         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
1020
1021         # renaming a file into a subdirectory should succeed (also tests Unicode names)
1022         d.addCallback(lambda ign: self.handler.renameFile(u"gro\u00DF".encode('utf-8'),
1023                                                           u"loop/neue_gro\u00DF".encode('utf-8')))
1024         d.addCallback(lambda ign: self.root.get(u"neue_gro\u00DF"))
1025         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.gross_uri))
1026
1027         # renaming a directory to a correct path should succeed
1028         d.addCallback(lambda ign: self.handler.renameFile("tiny_lit_dir", "new_tiny_lit_dir"))
1029         d.addCallback(lambda ign: self.root.get(u"new_tiny_lit_dir"))
1030         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.tiny_lit_dir_uri))
1031
1032         # renaming an unknown to a correct path should succeed
1033         d.addCallback(lambda ign: self.handler.renameFile("unknown", "new_unknown"))
1034         d.addCallback(lambda ign: self.root.get(u"new_unknown"))
1035         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.unknown_uri))
1036
1037         return d
1038
1039     def test_renameFile_posix(self):
1040         def _renameFile(fromPathstring, toPathstring):
1041             extData = (struct.pack('>L', len(fromPathstring)) + fromPathstring +
1042                        struct.pack('>L', len(toPathstring))   + toPathstring)
1043
1044             d2 = self.handler.extendedRequest('posix-rename@openssh.com', extData)
1045             def _check(res):
1046                 res.trap(sftp.SFTPError)
1047                 if res.value.code == sftp.FX_OK:
1048                     return None
1049                 return res
1050             d2.addCallbacks(lambda res: self.fail("posix-rename request was supposed to "
1051                                                   "raise an SFTPError, not get '%r'" % (res,)),
1052                             _check)
1053             return d2
1054
1055         d = self._set_up("renameFile_posix")
1056         d.addCallback(lambda ign: self._set_up_tree())
1057
1058         d.addCallback(lambda ign: self.root.set_node(u"loop2", self.root))
1059         d.addCallback(lambda ign: self.root.set_node(u"unknown2", self.unknown))
1060
1061         # POSIX-renaming a non-existent file should fail
1062         d.addCallback(lambda ign:
1063             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix nofile newfile",
1064                                          _renameFile, "nofile", "newfile"))
1065         d.addCallback(lambda ign:
1066             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix '' newfile",
1067                                          _renameFile, "", "newfile"))
1068
1069         # POSIX-renaming a file to a non-existent path should fail
1070         d.addCallback(lambda ign:
1071             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix small nodir/small",
1072                                          _renameFile, "small", "nodir/small"))
1073
1074         # POSIX-renaming a file to an invalid UTF-8 name should fail
1075         d.addCallback(lambda ign:
1076             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix small invalid",
1077                                          _renameFile, "small", "\xFF"))
1078
1079         # POSIX-renaming a file to or from an URI should fail
1080         d.addCallback(lambda ign:
1081             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix small from uri",
1082                                          _renameFile, "uri/"+self.small_uri, "new"))
1083         d.addCallback(lambda ign:
1084             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile_posix small to uri",
1085                                          _renameFile, "small", "uri/fake_uri"))
1086
1087         # POSIX-renaming a file onto an existing file, directory or unknown should succeed
1088         d.addCallback(lambda ign: _renameFile("small", "small2"))
1089         d.addCallback(lambda ign: self.root.get(u"small2"))
1090         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
1091
1092         d.addCallback(lambda ign: _renameFile("small2", "loop2"))
1093         d.addCallback(lambda ign: self.root.get(u"loop2"))
1094         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
1095
1096         d.addCallback(lambda ign: _renameFile("loop2", "unknown2"))
1097         d.addCallback(lambda ign: self.root.get(u"unknown2"))
1098         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
1099
1100         # POSIX-renaming a file to a correct new path should succeed
1101         d.addCallback(lambda ign: _renameFile("unknown2", "new_small"))
1102         d.addCallback(lambda ign: self.root.get(u"new_small"))
1103         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.small_uri))
1104
1105         # POSIX-renaming a file into a subdirectory should succeed (also tests Unicode names)
1106         d.addCallback(lambda ign: _renameFile(u"gro\u00DF".encode('utf-8'),
1107                                               u"loop/neue_gro\u00DF".encode('utf-8')))
1108         d.addCallback(lambda ign: self.root.get(u"neue_gro\u00DF"))
1109         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.gross_uri))
1110
1111         # POSIX-renaming a directory to a correct path should succeed
1112         d.addCallback(lambda ign: _renameFile("tiny_lit_dir", "new_tiny_lit_dir"))
1113         d.addCallback(lambda ign: self.root.get(u"new_tiny_lit_dir"))
1114         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.tiny_lit_dir_uri))
1115
1116         # POSIX-renaming an unknown to a correct path should succeed
1117         d.addCallback(lambda ign: _renameFile("unknown", "new_unknown"))
1118         d.addCallback(lambda ign: self.root.get(u"new_unknown"))
1119         d.addCallback(lambda node: self.failUnlessReallyEqual(node.get_uri(), self.unknown_uri))
1120
1121         return d
1122
1123     def test_makeDirectory(self):
1124         d = self._set_up("makeDirectory")
1125         d.addCallback(lambda ign: self._set_up_tree())
1126
1127         # making a directory at a correct path should succeed
1128         d.addCallback(lambda ign: self.handler.makeDirectory("newdir", {'ext_foo': 'bar', 'ctime': 42}))
1129
1130         d.addCallback(lambda ign: self.root.get_child_and_metadata(u"newdir"))
1131         def _got( (child, metadata) ):
1132             self.failUnless(IDirectoryNode.providedBy(child))
1133             self.failUnless(child.is_mutable())
1134             # FIXME
1135             #self.failUnless('ctime' in metadata, metadata)
1136             #self.failUnlessReallyEqual(metadata['ctime'], 42)
1137             #self.failUnless('ext_foo' in metadata, metadata)
1138             #self.failUnlessReallyEqual(metadata['ext_foo'], 'bar')
1139             # TODO: child should be empty
1140         d.addCallback(_got)
1141
1142         # making intermediate directories should also succeed
1143         d.addCallback(lambda ign: self.handler.makeDirectory("newparent/newchild", {}))
1144
1145         d.addCallback(lambda ign: self.root.get(u"newparent"))
1146         def _got_newparent(newparent):
1147             self.failUnless(IDirectoryNode.providedBy(newparent))
1148             self.failUnless(newparent.is_mutable())
1149             return newparent.get(u"newchild")
1150         d.addCallback(_got_newparent)
1151
1152         def _got_newchild(newchild):
1153             self.failUnless(IDirectoryNode.providedBy(newchild))
1154             self.failUnless(newchild.is_mutable())
1155         d.addCallback(_got_newchild)
1156
1157         d.addCallback(lambda ign:
1158             self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "makeDirectory invalid UTF-8",
1159                                          self.handler.makeDirectory, "\xFF", {}))
1160
1161         # should fail because there is an existing file "small"
1162         d.addCallback(lambda ign:
1163             self.shouldFailWithSFTPError(sftp.FX_FAILURE, "makeDirectory small",
1164                                          self.handler.makeDirectory, "small", {}))
1165         return d
1166
1167     def test_execCommand_and_openShell(self):
1168         class FakeProtocol:
1169             def __init__(self):
1170                 self.output = ""
1171                 self.reason = None
1172             def write(self, data):
1173                 self.output += data
1174                 return defer.succeed(None)
1175             def processEnded(self, reason):
1176                 self.reason = reason
1177                 return defer.succeed(None)
1178
1179         d = self._set_up("execCommand_and_openShell")
1180
1181         d.addCallback(lambda ign: conch_interfaces.ISession(self.handler))
1182         def _exec_df(session):
1183             protocol = FakeProtocol()
1184             d2 = session.execCommand(protocol, "df -P -k /")
1185             d2.addCallback(lambda ign: self.failUnlessIn("1024-blocks", protocol.output))
1186             d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessDone)))
1187             d2.addCallback(lambda ign: session.eofReceived())
1188             d2.addCallback(lambda ign: session.closed())
1189             return d2
1190         d.addCallback(_exec_df)
1191
1192         d.addCallback(lambda ign: conch_interfaces.ISession(self.handler))
1193         def _exec_error(session):
1194             protocol = FakeProtocol()
1195             d2 = session.execCommand(protocol, "error")
1196             d2.addCallback(lambda ign: session.windowChanged(None))
1197             d2.addCallback(lambda ign: self.failUnlessEqual("", protocol.output))
1198             d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessTerminated)))
1199             d2.addCallback(lambda ign: self.failUnlessEqual(protocol.reason.value.exitCode, 1))
1200             d2.addCallback(lambda ign: session.closed())
1201             return d2
1202         d.addCallback(_exec_error)
1203
1204         d.addCallback(lambda ign: conch_interfaces.ISession(self.handler))
1205         def _openShell(session):
1206             protocol = FakeProtocol()
1207             d2 = session.openShell(protocol)
1208             d2.addCallback(lambda ign: self.failUnlessIn("only SFTP", protocol.output))
1209             d2.addCallback(lambda ign: self.failUnless(isinstance(protocol.reason.value, ProcessTerminated)))
1210             d2.addCallback(lambda ign: self.failUnlessEqual(protocol.reason.value.exitCode, 1))
1211             d2.addCallback(lambda ign: session.closed())
1212             return d2
1213         d.addCallback(_exec_error)
1214
1215         return d
1216
1217     def test_extendedRequest(self):
1218         d = self._set_up("extendedRequest")
1219
1220         d.addCallback(lambda ign: self.handler.extendedRequest("statvfs@openssh.com", "/"))
1221         def _check(res):
1222             self.failUnless(isinstance(res, str))
1223             self.failUnlessEqual(len(res), 8*11)
1224         d.addCallback(_check)
1225
1226         d.addCallback(lambda ign:
1227             self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "extendedRequest foo bar",
1228                                          self.handler.extendedRequest, "foo", "bar"))
1229
1230         return d