src/allmydata/test/test_cli_put.py

   1 import os.path
   2 from twisted.trial import unittest
   3 from twisted.python import usage
   4
   5 from allmydata.util import fileutil
   6 from allmydata.scripts.common import get_aliases
   7 from allmydata.scripts import cli
   8 from allmydata.test.no_network import GridTestMixin
   9 from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv
  10 from allmydata.util.fileutil import abspath_expanduser_unicode
  11 from .test_cli import CLITestMixin
  12
  13 timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s
  14
  15 class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
  16
  17     def test_unlinked_immutable_stdin(self):
  18         # tahoe get `echo DATA | tahoe put`
  19         # tahoe get `echo DATA | tahoe put -`
  20         self.basedir = "cli/Put/unlinked_immutable_stdin"
  21         DATA = "data" * 100
  22         self.set_up_grid()
  23         d = self.do_cli("put", stdin=DATA)
  24         def _uploaded(res):
  25             (rc, out, err) = res
  26             self.failUnlessIn("waiting for file data on stdin..", err)
  27             self.failUnlessIn("200 OK", err)
  28             self.readcap = out
  29             self.failUnless(self.readcap.startswith("URI:CHK:"))
  30         d.addCallback(_uploaded)
  31         d.addCallback(lambda res: self.do_cli("get", self.readcap))
  32         def _downloaded(res):
  33             (rc, out, err) = res
  34             self.failUnlessReallyEqual(err, "")
  35             self.failUnlessReallyEqual(out, DATA)
  36         d.addCallback(_downloaded)
  37         d.addCallback(lambda res: self.do_cli("put", "-", stdin=DATA))
  38         d.addCallback(lambda (rc, out, err):
  39                       self.failUnlessReallyEqual(out, self.readcap))
  40         return d
  41
  42     def test_unlinked_immutable_from_file(self):
  43         # tahoe put file.txt
  44         # tahoe put ./file.txt
  45         # tahoe put /tmp/file.txt
  46         # tahoe put ~/file.txt
  47         self.basedir = "cli/Put/unlinked_immutable_from_file"
  48         self.set_up_grid()
  49
  50         rel_fn = os.path.join(self.basedir, "DATAFILE")
  51         abs_fn = unicode_to_argv(abspath_expanduser_unicode(unicode(rel_fn)))
  52         # we make the file small enough to fit in a LIT file, for speed
  53         fileutil.write(rel_fn, "short file")
  54         d = self.do_cli("put", rel_fn)
  55         def _uploaded((rc, out, err)):
  56             readcap = out
  57             self.failUnless(readcap.startswith("URI:LIT:"), readcap)
  58             self.readcap = readcap
  59         d.addCallback(_uploaded)
  60         d.addCallback(lambda res: self.do_cli("put", "./" + rel_fn))
  61         d.addCallback(lambda (rc,stdout,stderr):
  62                       self.failUnlessReallyEqual(stdout, self.readcap))
  63         d.addCallback(lambda res: self.do_cli("put", abs_fn))
  64         d.addCallback(lambda (rc,stdout,stderr):
  65                       self.failUnlessReallyEqual(stdout, self.readcap))
  66         # we just have to assume that ~ is handled properly
  67         return d
  68
  69     def test_immutable_from_file(self):
  70         # tahoe put file.txt uploaded.txt
  71         # tahoe - uploaded.txt
  72         # tahoe put file.txt subdir/uploaded.txt
  73         # tahoe put file.txt tahoe:uploaded.txt
  74         # tahoe put file.txt tahoe:subdir/uploaded.txt
  75         # tahoe put file.txt DIRCAP:./uploaded.txt
  76         # tahoe put file.txt DIRCAP:./subdir/uploaded.txt
  77         self.basedir = "cli/Put/immutable_from_file"
  78         self.set_up_grid()
  79
  80         rel_fn = os.path.join(self.basedir, "DATAFILE")
  81         # we make the file small enough to fit in a LIT file, for speed
  82         DATA = "short file"
  83         DATA2 = "short file two"
  84         fileutil.write(rel_fn, DATA)
  85
  86         d = self.do_cli("create-alias", "tahoe")
  87
  88         d.addCallback(lambda res:
  89                       self.do_cli("put", rel_fn, "uploaded.txt"))
  90         def _uploaded((rc, out, err)):
  91             readcap = out.strip()
  92             self.failUnless(readcap.startswith("URI:LIT:"), readcap)
  93             self.failUnlessIn("201 Created", err)
  94             self.readcap = readcap
  95         d.addCallback(_uploaded)
  96         d.addCallback(lambda res:
  97                       self.do_cli("get", "tahoe:uploaded.txt"))
  98         d.addCallback(lambda (rc,stdout,stderr):
  99                       self.failUnlessReallyEqual(stdout, DATA))
 100
 101         d.addCallback(lambda res:
 102                       self.do_cli("put", "-", "uploaded.txt", stdin=DATA2))
 103         def _replaced((rc, out, err)):
 104             readcap = out.strip()
 105             self.failUnless(readcap.startswith("URI:LIT:"), readcap)
 106             self.failUnlessIn("200 OK", err)
 107         d.addCallback(_replaced)
 108
 109         d.addCallback(lambda res:
 110                       self.do_cli("put", rel_fn, "subdir/uploaded2.txt"))
 111         d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt"))
 112         d.addCallback(lambda (rc,stdout,stderr):
 113                       self.failUnlessReallyEqual(stdout, DATA))
 114
 115         d.addCallback(lambda res:
 116                       self.do_cli("put", rel_fn, "tahoe:uploaded3.txt"))
 117         d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt"))
 118         d.addCallback(lambda (rc,stdout,stderr):
 119                       self.failUnlessReallyEqual(stdout, DATA))
 120
 121         d.addCallback(lambda res:
 122                       self.do_cli("put", rel_fn, "tahoe:subdir/uploaded4.txt"))
 123         d.addCallback(lambda res:
 124                       self.do_cli("get", "tahoe:subdir/uploaded4.txt"))
 125         d.addCallback(lambda (rc,stdout,stderr):
 126                       self.failUnlessReallyEqual(stdout, DATA))
 127
 128         def _get_dircap(res):
 129             self.dircap = get_aliases(self.get_clientdir())["tahoe"]
 130         d.addCallback(_get_dircap)
 131
 132         d.addCallback(lambda res:
 133                       self.do_cli("put", rel_fn,
 134                                   self.dircap+":./uploaded5.txt"))
 135         d.addCallback(lambda res:
 136                       self.do_cli("get", "tahoe:uploaded5.txt"))
 137         d.addCallback(lambda (rc,stdout,stderr):
 138                       self.failUnlessReallyEqual(stdout, DATA))
 139
 140         d.addCallback(lambda res:
 141                       self.do_cli("put", rel_fn,
 142                                   self.dircap+":./subdir/uploaded6.txt"))
 143         d.addCallback(lambda res:
 144                       self.do_cli("get", "tahoe:subdir/uploaded6.txt"))
 145         d.addCallback(lambda (rc,stdout,stderr):
 146                       self.failUnlessReallyEqual(stdout, DATA))
 147
 148         return d
 149
 150     def test_mutable_unlinked(self):
 151         # FILECAP = `echo DATA | tahoe put --mutable`
 152         # tahoe get FILECAP, compare against DATA
 153         # echo DATA2 | tahoe put - FILECAP
 154         # tahoe get FILECAP, compare against DATA2
 155         # tahoe put file.txt FILECAP
 156         self.basedir = "cli/Put/mutable_unlinked"
 157         self.set_up_grid()
 158
 159         DATA = "data" * 100
 160         DATA2 = "two" * 100
 161         rel_fn = os.path.join(self.basedir, "DATAFILE")
 162         DATA3 = "three" * 100
 163         fileutil.write(rel_fn, DATA3)
 164
 165         d = self.do_cli("put", "--mutable", stdin=DATA)
 166         def _created(res):
 167             (rc, out, err) = res
 168             self.failUnlessIn("waiting for file data on stdin..", err)
 169             self.failUnlessIn("200 OK", err)
 170             self.filecap = out
 171             self.failUnless(self.filecap.startswith("URI:SSK:"), self.filecap)
 172         d.addCallback(_created)
 173         d.addCallback(lambda res: self.do_cli("get", self.filecap))
 174         d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA))
 175
 176         d.addCallback(lambda res: self.do_cli("put", "-", self.filecap, stdin=DATA2))
 177         def _replaced(res):
 178             (rc, out, err) = res
 179             self.failUnlessIn("waiting for file data on stdin..", err)
 180             self.failUnlessIn("200 OK", err)
 181             self.failUnlessReallyEqual(self.filecap, out)
 182         d.addCallback(_replaced)
 183         d.addCallback(lambda res: self.do_cli("get", self.filecap))
 184         d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA2))
 185
 186         d.addCallback(lambda res: self.do_cli("put", rel_fn, self.filecap))
 187         def _replaced2(res):
 188             (rc, out, err) = res
 189             self.failUnlessIn("200 OK", err)
 190             self.failUnlessReallyEqual(self.filecap, out)
 191         d.addCallback(_replaced2)
 192         d.addCallback(lambda res: self.do_cli("get", self.filecap))
 193         d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA3))
 194
 195         return d
 196
 197     def test_mutable(self):
 198         # echo DATA1 | tahoe put --mutable - uploaded.txt
 199         # echo DATA2 | tahoe put - uploaded.txt # should modify-in-place
 200         # tahoe get uploaded.txt, compare against DATA2
 201
 202         self.basedir = "cli/Put/mutable"
 203         self.set_up_grid()
 204
 205         DATA1 = "data" * 100
 206         fn1 = os.path.join(self.basedir, "DATA1")
 207         fileutil.write(fn1, DATA1)
 208         DATA2 = "two" * 100
 209         fn2 = os.path.join(self.basedir, "DATA2")
 210         fileutil.write(fn2, DATA2)
 211
 212         d = self.do_cli("create-alias", "tahoe")
 213         d.addCallback(lambda res:
 214                       self.do_cli("put", "--mutable", fn1, "tahoe:uploaded.txt"))
 215         def _check(res):
 216             (rc, out, err) = res
 217             self.failUnlessEqual(rc, 0, str(res))
 218             self.failUnlessEqual(err.strip(), "201 Created", str(res))
 219             self.uri = out
 220         d.addCallback(_check)
 221         d.addCallback(lambda res:
 222                       self.do_cli("put", fn2, "tahoe:uploaded.txt"))
 223         def _check2(res):
 224             (rc, out, err) = res
 225             self.failUnlessEqual(rc, 0, str(res))
 226             self.failUnlessEqual(err.strip(), "200 OK", str(res))
 227             self.failUnlessEqual(out, self.uri, str(res))
 228         d.addCallback(_check2)
 229         d.addCallback(lambda res:
 230                       self.do_cli("get", "tahoe:uploaded.txt"))
 231         d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA2))
 232         return d
 233
 234     def _check_mdmf_json(self, (rc, json, err)):
 235          self.failUnlessEqual(rc, 0)
 236          self.failUnlessEqual(err, "")
 237          self.failUnlessIn('"format": "MDMF"', json)
 238          # We also want a valid MDMF cap to be in the json.
 239          self.failUnlessIn("URI:MDMF", json)
 240          self.failUnlessIn("URI:MDMF-RO", json)
 241          self.failUnlessIn("URI:MDMF-Verifier", json)
 242
 243     def _check_sdmf_json(self, (rc, json, err)):
 244         self.failUnlessEqual(rc, 0)
 245         self.failUnlessEqual(err, "")
 246         self.failUnlessIn('"format": "SDMF"', json)
 247         # We also want to see the appropriate SDMF caps.
 248         self.failUnlessIn("URI:SSK", json)
 249         self.failUnlessIn("URI:SSK-RO", json)
 250         self.failUnlessIn("URI:SSK-Verifier", json)
 251
 252     def _check_chk_json(self, (rc, json, err)):
 253         self.failUnlessEqual(rc, 0)
 254         self.failUnlessEqual(err, "")
 255         self.failUnlessIn('"format": "CHK"', json)
 256         # We also want to see the appropriate CHK caps.
 257         self.failUnlessIn("URI:CHK", json)
 258         self.failUnlessIn("URI:CHK-Verifier", json)
 259
 260     def test_format(self):
 261         self.basedir = "cli/Put/format"
 262         self.set_up_grid()
 263         data = "data" * 40000 # 160kB total, two segments
 264         fn1 = os.path.join(self.basedir, "data")
 265         fileutil.write(fn1, data)
 266         d = self.do_cli("create-alias", "tahoe")
 267
 268         def _put_and_ls(ign, cmdargs, expected, filename=None):
 269             if filename:
 270                 args = ["put"] + cmdargs + [fn1, filename]
 271             else:
 272                 # unlinked
 273                 args = ["put"] + cmdargs + [fn1]
 274             d2 = self.do_cli(*args)
 275             def _list((rc, out, err)):
 276                 self.failUnlessEqual(rc, 0) # don't allow failure
 277                 if filename:
 278                     return self.do_cli("ls", "--json", filename)
 279                 else:
 280                     cap = out.strip()
 281                     return self.do_cli("ls", "--json", cap)
 282             d2.addCallback(_list)
 283             return d2
 284
 285         # 'tahoe put' to a directory
 286         d.addCallback(_put_and_ls, ["--mutable"], "SDMF", "tahoe:s1.txt")
 287         d.addCallback(self._check_sdmf_json) # backwards-compatibility
 288         d.addCallback(_put_and_ls, ["--format=SDMF"], "SDMF", "tahoe:s2.txt")
 289         d.addCallback(self._check_sdmf_json)
 290         d.addCallback(_put_and_ls, ["--format=sdmf"], "SDMF", "tahoe:s3.txt")
 291         d.addCallback(self._check_sdmf_json)
 292         d.addCallback(_put_and_ls, ["--mutable", "--format=SDMF"], "SDMF", "tahoe:s4.txt")
 293         d.addCallback(self._check_sdmf_json)
 294
 295         d.addCallback(_put_and_ls, ["--format=MDMF"], "MDMF", "tahoe:m1.txt")
 296         d.addCallback(self._check_mdmf_json)
 297         d.addCallback(_put_and_ls, ["--mutable", "--format=MDMF"], "MDMF", "tahoe:m2.txt")
 298         d.addCallback(self._check_mdmf_json)
 299
 300         d.addCallback(_put_and_ls, ["--format=CHK"], "CHK", "tahoe:c1.txt")
 301         d.addCallback(self._check_chk_json)
 302         d.addCallback(_put_and_ls, [], "CHK", "tahoe:c1.txt")
 303         d.addCallback(self._check_chk_json)
 304
 305         # 'tahoe put' unlinked
 306         d.addCallback(_put_and_ls, ["--mutable"], "SDMF")
 307         d.addCallback(self._check_sdmf_json) # backwards-compatibility
 308         d.addCallback(_put_and_ls, ["--format=SDMF"], "SDMF")
 309         d.addCallback(self._check_sdmf_json)
 310         d.addCallback(_put_and_ls, ["--format=sdmf"], "SDMF")
 311         d.addCallback(self._check_sdmf_json)
 312         d.addCallback(_put_and_ls, ["--mutable", "--format=SDMF"], "SDMF")
 313         d.addCallback(self._check_sdmf_json)
 314
 315         d.addCallback(_put_and_ls, ["--format=MDMF"], "MDMF")
 316         d.addCallback(self._check_mdmf_json)
 317         d.addCallback(_put_and_ls, ["--mutable", "--format=MDMF"], "MDMF")
 318         d.addCallback(self._check_mdmf_json)
 319
 320         d.addCallback(_put_and_ls, ["--format=CHK"], "CHK")
 321         d.addCallback(self._check_chk_json)
 322         d.addCallback(_put_and_ls, [], "CHK")
 323         d.addCallback(self._check_chk_json)
 324
 325         return d
 326
 327     def test_put_to_mdmf_cap(self):
 328         self.basedir = "cli/Put/put_to_mdmf_cap"
 329         self.set_up_grid()
 330         data = "data" * 100000
 331         fn1 = os.path.join(self.basedir, "data")
 332         fileutil.write(fn1, data)
 333         d = self.do_cli("put", "--format=MDMF", fn1)
 334         def _got_cap((rc, out, err)):
 335             self.failUnlessEqual(rc, 0)
 336             self.cap = out.strip()
 337         d.addCallback(_got_cap)
 338         # Now try to write something to the cap using put.
 339         data2 = "data2" * 100000
 340         fn2 = os.path.join(self.basedir, "data2")
 341         fileutil.write(fn2, data2)
 342         d.addCallback(lambda ignored:
 343             self.do_cli("put", fn2, self.cap))
 344         def _got_put((rc, out, err)):
 345             self.failUnlessEqual(rc, 0)
 346             self.failUnlessIn(self.cap, out)
 347         d.addCallback(_got_put)
 348         # Now get the cap. We should see the data we just put there.
 349         d.addCallback(lambda ignored:
 350             self.do_cli("get", self.cap))
 351         def _got_data((rc, out, err)):
 352             self.failUnlessEqual(rc, 0)
 353             self.failUnlessEqual(out, data2)
 354         d.addCallback(_got_data)
 355         # add some extension information to the cap and try to put something
 356         # to it.
 357         def _make_extended_cap(ignored):
 358             self.cap = self.cap + ":Extension-Stuff"
 359         d.addCallback(_make_extended_cap)
 360         data3 = "data3" * 100000
 361         fn3 = os.path.join(self.basedir, "data3")
 362         fileutil.write(fn3, data3)
 363         d.addCallback(lambda ignored:
 364             self.do_cli("put", fn3, self.cap))
 365         d.addCallback(lambda ignored:
 366             self.do_cli("get", self.cap))
 367         def _got_data3((rc, out, err)):
 368             self.failUnlessEqual(rc, 0)
 369             self.failUnlessEqual(out, data3)
 370         d.addCallback(_got_data3)
 371         return d
 372
 373     def test_put_to_sdmf_cap(self):
 374         self.basedir = "cli/Put/put_to_sdmf_cap"
 375         self.set_up_grid()
 376         data = "data" * 100000
 377         fn1 = os.path.join(self.basedir, "data")
 378         fileutil.write(fn1, data)
 379         d = self.do_cli("put", "--format=SDMF", fn1)
 380         def _got_cap((rc, out, err)):
 381             self.failUnlessEqual(rc, 0)
 382             self.cap = out.strip()
 383         d.addCallback(_got_cap)
 384         # Now try to write something to the cap using put.
 385         data2 = "data2" * 100000
 386         fn2 = os.path.join(self.basedir, "data2")
 387         fileutil.write(fn2, data2)
 388         d.addCallback(lambda ignored:
 389             self.do_cli("put", fn2, self.cap))
 390         def _got_put((rc, out, err)):
 391             self.failUnlessEqual(rc, 0)
 392             self.failUnlessIn(self.cap, out)
 393         d.addCallback(_got_put)
 394         # Now get the cap. We should see the data we just put there.
 395         d.addCallback(lambda ignored:
 396             self.do_cli("get", self.cap))
 397         def _got_data((rc, out, err)):
 398             self.failUnlessEqual(rc, 0)
 399             self.failUnlessEqual(out, data2)
 400         d.addCallback(_got_data)
 401         return d
 402
 403     def test_mutable_type_invalid_format(self):
 404         o = cli.PutOptions()
 405         self.failUnlessRaises(usage.UsageError,
 406                               o.parseOptions,
 407                               ["--format=LDMF"])
 408
 409     def test_put_with_nonexistent_alias(self):
 410         # when invoked with an alias that doesn't exist, 'tahoe put'
 411         # should output a useful error message, not a stack trace
 412         self.basedir = "cli/Put/put_with_nonexistent_alias"
 413         self.set_up_grid()
 414         d = self.do_cli("put", "somefile", "fake:afile")
 415         def _check((rc, out, err)):
 416             self.failUnlessReallyEqual(rc, 1)
 417             self.failUnlessIn("error:", err)
 418             self.failUnlessReallyEqual(out, "")
 419         d.addCallback(_check)
 420         return d
 421
 422     def test_immutable_from_file_unicode(self):
 423         # tahoe put "\u00E0 trier.txt" "\u00E0 trier.txt"
 424
 425         try:
 426             a_trier_arg = u"\u00E0 trier.txt".encode(get_io_encoding())
 427         except UnicodeEncodeError:
 428             raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
 429
 430         self.skip_if_cannot_represent_filename(u"\u00E0 trier.txt")
 431
 432         self.basedir = "cli/Put/immutable_from_file_unicode"
 433         self.set_up_grid()
 434
 435         rel_fn = os.path.join(unicode(self.basedir), u"\u00E0 trier.txt")
 436         # we make the file small enough to fit in a LIT file, for speed
 437         DATA = "short file"
 438         fileutil.write(rel_fn, DATA)
 439
 440         d = self.do_cli("create-alias", "tahoe")
 441
 442         d.addCallback(lambda res:
 443                       self.do_cli("put", rel_fn.encode(get_io_encoding()), a_trier_arg))
 444         def _uploaded((rc, out, err)):
 445             readcap = out.strip()
 446             self.failUnless(readcap.startswith("URI:LIT:"), readcap)
 447             self.failUnlessIn("201 Created", err)
 448             self.readcap = readcap
 449         d.addCallback(_uploaded)
 450
 451         d.addCallback(lambda res:
 452                       self.do_cli("get", "tahoe:" + a_trier_arg))
 453         d.addCallback(lambda (rc, out, err):
 454                       self.failUnlessReallyEqual(out, DATA))
 455
 456         return d
 457