src/allmydata/scripts/tahoe_cp.py

   1
   2 import os.path
   3 import urllib
   4 import simplejson
   5 from cStringIO import StringIO
   6 from twisted.python.failure import Failure
   7 from allmydata.scripts.common import get_alias, escape_path, \
   8                                      DefaultAliasMarker, TahoeError
   9 from allmydata.scripts.common_http import do_http, HTTPError
  10 from allmydata import uri
  11 from allmydata.util import fileutil
  12 from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath
  13 from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \
  14     quote_local_unicode_path, to_str
  15 from allmydata.util.assertutil import precondition
  16
  17
  18 class MissingSourceError(TahoeError):
  19     def __init__(self, name, quotefn=quote_output):
  20         TahoeError.__init__(self, "No such file or directory %s" % quotefn(name))
  21
  22
  23 def GET_to_file(url):
  24     resp = do_http("GET", url)
  25     if resp.status == 200:
  26         return resp
  27     raise HTTPError("Error during GET", resp)
  28
  29 def GET_to_string(url):
  30     f = GET_to_file(url)
  31     return f.read()
  32
  33 def PUT(url, data):
  34     resp = do_http("PUT", url, data)
  35     if resp.status in (200, 201):
  36         return resp.read()
  37     raise HTTPError("Error during PUT", resp)
  38
  39 def POST(url, data):
  40     resp = do_http("POST", url, data)
  41     if resp.status in (200, 201):
  42         return resp.read()
  43     raise HTTPError("Error during POST", resp)
  44
  45 def mkdir(targeturl):
  46     url = targeturl + "?t=mkdir"
  47     resp = do_http("POST", url)
  48     if resp.status in (200, 201):
  49         return resp.read().strip()
  50     raise HTTPError("Error during mkdir", resp)
  51
  52 def make_tahoe_subdirectory(nodeurl, parent_writecap, name):
  53     url = nodeurl + "/".join(["uri",
  54                               urllib.quote(parent_writecap),
  55                               urllib.quote(unicode_to_url(name)),
  56                               ]) + "?t=mkdir"
  57     resp = do_http("POST", url)
  58     if resp.status in (200, 201):
  59         return resp.read().strip()
  60     raise HTTPError("Error during mkdir", resp)
  61
  62
  63 class LocalFileSource:
  64     def __init__(self, pathname, basename):
  65         precondition_abspath(pathname)
  66         self.pathname = pathname
  67         self._basename = basename
  68
  69     def basename(self):
  70         return self._basename
  71
  72     def need_to_copy_bytes(self):
  73         return True
  74
  75     def open(self, caps_only):
  76         return open(self.pathname, "rb")
  77
  78 class LocalFileTarget:
  79     def __init__(self, pathname):
  80         precondition_abspath(pathname)
  81         self.pathname = pathname
  82
  83     def put_file(self, inf):
  84         fileutil.put_file(self.pathname, inf)
  85
  86 class LocalMissingTarget:
  87     def __init__(self, pathname):
  88         precondition_abspath(pathname)
  89         self.pathname = pathname
  90
  91     def put_file(self, inf):
  92         fileutil.put_file(self.pathname, inf)
  93
  94 class LocalDirectorySource:
  95     def __init__(self, progressfunc, pathname, basename):
  96         precondition_abspath(pathname)
  97
  98         self.progressfunc = progressfunc
  99         self.pathname = pathname
 100         self.children = None
 101         self._basename = basename
 102
 103     def basename(self):
 104         return self._basename
 105
 106     def populate(self, recurse):
 107         if self.children is not None:
 108             return
 109         self.children = {}
 110         children = listdir_unicode(self.pathname)
 111         for i,n in enumerate(children):
 112             self.progressfunc("examining %d of %d" % (i+1, len(children)))
 113             pn = os.path.join(self.pathname, n)
 114             if os.path.isdir(pn):
 115                 child = LocalDirectorySource(self.progressfunc, pn, n)
 116                 self.children[n] = child
 117                 if recurse:
 118                     child.populate(True)
 119             elif os.path.isfile(pn):
 120                 self.children[n] = LocalFileSource(pn, n)
 121             else:
 122                 # Could be dangling symlink; probably not copy-able.
 123                 # TODO: output a warning
 124                 pass
 125
 126 class LocalDirectoryTarget:
 127     def __init__(self, progressfunc, pathname):
 128         precondition_abspath(pathname)
 129
 130         self.progressfunc = progressfunc
 131         self.pathname = pathname
 132         self.children = None
 133
 134     def populate(self, recurse):
 135         if self.children is not None:
 136             return
 137         self.children = {}
 138         children = listdir_unicode(self.pathname)
 139         for i,n in enumerate(children):
 140             self.progressfunc("examining %d of %d" % (i+1, len(children)))
 141             pn = os.path.join(self.pathname, n)
 142             if os.path.isdir(pn):
 143                 child = LocalDirectoryTarget(self.progressfunc, pn)
 144                 self.children[n] = child
 145                 if recurse:
 146                     child.populate(True)
 147             else:
 148                 assert os.path.isfile(pn)
 149                 self.children[n] = LocalFileTarget(pn)
 150
 151     def get_child_target(self, name):
 152         precondition(isinstance(name, unicode), name)
 153         if self.children is None:
 154             self.populate(False)
 155         if name in self.children:
 156             return self.children[name]
 157         pathname = os.path.join(self.pathname, name)
 158         os.makedirs(pathname)
 159         return LocalDirectoryTarget(self.progressfunc, pathname)
 160
 161     def put_file(self, name, inf):
 162         precondition(isinstance(name, unicode), name)
 163         pathname = os.path.join(self.pathname, name)
 164         fileutil.put_file(pathname, inf)
 165
 166     def set_children(self):
 167         pass
 168
 169
 170 class TahoeFileSource:
 171     def __init__(self, nodeurl, mutable, writecap, readcap, basename):
 172         self.nodeurl = nodeurl
 173         self.mutable = mutable
 174         self.writecap = writecap
 175         self.readcap = readcap
 176         self._basename = basename # unicode, or None for raw filecaps
 177
 178     def basename(self):
 179         return self._basename
 180
 181     def need_to_copy_bytes(self):
 182         if self.mutable:
 183             return True
 184         return False
 185
 186     def open(self, caps_only):
 187         if caps_only:
 188             return StringIO(self.readcap)
 189         url = self.nodeurl + "uri/" + urllib.quote(self.readcap)
 190         return GET_to_file(url)
 191
 192     def bestcap(self):
 193         return self.writecap or self.readcap
 194
 195 class TahoeFileTarget:
 196     def __init__(self, nodeurl, mutable, writecap, readcap, url):
 197         self.nodeurl = nodeurl
 198         self.mutable = mutable
 199         self.writecap = writecap
 200         self.readcap = readcap
 201         self.url = url
 202
 203     def put_file(self, inf):
 204         # We want to replace this object in-place.
 205         assert self.url
 206         # our do_http() call currently requires a string or a filehandle with
 207         # a real .seek
 208         if not hasattr(inf, "seek"):
 209             inf = inf.read()
 210         PUT(self.url, inf)
 211         # TODO: this always creates immutable files. We might want an option
 212         # to always create mutable files, or to copy mutable files into new
 213         # mutable files. ticket #835
 214
 215 class TahoeDirectorySource:
 216     def __init__(self, nodeurl, cache, progressfunc, basename):
 217         self.nodeurl = nodeurl
 218         self.cache = cache
 219         self.progressfunc = progressfunc
 220         self._basename = basename # unicode, or None for raw dircaps
 221
 222     def basename(self):
 223         return self._basename
 224
 225     def init_from_grid(self, writecap, readcap):
 226         self.writecap = writecap
 227         self.readcap = readcap
 228         bestcap = writecap or readcap
 229         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 230         resp = do_http("GET", url + "?t=json")
 231         if resp.status != 200:
 232             raise HTTPError("Error examining source directory", resp)
 233         parsed = simplejson.loads(resp.read())
 234         nodetype, d = parsed
 235         assert nodetype == "dirnode"
 236         self.mutable = d.get("mutable", False) # older nodes don't provide it
 237         self.children_d = dict( [(unicode(name),value)
 238                                  for (name,value)
 239                                  in d["children"].iteritems()] )
 240         self.children = None
 241
 242     def init_from_parsed(self, parsed):
 243         nodetype, d = parsed
 244         self.writecap = to_str(d.get("rw_uri"))
 245         self.readcap = to_str(d.get("ro_uri"))
 246         self.mutable = d.get("mutable", False) # older nodes don't provide it
 247         self.children_d = dict( [(unicode(name),value)
 248                                  for (name,value)
 249                                  in d["children"].iteritems()] )
 250         self.children = None
 251
 252     def populate(self, recurse):
 253         if self.children is not None:
 254             return
 255         self.children = {}
 256         for i,(name, data) in enumerate(self.children_d.items()):
 257             self.progressfunc("examining %d of %d" % (i+1, len(self.children_d)))
 258             if data[0] == "filenode":
 259                 mutable = data[1].get("mutable", False)
 260                 writecap = to_str(data[1].get("rw_uri"))
 261                 readcap = to_str(data[1].get("ro_uri"))
 262                 self.children[name] = TahoeFileSource(self.nodeurl, mutable,
 263                                                       writecap, readcap, name)
 264             elif data[0] == "dirnode":
 265                 writecap = to_str(data[1].get("rw_uri"))
 266                 readcap = to_str(data[1].get("ro_uri"))
 267                 if writecap and writecap in self.cache:
 268                     child = self.cache[writecap]
 269                 elif readcap and readcap in self.cache:
 270                     child = self.cache[readcap]
 271                 else:
 272                     child = TahoeDirectorySource(self.nodeurl, self.cache,
 273                                                  self.progressfunc, name)
 274                     child.init_from_grid(writecap, readcap)
 275                     if writecap:
 276                         self.cache[writecap] = child
 277                     if readcap:
 278                         self.cache[readcap] = child
 279                     if recurse:
 280                         child.populate(True)
 281                 self.children[name] = child
 282             else:
 283                 # TODO: there should be an option to skip unknown nodes.
 284                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 285                                  "You probably need to use a later version of "
 286                                  "Tahoe-LAFS to copy this directory.")
 287
 288 class TahoeMissingTarget:
 289     def __init__(self, url):
 290         self.url = url
 291
 292     def put_file(self, inf):
 293         # We want to replace this object in-place.
 294         if not hasattr(inf, "seek"):
 295             inf = inf.read()
 296         PUT(self.url, inf)
 297         # TODO: this always creates immutable files. We might want an option
 298         # to always create mutable files, or to copy mutable files into new
 299         # mutable files.
 300
 301     def put_uri(self, filecap):
 302         # I'm not sure this will always work
 303         return PUT(self.url + "?t=uri", filecap)
 304
 305 class TahoeDirectoryTarget:
 306     def __init__(self, nodeurl, cache, progressfunc):
 307         self.nodeurl = nodeurl
 308         self.cache = cache
 309         self.progressfunc = progressfunc
 310         self.new_children = {}
 311
 312     def init_from_parsed(self, parsed):
 313         nodetype, d = parsed
 314         self.writecap = to_str(d.get("rw_uri"))
 315         self.readcap = to_str(d.get("ro_uri"))
 316         self.mutable = d.get("mutable", False) # older nodes don't provide it
 317         self.children_d = dict( [(unicode(name),value)
 318                                  for (name,value)
 319                                  in d["children"].iteritems()] )
 320         self.children = None
 321
 322     def init_from_grid(self, writecap, readcap):
 323         self.writecap = writecap
 324         self.readcap = readcap
 325         bestcap = writecap or readcap
 326         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 327         resp = do_http("GET", url + "?t=json")
 328         if resp.status != 200:
 329             raise HTTPError("Error examining target directory", resp)
 330         parsed = simplejson.loads(resp.read())
 331         nodetype, d = parsed
 332         assert nodetype == "dirnode"
 333         self.mutable = d.get("mutable", False) # older nodes don't provide it
 334         self.children_d = dict( [(unicode(name),value)
 335                                  for (name,value)
 336                                  in d["children"].iteritems()] )
 337         self.children = None
 338
 339     def just_created(self, writecap):
 340         self.writecap = writecap
 341         self.readcap = uri.from_string(writecap).get_readonly().to_string()
 342         self.mutable = True
 343         self.children_d = {}
 344         self.children = {}
 345
 346     def populate(self, recurse):
 347         if self.children is not None:
 348             return
 349         self.children = {}
 350         for i,(name, data) in enumerate(self.children_d.items()):
 351             self.progressfunc("examining %d of %d" % (i+1, len(self.children_d)))
 352             if data[0] == "filenode":
 353                 mutable = data[1].get("mutable", False)
 354                 writecap = to_str(data[1].get("rw_uri"))
 355                 readcap = to_str(data[1].get("ro_uri"))
 356                 url = None
 357                 if self.writecap:
 358                     url = self.nodeurl + "/".join(["uri",
 359                                                    urllib.quote(self.writecap),
 360                                                    urllib.quote(unicode_to_url(name))])
 361                 self.children[name] = TahoeFileTarget(self.nodeurl, mutable,
 362                                                       writecap, readcap, url)
 363             elif data[0] == "dirnode":
 364                 writecap = to_str(data[1].get("rw_uri"))
 365                 readcap = to_str(data[1].get("ro_uri"))
 366                 if writecap and writecap in self.cache:
 367                     child = self.cache[writecap]
 368                 elif readcap and readcap in self.cache:
 369                     child = self.cache[readcap]
 370                 else:
 371                     child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 372                                                  self.progressfunc)
 373                     child.init_from_grid(writecap, readcap)
 374                     if writecap:
 375                         self.cache[writecap] = child
 376                     if readcap:
 377                         self.cache[readcap] = child
 378                     if recurse:
 379                         child.populate(True)
 380                 self.children[name] = child
 381             else:
 382                 # TODO: there should be an option to skip unknown nodes.
 383                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 384                                  "You probably need to use a later version of "
 385                                  "Tahoe-LAFS to copy this directory.")
 386
 387     def get_child_target(self, name):
 388         # return a new target for a named subdirectory of this dir
 389         precondition(isinstance(name, unicode), name)
 390         if self.children is None:
 391             self.populate(False)
 392         if name in self.children:
 393             return self.children[name]
 394         writecap = make_tahoe_subdirectory(self.nodeurl, self.writecap, name)
 395         child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 396                                      self.progressfunc)
 397         child.just_created(writecap)
 398         self.children[name] = child
 399         return child
 400
 401     def put_file(self, name, inf):
 402         precondition(isinstance(name, unicode), name)
 403         url = self.nodeurl + "uri"
 404         if not hasattr(inf, "seek"):
 405             inf = inf.read()
 406
 407         if self.children is None:
 408             self.populate(False)
 409
 410         # Check to see if we already have a mutable file by this name.
 411         # If so, overwrite that file in place.
 412         if name in self.children and self.children[name].mutable:
 413             self.children[name].put_file(inf)
 414         else:
 415             filecap = PUT(url, inf)
 416             # TODO: this always creates immutable files. We might want an option
 417             # to always create mutable files, or to copy mutable files into new
 418             # mutable files.
 419             self.new_children[name] = filecap
 420
 421     def put_uri(self, name, filecap):
 422         precondition(isinstance(name, unicode), name)
 423         self.new_children[name] = filecap
 424
 425     def set_children(self):
 426         if not self.new_children:
 427             return
 428         url = (self.nodeurl + "uri/" + urllib.quote(self.writecap)
 429                + "?t=set_children")
 430         set_data = {}
 431         for (name, filecap) in self.new_children.items():
 432             # it just so happens that ?t=set_children will accept both file
 433             # read-caps and write-caps as ['rw_uri'], and will handle either
 434             # correctly. So don't bother trying to figure out whether the one
 435             # we have is read-only or read-write.
 436             # TODO: think about how this affects forward-compatibility for
 437             # unknown caps
 438             set_data[name] = ["filenode", {"rw_uri": filecap}]
 439         body = simplejson.dumps(set_data)
 440         POST(url, body)
 441
 442 class Copier:
 443
 444     def do_copy(self, options, progressfunc=None):
 445         if options['quiet']:
 446             verbosity = 0
 447         elif options['verbose']:
 448             verbosity = 2
 449         else:
 450             verbosity = 1
 451
 452         nodeurl = options['node-url']
 453         if nodeurl[-1] != "/":
 454             nodeurl += "/"
 455         self.nodeurl = nodeurl
 456         self.progressfunc = progressfunc
 457         self.options = options
 458         self.aliases = options.aliases
 459         self.verbosity = verbosity
 460         self.stdout = options.stdout
 461         self.stderr = options.stderr
 462         if verbosity >= 2 and not self.progressfunc:
 463             def progress(message):
 464                 print >>self.stderr, message
 465             self.progressfunc = progress
 466         self.caps_only = options["caps-only"]
 467         self.cache = {}
 468         try:
 469             status = self.try_copy()
 470             return status
 471         except TahoeError, te:
 472             if verbosity >= 2:
 473                 Failure().printTraceback(self.stderr)
 474                 print >>self.stderr
 475             te.display(self.stderr)
 476             return 1
 477
 478     def try_copy(self):
 479         source_specs = self.options.sources
 480         destination_spec = self.options.destination
 481         recursive = self.options["recursive"]
 482
 483         target = self.get_target_info(destination_spec)
 484
 485         sources = [] # list of source objects
 486         for ss in source_specs:
 487             sources.append(self.get_source_info(ss))
 488
 489         have_source_dirs = any([isinstance(s, (LocalDirectorySource,
 490                                                TahoeDirectorySource))
 491                                 for s in sources])
 492
 493         if have_source_dirs and not recursive:
 494             self.to_stderr("cannot copy directories without --recursive")
 495             return 1
 496
 497         if isinstance(target, (LocalFileTarget, TahoeFileTarget)):
 498             # cp STUFF foo.txt, where foo.txt already exists. This limits the
 499             # possibilities considerably.
 500             if len(sources) > 1:
 501                 self.to_stderr("target %s is not a directory" % quote_output(destination_spec))
 502                 return 1
 503             if have_source_dirs:
 504                 self.to_stderr("cannot copy directory into a file")
 505                 return 1
 506             return self.copy_file(sources[0], target)
 507
 508         if isinstance(target, (LocalMissingTarget, TahoeMissingTarget)):
 509             if recursive:
 510                 return self.copy_to_directory(sources, target)
 511             if len(sources) > 1:
 512                 # if we have -r, we'll auto-create the target directory. Without
 513                 # it, we'll only create a file.
 514                 self.to_stderr("cannot copy multiple files into a file without -r")
 515                 return 1
 516             # cp file1 newfile
 517             return self.copy_file(sources[0], target)
 518
 519         if isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)):
 520             # We're copying to an existing directory -- make sure that we
 521             # have target names for everything
 522             for source in sources:
 523                 if source.basename() is None and isinstance(source, TahoeFileSource):
 524                     self.to_stderr(
 525                         "error: you must specify a destination filename")
 526                     return 1
 527             return self.copy_to_directory(sources, target)
 528
 529         self.to_stderr("unknown target")
 530         return 1
 531
 532     def to_stderr(self, text):
 533         print >>self.stderr, text
 534
 535     # FIXME reduce the amount of near-duplicate code between get_target_info
 536     # and get_source_info.
 537
 538     def get_target_info(self, destination_spec):
 539         precondition(isinstance(destination_spec, unicode), destination_spec)
 540         rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None)
 541         path = path_utf8.decode("utf-8")
 542         if rootcap == DefaultAliasMarker:
 543             # no alias, so this is a local file
 544             pathname = abspath_expanduser_unicode(path)
 545             if not os.path.exists(pathname):
 546                 t = LocalMissingTarget(pathname)
 547             elif os.path.isdir(pathname):
 548                 t = LocalDirectoryTarget(self.progress, pathname)
 549             else:
 550                 # TODO: should this be _assert? what happens if the target is
 551                 # a special file?
 552                 assert os.path.isfile(pathname), pathname
 553                 t = LocalFileTarget(pathname) # non-empty
 554         else:
 555             # this is a tahoe object
 556             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 557             if path:
 558                 url += "/" + escape_path(path)
 559
 560             resp = do_http("GET", url + "?t=json")
 561             if resp.status == 404:
 562                 # doesn't exist yet
 563                 t = TahoeMissingTarget(url)
 564             elif resp.status == 200:
 565                 parsed = simplejson.loads(resp.read())
 566                 nodetype, d = parsed
 567                 if nodetype == "dirnode":
 568                     t = TahoeDirectoryTarget(self.nodeurl, self.cache,
 569                                              self.progress)
 570                     t.init_from_parsed(parsed)
 571                 else:
 572                     writecap = to_str(d.get("rw_uri"))
 573                     readcap = to_str(d.get("ro_uri"))
 574                     mutable = d.get("mutable", False)
 575                     t = TahoeFileTarget(self.nodeurl, mutable,
 576                                         writecap, readcap, url)
 577             else:
 578                 raise HTTPError("Error examining target %s"
 579                                  % quote_output(destination_spec), resp)
 580         return t
 581
 582     def get_source_info(self, source_spec):
 583         precondition(isinstance(source_spec, unicode), source_spec)
 584         rootcap, path_utf8 = get_alias(self.aliases, source_spec, None)
 585         path = path_utf8.decode("utf-8")
 586         if rootcap == DefaultAliasMarker:
 587             # no alias, so this is a local file
 588             pathname = abspath_expanduser_unicode(path)
 589             name = os.path.basename(pathname)
 590             if not os.path.exists(pathname):
 591                 raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path)
 592             if os.path.isdir(pathname):
 593                 t = LocalDirectorySource(self.progress, pathname, name)
 594             else:
 595                 assert os.path.isfile(pathname)
 596                 t = LocalFileSource(pathname, name) # non-empty
 597         else:
 598             # this is a tahoe object
 599             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 600             name = None
 601             if path:
 602                 url += "/" + escape_path(path)
 603                 last_slash = path.rfind(u"/")
 604                 name = path
 605                 if last_slash != -1:
 606                     name = path[last_slash+1:]
 607
 608             resp = do_http("GET", url + "?t=json")
 609             if resp.status == 404:
 610                 raise MissingSourceError(source_spec)
 611             elif resp.status != 200:
 612                 raise HTTPError("Error examining source %s" % quote_output(source_spec),
 613                                 resp)
 614             parsed = simplejson.loads(resp.read())
 615             nodetype, d = parsed
 616             if nodetype == "dirnode":
 617                 t = TahoeDirectorySource(self.nodeurl, self.cache,
 618                                          self.progress, name)
 619                 t.init_from_parsed(parsed)
 620             else:
 621                 writecap = to_str(d.get("rw_uri"))
 622                 readcap = to_str(d.get("ro_uri"))
 623                 mutable = d.get("mutable", False) # older nodes don't provide it
 624
 625                 last_slash = source_spec.rfind(u"/")
 626                 if last_slash != -1:
 627                     # TODO: this looks funny and redundant with the 'name'
 628                     # assignment above. cf #2329
 629                     name = source_spec[last_slash+1:]
 630
 631                 t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name)
 632         return t
 633
 634
 635     def dump_graph(self, s, indent=" "):
 636         for name, child in s.children.items():
 637             print "%s%s: %r" % (indent, quote_output(name), child)
 638             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 639                 self.dump_graph(child, indent+"  ")
 640
 641     def copy_to_directory(self, sources, target):
 642         # step one: build a recursive graph of the source tree. This returns
 643         # a dictionary, with child names as keys, and values that are either
 644         # Directory or File instances (local or tahoe).
 645         source_dirs = self.build_graphs(sources)
 646         source_files = [s for s in sources
 647                         if isinstance(s, (LocalFileSource, TahoeFileSource))]
 648
 649         #print "graphs"
 650         #for s in source_dirs:
 651         #    self.dump_graph(s)
 652
 653         # step two: create the top-level target directory object
 654         if isinstance(target, LocalMissingTarget):
 655             os.makedirs(target.pathname)
 656             target = LocalDirectoryTarget(self.progress, target.pathname)
 657         elif isinstance(target, TahoeMissingTarget):
 658             writecap = mkdir(target.url)
 659             target = TahoeDirectoryTarget(self.nodeurl, self.cache,
 660                                           self.progress)
 661             target.just_created(writecap)
 662         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 663         target.populate(False)
 664
 665         # step three: find a target for each source node, creating
 666         # directories as necessary. 'targetmap' is a dictionary that uses
 667         # target Directory instances as keys, and has values of
 668         # (name->sourceobject) dicts for all the files that need to wind up
 669         # there.
 670
 671         # sources are all LocalFile/LocalDirectory/TahoeFile/TahoeDirectory
 672         # target is LocalDirectory/TahoeDirectory
 673
 674         self.progress("attaching sources to targets, "
 675                       "%d files / %d dirs in root" %
 676                       (len(source_files), len(source_dirs)))
 677
 678         self.targetmap = {}
 679         self.files_to_copy = 0
 680
 681         for s in source_files:
 682             self.attach_to_target(s, s.basename(), target)
 683
 684         for (name, source) in source_dirs:
 685             new_target = target.get_child_target(name)
 686             self.assign_targets(source, new_target)
 687
 688         self.progress("targets assigned, %s dirs, %s files" %
 689                       (len(self.targetmap), self.files_to_copy))
 690
 691         self.progress("starting copy, %d files, %d directories" %
 692                       (self.files_to_copy, len(self.targetmap)))
 693         self.files_copied = 0
 694         self.targets_finished = 0
 695
 696         # step four: walk through the list of targets. For each one, copy all
 697         # the files. If the target is a TahoeDirectory, upload and create
 698         # read-caps, then do a set_children to the target directory.
 699
 700         for target in self.targetmap:
 701             self.copy_files_to_target(self.targetmap[target], target)
 702             self.targets_finished += 1
 703             self.progress("%d/%d directories" %
 704                           (self.targets_finished, len(self.targetmap)))
 705
 706         return self.announce_success("files copied")
 707
 708     def attach_to_target(self, source, name, target):
 709         precondition(isinstance(name, unicode), name)
 710         if target not in self.targetmap:
 711             self.targetmap[target] = {}
 712         self.targetmap[target][name] = source
 713         self.files_to_copy += 1
 714
 715     def assign_targets(self, source, target):
 716         # copy everything in the source into the target
 717         precondition(isinstance(source, (LocalDirectorySource, TahoeDirectorySource)), source)
 718
 719         for name, child in source.children.items():
 720             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 721                 # we will need a target directory for this one
 722                 subtarget = target.get_child_target(name)
 723                 self.assign_targets(child, subtarget)
 724             else:
 725                 precondition(isinstance(child, (LocalFileSource, TahoeFileSource)), child)
 726                 self.attach_to_target(child, name, target)
 727
 728     def copy_files_to_target(self, targetmap, target):
 729         for name, source in targetmap.items():
 730             precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
 731             self.copy_file_into(source, name, target)
 732             self.files_copied += 1
 733             self.progress("%d/%d files, %d/%d directories" %
 734                           (self.files_copied, self.files_to_copy,
 735                            self.targets_finished, len(self.targetmap)))
 736         target.set_children()
 737
 738     def need_to_copy_bytes(self, source, target):
 739         if source.need_to_copy_bytes:
 740             # mutable tahoe files, and local files
 741             return True
 742         if isinstance(target, (LocalFileTarget, LocalDirectoryTarget)):
 743             return True
 744         return False
 745
 746     def announce_success(self, msg):
 747         if self.verbosity >= 1:
 748             print >>self.stdout, "Success: %s" % msg
 749         return 0
 750
 751     def copy_file(self, source, target):
 752         precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
 753         precondition(isinstance(target, (LocalFileTarget, TahoeFileTarget,
 754                                          LocalMissingTarget, TahoeMissingTarget)), target)
 755         if self.need_to_copy_bytes(source, target):
 756             # if the target is a local directory, this will just write the
 757             # bytes to disk. If it is a tahoe directory, it will upload the
 758             # data, and stash the new filecap for a later set_children call.
 759             f = source.open(self.caps_only)
 760             target.put_file(f)
 761             return self.announce_success("file copied")
 762         # otherwise we're copying tahoe to tahoe, and using immutable files,
 763         # so we can just make a link. TODO: this probably won't always work:
 764         # need to enumerate the cases and analyze them.
 765         target.put_uri(source.bestcap())
 766         return self.announce_success("file linked")
 767
 768     def copy_file_into(self, source, name, target):
 769         precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
 770         precondition(isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)), target)
 771         precondition(isinstance(name, unicode), name)
 772         if self.need_to_copy_bytes(source, target):
 773             # if the target is a local directory, this will just write the
 774             # bytes to disk. If it is a tahoe directory, it will upload the
 775             # data, and stash the new filecap for a later set_children call.
 776             f = source.open(self.caps_only)
 777             target.put_file(name, f)
 778             return
 779         # otherwise we're copying tahoe to tahoe, and using immutable files,
 780         # so we can just make a link
 781         target.put_uri(name, source.bestcap())
 782
 783
 784     def progress(self, message):
 785         #print message
 786         if self.progressfunc:
 787             self.progressfunc(message)
 788
 789     def build_graphs(self, sources):
 790         graphs = []
 791         for source in sources:
 792             if isinstance(source, (LocalDirectorySource, TahoeDirectorySource)):
 793                 source.populate(True)
 794                 graphs.append((source.basename(), source))
 795         return graphs
 796
 797
 798 def copy(options):
 799     return Copier().do_copy(options)
 800
 801 # error cases that need improvement:
 802 #  local-file-in-the-way
 803 #   touch proposed
 804 #   tahoe cp -r my:docs/proposed/denver.txt proposed/denver.txt
 805 #  handling of unknown nodes
 806
 807 # things that maybe should be errors but aren't
 808 #  local-dir-in-the-way
 809 #   mkdir denver.txt
 810 #   tahoe cp -r my:docs/proposed/denver.txt denver.txt
 811 #   (creates denver.txt/denver.txt)
 812
 813 # error cases that look good:
 814 #  tahoe cp -r my:docs/missing missing
 815 #  disconnect servers
 816 #   tahoe cp -r my:docs/missing missing  -> No JSON object could be decoded
 817 #  tahoe-file-in-the-way (when we want to make a directory)
 818 #   tahoe put README my:docs
 819 #   tahoe cp -r docs/proposed my:docs/proposed