src/allmydata/scripts/tahoe_cp.py

   1
   2 import os.path
   3 import urllib
   4 import simplejson
   5 from cStringIO import StringIO
   6 from twisted.python.failure import Failure
   7 from allmydata.scripts.common import get_alias, escape_path, \
   8                                      DefaultAliasMarker, TahoeError
   9 from allmydata.scripts.common_http import do_http, HTTPError
  10 from allmydata import uri
  11 from allmydata.util import fileutil
  12 from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath
  13 from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \
  14     quote_local_unicode_path, to_str
  15 from allmydata.util.assertutil import precondition
  16
  17
  18 class MissingSourceError(TahoeError):
  19     def __init__(self, name, quotefn=quote_output):
  20         TahoeError.__init__(self, "No such file or directory %s" % quotefn(name))
  21
  22
  23 def GET_to_file(url):
  24     resp = do_http("GET", url)
  25     if resp.status == 200:
  26         return resp
  27     raise HTTPError("Error during GET", resp)
  28
  29 def GET_to_string(url):
  30     f = GET_to_file(url)
  31     return f.read()
  32
  33 def PUT(url, data):
  34     resp = do_http("PUT", url, data)
  35     if resp.status in (200, 201):
  36         return resp.read()
  37     raise HTTPError("Error during PUT", resp)
  38
  39 def POST(url, data):
  40     resp = do_http("POST", url, data)
  41     if resp.status in (200, 201):
  42         return resp.read()
  43     raise HTTPError("Error during POST", resp)
  44
  45 def mkdir(targeturl):
  46     url = targeturl + "?t=mkdir"
  47     resp = do_http("POST", url)
  48     if resp.status in (200, 201):
  49         return resp.read().strip()
  50     raise HTTPError("Error during mkdir", resp)
  51
  52 def make_tahoe_subdirectory(nodeurl, parent_writecap, name):
  53     url = nodeurl + "/".join(["uri",
  54                               urllib.quote(parent_writecap),
  55                               urllib.quote(unicode_to_url(name)),
  56                               ]) + "?t=mkdir"
  57     resp = do_http("POST", url)
  58     if resp.status in (200, 201):
  59         return resp.read().strip()
  60     raise HTTPError("Error during mkdir", resp)
  61
  62
  63 class LocalFileSource:
  64     def __init__(self, pathname, basename):
  65         precondition_abspath(pathname)
  66         self.pathname = pathname
  67         self._basename = basename
  68
  69     def basename(self):
  70         return self._basename
  71
  72     def need_to_copy_bytes(self):
  73         return True
  74
  75     def open(self, caps_only):
  76         return open(self.pathname, "rb")
  77
  78 class LocalFileTarget:
  79     def __init__(self, pathname):
  80         precondition_abspath(pathname)
  81         self.pathname = pathname
  82
  83     def put_file(self, inf):
  84         fileutil.put_file(self.pathname, inf)
  85
  86 class LocalMissingTarget:
  87     def __init__(self, pathname):
  88         precondition_abspath(pathname)
  89         self.pathname = pathname
  90
  91     def put_file(self, inf):
  92         fileutil.put_file(self.pathname, inf)
  93
  94 class LocalDirectorySource:
  95     def __init__(self, progressfunc, pathname, basename):
  96         precondition_abspath(pathname)
  97
  98         self.progressfunc = progressfunc
  99         self.pathname = pathname
 100         self.children = None
 101         self._basename = basename
 102
 103     def basename(self):
 104         return self._basename
 105
 106     def populate(self, recurse):
 107         if self.children is not None:
 108             return
 109         self.children = {}
 110         children = listdir_unicode(self.pathname)
 111         for i,n in enumerate(children):
 112             self.progressfunc("examining %d of %d" % (i+1, len(children)))
 113             pn = os.path.join(self.pathname, n)
 114             if os.path.isdir(pn):
 115                 child = LocalDirectorySource(self.progressfunc, pn, n)
 116                 self.children[n] = child
 117                 if recurse:
 118                     child.populate(True)
 119             elif os.path.isfile(pn):
 120                 self.children[n] = LocalFileSource(pn, n)
 121             else:
 122                 # Could be dangling symlink; probably not copy-able.
 123                 # TODO: output a warning
 124                 pass
 125
 126 class LocalDirectoryTarget:
 127     def __init__(self, progressfunc, pathname):
 128         precondition_abspath(pathname)
 129
 130         self.progressfunc = progressfunc
 131         self.pathname = pathname
 132         self.children = None
 133
 134     def populate(self, recurse):
 135         if self.children is not None:
 136             return
 137         self.children = {}
 138         children = listdir_unicode(self.pathname)
 139         for i,n in enumerate(children):
 140             self.progressfunc("examining %d of %d" % (i+1, len(children)))
 141             pn = os.path.join(self.pathname, n)
 142             if os.path.isdir(pn):
 143                 child = LocalDirectoryTarget(self.progressfunc, pn)
 144                 self.children[n] = child
 145                 if recurse:
 146                     child.populate(True)
 147             else:
 148                 assert os.path.isfile(pn)
 149                 self.children[n] = LocalFileTarget(pn)
 150
 151     def get_child_target(self, name):
 152         precondition(isinstance(name, unicode), name)
 153         if self.children is None:
 154             self.populate(False)
 155         if name in self.children:
 156             return self.children[name]
 157         pathname = os.path.join(self.pathname, name)
 158         os.makedirs(pathname)
 159         return LocalDirectoryTarget(self.progressfunc, pathname)
 160
 161     def put_file(self, name, inf):
 162         precondition(isinstance(name, unicode), name)
 163         pathname = os.path.join(self.pathname, name)
 164         fileutil.put_file(pathname, inf)
 165
 166     def set_children(self):
 167         pass
 168
 169
 170 class TahoeFileSource:
 171     def __init__(self, nodeurl, mutable, writecap, readcap, basename):
 172         self.nodeurl = nodeurl
 173         self.mutable = mutable
 174         self.writecap = writecap
 175         self.readcap = readcap
 176         self._basename = basename # unicode, or None for raw filecaps
 177
 178     def basename(self):
 179         return self._basename
 180
 181     def need_to_copy_bytes(self):
 182         if self.mutable:
 183             return True
 184         return False
 185
 186     def open(self, caps_only):
 187         if caps_only:
 188             return StringIO(self.readcap)
 189         url = self.nodeurl + "uri/" + urllib.quote(self.readcap)
 190         return GET_to_file(url)
 191
 192     def bestcap(self):
 193         return self.writecap or self.readcap
 194
 195 class TahoeFileTarget:
 196     def __init__(self, nodeurl, mutable, writecap, readcap, url):
 197         self.nodeurl = nodeurl
 198         self.mutable = mutable
 199         self.writecap = writecap
 200         self.readcap = readcap
 201         self.url = url
 202
 203     def put_file(self, inf):
 204         # We want to replace this object in-place.
 205         assert self.url
 206         # our do_http() call currently requires a string or a filehandle with
 207         # a real .seek
 208         if not hasattr(inf, "seek"):
 209             inf = inf.read()
 210         PUT(self.url, inf)
 211         # TODO: this always creates immutable files. We might want an option
 212         # to always create mutable files, or to copy mutable files into new
 213         # mutable files. ticket #835
 214
 215 class TahoeDirectorySource:
 216     def __init__(self, nodeurl, cache, progressfunc, basename):
 217         self.nodeurl = nodeurl
 218         self.cache = cache
 219         self.progressfunc = progressfunc
 220         self._basename = basename # unicode, or None for raw dircaps
 221
 222     def basename(self):
 223         return self._basename
 224
 225     def init_from_grid(self, writecap, readcap):
 226         self.writecap = writecap
 227         self.readcap = readcap
 228         bestcap = writecap or readcap
 229         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 230         resp = do_http("GET", url + "?t=json")
 231         if resp.status != 200:
 232             raise HTTPError("Error examining source directory", resp)
 233         parsed = simplejson.loads(resp.read())
 234         nodetype, d = parsed
 235         assert nodetype == "dirnode"
 236         self.mutable = d.get("mutable", False) # older nodes don't provide it
 237         self.children_d = dict( [(unicode(name),value)
 238                                  for (name,value)
 239                                  in d["children"].iteritems()] )
 240         self.children = None
 241
 242     def init_from_parsed(self, parsed):
 243         nodetype, d = parsed
 244         self.writecap = to_str(d.get("rw_uri"))
 245         self.readcap = to_str(d.get("ro_uri"))
 246         self.mutable = d.get("mutable", False) # older nodes don't provide it
 247         self.children_d = dict( [(unicode(name),value)
 248                                  for (name,value)
 249                                  in d["children"].iteritems()] )
 250         self.children = None
 251
 252     def populate(self, recurse):
 253         if self.children is not None:
 254             return
 255         self.children = {}
 256         for i,(name, data) in enumerate(self.children_d.items()):
 257             self.progressfunc("examining %d of %d" % (i+1, len(self.children_d)))
 258             if data[0] == "filenode":
 259                 mutable = data[1].get("mutable", False)
 260                 writecap = to_str(data[1].get("rw_uri"))
 261                 readcap = to_str(data[1].get("ro_uri"))
 262                 self.children[name] = TahoeFileSource(self.nodeurl, mutable,
 263                                                       writecap, readcap, name)
 264             elif data[0] == "dirnode":
 265                 writecap = to_str(data[1].get("rw_uri"))
 266                 readcap = to_str(data[1].get("ro_uri"))
 267                 if writecap and writecap in self.cache:
 268                     child = self.cache[writecap]
 269                 elif readcap and readcap in self.cache:
 270                     child = self.cache[readcap]
 271                 else:
 272                     child = TahoeDirectorySource(self.nodeurl, self.cache,
 273                                                  self.progressfunc, name)
 274                     child.init_from_grid(writecap, readcap)
 275                     if writecap:
 276                         self.cache[writecap] = child
 277                     if readcap:
 278                         self.cache[readcap] = child
 279                     if recurse:
 280                         child.populate(True)
 281                 self.children[name] = child
 282             else:
 283                 # TODO: there should be an option to skip unknown nodes.
 284                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 285                                  "You probably need to use a later version of "
 286                                  "Tahoe-LAFS to copy this directory.")
 287
 288 class TahoeMissingTarget:
 289     def __init__(self, url):
 290         self.url = url
 291
 292     def put_file(self, inf):
 293         # We want to replace this object in-place.
 294         if not hasattr(inf, "seek"):
 295             inf = inf.read()
 296         PUT(self.url, inf)
 297         # TODO: this always creates immutable files. We might want an option
 298         # to always create mutable files, or to copy mutable files into new
 299         # mutable files.
 300
 301     def put_uri(self, filecap):
 302         # I'm not sure this will always work
 303         return PUT(self.url + "?t=uri", filecap)
 304
 305 class TahoeDirectoryTarget:
 306     def __init__(self, nodeurl, cache, progressfunc):
 307         self.nodeurl = nodeurl
 308         self.cache = cache
 309         self.progressfunc = progressfunc
 310         self.new_children = {}
 311
 312     def init_from_parsed(self, parsed):
 313         nodetype, d = parsed
 314         self.writecap = to_str(d.get("rw_uri"))
 315         self.readcap = to_str(d.get("ro_uri"))
 316         self.mutable = d.get("mutable", False) # older nodes don't provide it
 317         self.children_d = dict( [(unicode(name),value)
 318                                  for (name,value)
 319                                  in d["children"].iteritems()] )
 320         self.children = None
 321
 322     def init_from_grid(self, writecap, readcap):
 323         self.writecap = writecap
 324         self.readcap = readcap
 325         bestcap = writecap or readcap
 326         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 327         resp = do_http("GET", url + "?t=json")
 328         if resp.status != 200:
 329             raise HTTPError("Error examining target directory", resp)
 330         parsed = simplejson.loads(resp.read())
 331         nodetype, d = parsed
 332         assert nodetype == "dirnode"
 333         self.mutable = d.get("mutable", False) # older nodes don't provide it
 334         self.children_d = dict( [(unicode(name),value)
 335                                  for (name,value)
 336                                  in d["children"].iteritems()] )
 337         self.children = None
 338
 339     def just_created(self, writecap):
 340         self.writecap = writecap
 341         self.readcap = uri.from_string(writecap).get_readonly().to_string()
 342         self.mutable = True
 343         self.children_d = {}
 344         self.children = {}
 345
 346     def populate(self, recurse):
 347         if self.children is not None:
 348             return
 349         self.children = {}
 350         for i,(name, data) in enumerate(self.children_d.items()):
 351             self.progressfunc("examining %d of %d" % (i+1, len(self.children_d)))
 352             if data[0] == "filenode":
 353                 mutable = data[1].get("mutable", False)
 354                 writecap = to_str(data[1].get("rw_uri"))
 355                 readcap = to_str(data[1].get("ro_uri"))
 356                 url = None
 357                 if self.writecap:
 358                     url = self.nodeurl + "/".join(["uri",
 359                                                    urllib.quote(self.writecap),
 360                                                    urllib.quote(unicode_to_url(name))])
 361                 self.children[name] = TahoeFileTarget(self.nodeurl, mutable,
 362                                                       writecap, readcap, url)
 363             elif data[0] == "dirnode":
 364                 writecap = to_str(data[1].get("rw_uri"))
 365                 readcap = to_str(data[1].get("ro_uri"))
 366                 if writecap and writecap in self.cache:
 367                     child = self.cache[writecap]
 368                 elif readcap and readcap in self.cache:
 369                     child = self.cache[readcap]
 370                 else:
 371                     child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 372                                                  self.progressfunc)
 373                     child.init_from_grid(writecap, readcap)
 374                     if writecap:
 375                         self.cache[writecap] = child
 376                     if readcap:
 377                         self.cache[readcap] = child
 378                     if recurse:
 379                         child.populate(True)
 380                 self.children[name] = child
 381             else:
 382                 # TODO: there should be an option to skip unknown nodes.
 383                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 384                                  "You probably need to use a later version of "
 385                                  "Tahoe-LAFS to copy this directory.")
 386
 387     def get_child_target(self, name):
 388         # return a new target for a named subdirectory of this dir
 389         precondition(isinstance(name, unicode), name)
 390         if self.children is None:
 391             self.populate(False)
 392         if name in self.children:
 393             return self.children[name]
 394         writecap = make_tahoe_subdirectory(self.nodeurl, self.writecap, name)
 395         child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 396                                      self.progressfunc)
 397         child.just_created(writecap)
 398         self.children[name] = child
 399         return child
 400
 401     def put_file(self, name, inf):
 402         precondition(isinstance(name, unicode), name)
 403         url = self.nodeurl + "uri"
 404         if not hasattr(inf, "seek"):
 405             inf = inf.read()
 406
 407         if self.children is None:
 408             self.populate(False)
 409
 410         # Check to see if we already have a mutable file by this name.
 411         # If so, overwrite that file in place.
 412         if name in self.children and self.children[name].mutable:
 413             self.children[name].put_file(inf)
 414         else:
 415             filecap = PUT(url, inf)
 416             # TODO: this always creates immutable files. We might want an option
 417             # to always create mutable files, or to copy mutable files into new
 418             # mutable files.
 419             self.new_children[name] = filecap
 420
 421     def put_uri(self, name, filecap):
 422         precondition(isinstance(name, unicode), name)
 423         self.new_children[name] = filecap
 424
 425     def set_children(self):
 426         if not self.new_children:
 427             return
 428         url = (self.nodeurl + "uri/" + urllib.quote(self.writecap)
 429                + "?t=set_children")
 430         set_data = {}
 431         for (name, filecap) in self.new_children.items():
 432             # it just so happens that ?t=set_children will accept both file
 433             # read-caps and write-caps as ['rw_uri'], and will handle either
 434             # correctly. So don't bother trying to figure out whether the one
 435             # we have is read-only or read-write.
 436             # TODO: think about how this affects forward-compatibility for
 437             # unknown caps
 438             set_data[name] = ["filenode", {"rw_uri": filecap}]
 439         body = simplejson.dumps(set_data)
 440         POST(url, body)
 441
 442 class Copier:
 443
 444     def do_copy(self, options, progressfunc=None):
 445         if options['quiet']:
 446             verbosity = 0
 447         elif options['verbose']:
 448             verbosity = 2
 449         else:
 450             verbosity = 1
 451
 452         nodeurl = options['node-url']
 453         if nodeurl[-1] != "/":
 454             nodeurl += "/"
 455         self.nodeurl = nodeurl
 456         self.progressfunc = progressfunc
 457         self.options = options
 458         self.aliases = options.aliases
 459         self.verbosity = verbosity
 460         self.stdout = options.stdout
 461         self.stderr = options.stderr
 462         if verbosity >= 2 and not self.progressfunc:
 463             def progress(message):
 464                 print >>self.stderr, message
 465             self.progressfunc = progress
 466         self.caps_only = options["caps-only"]
 467         self.cache = {}
 468         try:
 469             status = self.try_copy()
 470             return status
 471         except TahoeError, te:
 472             if verbosity >= 2:
 473                 Failure().printTraceback(self.stderr)
 474                 print >>self.stderr
 475             te.display(self.stderr)
 476             return 1
 477
 478     def try_copy(self):
 479         source_specs = self.options.sources
 480         destination_spec = self.options.destination
 481         recursive = self.options["recursive"]
 482
 483         target = self.get_target_info(destination_spec)
 484
 485         sources = [] # list of (name, source object)
 486         for ss in source_specs:
 487             name, source = self.get_source_info(ss)
 488             sources.append( (name, source) )
 489
 490         del name
 491         have_source_dirs = bool([s for (name,s) in sources
 492                                  if isinstance(s, (LocalDirectorySource,
 493                                                    TahoeDirectorySource))])
 494
 495         if have_source_dirs and not recursive:
 496             self.to_stderr("cannot copy directories without --recursive")
 497             return 1
 498
 499         if isinstance(target, (LocalFileTarget, TahoeFileTarget)):
 500             # cp STUFF foo.txt, where foo.txt already exists. This limits the
 501             # possibilities considerably.
 502             if len(sources) > 1:
 503                 self.to_stderr("target %s is not a directory" % quote_output(destination_spec))
 504                 return 1
 505             if have_source_dirs:
 506                 self.to_stderr("cannot copy directory into a file")
 507                 return 1
 508             name, source = sources[0]
 509             return self.copy_file(source, target)
 510
 511         if isinstance(target, (LocalMissingTarget, TahoeMissingTarget)):
 512             if recursive:
 513                 return self.copy_to_directory(sources, target)
 514             if len(sources) > 1:
 515                 # if we have -r, we'll auto-create the target directory. Without
 516                 # it, we'll only create a file.
 517                 self.to_stderr("cannot copy multiple files into a file without -r")
 518                 return 1
 519             # cp file1 newfile
 520             name, source = sources[0]
 521             return self.copy_file(source, target)
 522
 523         if isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)):
 524             # We're copying to an existing directory -- make sure that we
 525             # have target names for everything
 526             for (name, source) in sources:
 527                 if name is None and isinstance(source, TahoeFileSource):
 528                     self.to_stderr(
 529                         "error: you must specify a destination filename")
 530                     return 1
 531             return self.copy_to_directory(sources, target)
 532
 533         self.to_stderr("unknown target")
 534         return 1
 535
 536     def to_stderr(self, text):
 537         print >>self.stderr, text
 538
 539     # FIXME reduce the amount of near-duplicate code between get_target_info
 540     # and get_source_info.
 541
 542     def get_target_info(self, destination_spec):
 543         precondition(isinstance(destination_spec, unicode), destination_spec)
 544         rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None)
 545         path = path_utf8.decode("utf-8")
 546         if rootcap == DefaultAliasMarker:
 547             # no alias, so this is a local file
 548             pathname = abspath_expanduser_unicode(path)
 549             if not os.path.exists(pathname):
 550                 t = LocalMissingTarget(pathname)
 551             elif os.path.isdir(pathname):
 552                 t = LocalDirectoryTarget(self.progress, pathname)
 553             else:
 554                 # TODO: should this be _assert? what happens if the target is
 555                 # a special file?
 556                 assert os.path.isfile(pathname), pathname
 557                 t = LocalFileTarget(pathname) # non-empty
 558         else:
 559             # this is a tahoe object
 560             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 561             if path:
 562                 url += "/" + escape_path(path)
 563
 564             resp = do_http("GET", url + "?t=json")
 565             if resp.status == 404:
 566                 # doesn't exist yet
 567                 t = TahoeMissingTarget(url)
 568             elif resp.status == 200:
 569                 parsed = simplejson.loads(resp.read())
 570                 nodetype, d = parsed
 571                 if nodetype == "dirnode":
 572                     t = TahoeDirectoryTarget(self.nodeurl, self.cache,
 573                                              self.progress)
 574                     t.init_from_parsed(parsed)
 575                 else:
 576                     writecap = to_str(d.get("rw_uri"))
 577                     readcap = to_str(d.get("ro_uri"))
 578                     mutable = d.get("mutable", False)
 579                     t = TahoeFileTarget(self.nodeurl, mutable,
 580                                         writecap, readcap, url)
 581             else:
 582                 raise HTTPError("Error examining target %s"
 583                                  % quote_output(destination_spec), resp)
 584         return t
 585
 586     def get_source_info(self, source_spec):
 587         precondition(isinstance(source_spec, unicode), source_spec)
 588         rootcap, path_utf8 = get_alias(self.aliases, source_spec, None)
 589         path = path_utf8.decode("utf-8")
 590         if rootcap == DefaultAliasMarker:
 591             # no alias, so this is a local file
 592             pathname = abspath_expanduser_unicode(path)
 593             name = os.path.basename(pathname)
 594             if not os.path.exists(pathname):
 595                 raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path)
 596             if os.path.isdir(pathname):
 597                 t = LocalDirectorySource(self.progress, pathname, name)
 598             else:
 599                 assert os.path.isfile(pathname)
 600                 t = LocalFileSource(pathname, name) # non-empty
 601         else:
 602             # this is a tahoe object
 603             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 604             name = None
 605             if path:
 606                 url += "/" + escape_path(path)
 607                 last_slash = path.rfind(u"/")
 608                 name = path
 609                 if last_slash != -1:
 610                     name = path[last_slash+1:]
 611
 612             resp = do_http("GET", url + "?t=json")
 613             if resp.status == 404:
 614                 raise MissingSourceError(source_spec)
 615             elif resp.status != 200:
 616                 raise HTTPError("Error examining source %s" % quote_output(source_spec),
 617                                 resp)
 618             parsed = simplejson.loads(resp.read())
 619             nodetype, d = parsed
 620             if nodetype == "dirnode":
 621                 t = TahoeDirectorySource(self.nodeurl, self.cache,
 622                                          self.progress, name)
 623                 t.init_from_parsed(parsed)
 624             else:
 625                 writecap = to_str(d.get("rw_uri"))
 626                 readcap = to_str(d.get("ro_uri"))
 627                 mutable = d.get("mutable", False) # older nodes don't provide it
 628
 629                 last_slash = source_spec.rfind(u"/")
 630                 if last_slash != -1:
 631                     # TODO: this looks funny and redundant with the 'name'
 632                     # assignment above. cf #2329
 633                     name = source_spec[last_slash+1:]
 634
 635                 t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name)
 636         return name, t
 637
 638
 639     def dump_graph(self, s, indent=" "):
 640         for name, child in s.children.items():
 641             print "%s%s: %r" % (indent, quote_output(name), child)
 642             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 643                 self.dump_graph(child, indent+"  ")
 644
 645     def copy_to_directory(self, source_infos, target):
 646         # step one: build a recursive graph of the source tree. This returns
 647         # a dictionary, with child names as keys, and values that are either
 648         # Directory or File instances (local or tahoe).
 649         source_dirs = self.build_graphs(source_infos)
 650         source_files = [source for source in source_infos
 651                         if isinstance(source[1], (LocalFileSource,
 652                                                   TahoeFileSource))]
 653
 654         #print "graphs"
 655         #for s in source_dirs:
 656         #    self.dump_graph(s)
 657
 658         # step two: create the top-level target directory object
 659         if isinstance(target, LocalMissingTarget):
 660             os.makedirs(target.pathname)
 661             target = LocalDirectoryTarget(self.progress, target.pathname)
 662         elif isinstance(target, TahoeMissingTarget):
 663             writecap = mkdir(target.url)
 664             target = TahoeDirectoryTarget(self.nodeurl, self.cache,
 665                                           self.progress)
 666             target.just_created(writecap)
 667         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 668         target.populate(False)
 669
 670         # step three: find a target for each source node, creating
 671         # directories as necessary. 'targetmap' is a dictionary that uses
 672         # target Directory instances as keys, and has values of
 673         # (name->sourceobject) dicts for all the files that need to wind up
 674         # there.
 675
 676         # sources are all LocalFile/LocalDirectory/TahoeFile/TahoeDirectory
 677         # target is LocalDirectory/TahoeDirectory
 678
 679         self.progress("attaching sources to targets, "
 680                       "%d files / %d dirs in root" %
 681                       (len(source_files), len(source_dirs)))
 682
 683         self.targetmap = {}
 684         self.files_to_copy = 0
 685
 686         for (name,s) in source_files:
 687             self.attach_to_target(s, name, target)
 688
 689         for (name, source) in source_dirs:
 690             new_target = target.get_child_target(name)
 691             self.assign_targets(source, new_target)
 692
 693         self.progress("targets assigned, %s dirs, %s files" %
 694                       (len(self.targetmap), self.files_to_copy))
 695
 696         self.progress("starting copy, %d files, %d directories" %
 697                       (self.files_to_copy, len(self.targetmap)))
 698         self.files_copied = 0
 699         self.targets_finished = 0
 700
 701         # step four: walk through the list of targets. For each one, copy all
 702         # the files. If the target is a TahoeDirectory, upload and create
 703         # read-caps, then do a set_children to the target directory.
 704
 705         for target in self.targetmap:
 706             self.copy_files_to_target(self.targetmap[target], target)
 707             self.targets_finished += 1
 708             self.progress("%d/%d directories" %
 709                           (self.targets_finished, len(self.targetmap)))
 710
 711         return self.announce_success("files copied")
 712
 713     def attach_to_target(self, source, name, target):
 714         precondition(isinstance(name, unicode), name)
 715         if target not in self.targetmap:
 716             self.targetmap[target] = {}
 717         self.targetmap[target][name] = source
 718         self.files_to_copy += 1
 719
 720     def assign_targets(self, source, target):
 721         # copy everything in the source into the target
 722         precondition(isinstance(source, (LocalDirectorySource, TahoeDirectorySource)), source)
 723
 724         for name, child in source.children.items():
 725             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 726                 # we will need a target directory for this one
 727                 subtarget = target.get_child_target(name)
 728                 self.assign_targets(child, subtarget)
 729             else:
 730                 precondition(isinstance(child, (LocalFileSource, TahoeFileSource)), child)
 731                 self.attach_to_target(child, name, target)
 732
 733     def copy_files_to_target(self, targetmap, target):
 734         for name, source in targetmap.items():
 735             precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
 736             self.copy_file_into(source, name, target)
 737             self.files_copied += 1
 738             self.progress("%d/%d files, %d/%d directories" %
 739                           (self.files_copied, self.files_to_copy,
 740                            self.targets_finished, len(self.targetmap)))
 741         target.set_children()
 742
 743     def need_to_copy_bytes(self, source, target):
 744         if source.need_to_copy_bytes:
 745             # mutable tahoe files, and local files
 746             return True
 747         if isinstance(target, (LocalFileTarget, LocalDirectoryTarget)):
 748             return True
 749         return False
 750
 751     def announce_success(self, msg):
 752         if self.verbosity >= 1:
 753             print >>self.stdout, "Success: %s" % msg
 754         return 0
 755
 756     def copy_file(self, source, target):
 757         precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
 758         precondition(isinstance(target, (LocalFileTarget, TahoeFileTarget,
 759                                          LocalMissingTarget, TahoeMissingTarget)), target)
 760         if self.need_to_copy_bytes(source, target):
 761             # if the target is a local directory, this will just write the
 762             # bytes to disk. If it is a tahoe directory, it will upload the
 763             # data, and stash the new filecap for a later set_children call.
 764             f = source.open(self.caps_only)
 765             target.put_file(f)
 766             return self.announce_success("file copied")
 767         # otherwise we're copying tahoe to tahoe, and using immutable files,
 768         # so we can just make a link. TODO: this probably won't always work:
 769         # need to enumerate the cases and analyze them.
 770         target.put_uri(source.bestcap())
 771         return self.announce_success("file linked")
 772
 773     def copy_file_into(self, source, name, target):
 774         precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
 775         precondition(isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)), target)
 776         precondition(isinstance(name, unicode), name)
 777         if self.need_to_copy_bytes(source, target):
 778             # if the target is a local directory, this will just write the
 779             # bytes to disk. If it is a tahoe directory, it will upload the
 780             # data, and stash the new filecap for a later set_children call.
 781             f = source.open(self.caps_only)
 782             target.put_file(name, f)
 783             return
 784         # otherwise we're copying tahoe to tahoe, and using immutable files,
 785         # so we can just make a link
 786         target.put_uri(name, source.bestcap())
 787
 788
 789     def progress(self, message):
 790         #print message
 791         if self.progressfunc:
 792             self.progressfunc(message)
 793
 794     def build_graphs(self, source_infos):
 795         graphs = []
 796         for name,source in source_infos:
 797             if isinstance(source, (LocalDirectorySource, TahoeDirectorySource)):
 798                 source.populate(True)
 799                 # Remove trailing slash (if applicable) and get dir name
 800                 name = os.path.basename(os.path.normpath(name))
 801                 graphs.append((name, source))
 802         return graphs
 803
 804
 805 def copy(options):
 806     return Copier().do_copy(options)
 807
 808 # error cases that need improvement:
 809 #  local-file-in-the-way
 810 #   touch proposed
 811 #   tahoe cp -r my:docs/proposed/denver.txt proposed/denver.txt
 812 #  handling of unknown nodes
 813
 814 # things that maybe should be errors but aren't
 815 #  local-dir-in-the-way
 816 #   mkdir denver.txt
 817 #   tahoe cp -r my:docs/proposed/denver.txt denver.txt
 818 #   (creates denver.txt/denver.txt)
 819
 820 # error cases that look good:
 821 #  tahoe cp -r my:docs/missing missing
 822 #  disconnect servers
 823 #   tahoe cp -r my:docs/missing missing  -> No JSON object could be decoded
 824 #  tahoe-file-in-the-way (when we want to make a directory)
 825 #   tahoe put README my:docs
 826 #   tahoe cp -r docs/proposed my:docs/proposed