src/allmydata/scripts/tahoe_cp.py

   1
   2 import os.path
   3 import urllib
   4 import simplejson
   5 from cStringIO import StringIO
   6 from twisted.python.failure import Failure
   7 from allmydata.scripts.common import get_alias, escape_path, \
   8                                      DefaultAliasMarker, TahoeError
   9 from allmydata.scripts.common_http import do_http, HTTPError
  10 from allmydata import uri
  11 from allmydata.util import fileutil
  12 from allmydata.util.fileutil import abspath_expanduser_unicode
  13 from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, to_str
  14 from allmydata.util.assertutil import precondition
  15
  16
  17 class MissingSourceError(TahoeError):
  18     def __init__(self, name):
  19         TahoeError.__init__(self, "No such file or directory %s" % quote_output(name))
  20
  21
  22 def GET_to_file(url):
  23     resp = do_http("GET", url)
  24     if resp.status == 200:
  25         return resp
  26     raise HTTPError("Error during GET", resp)
  27
  28 def GET_to_string(url):
  29     f = GET_to_file(url)
  30     return f.read()
  31
  32 def PUT(url, data):
  33     resp = do_http("PUT", url, data)
  34     if resp.status in (200, 201):
  35         return resp.read()
  36     raise HTTPError("Error during PUT", resp)
  37
  38 def POST(url, data):
  39     resp = do_http("POST", url, data)
  40     if resp.status in (200, 201):
  41         return resp.read()
  42     raise HTTPError("Error during POST", resp)
  43
  44 def mkdir(targeturl):
  45     url = targeturl + "?t=mkdir"
  46     resp = do_http("POST", url)
  47     if resp.status in (200, 201):
  48         return resp.read().strip()
  49     raise HTTPError("Error during mkdir", resp)
  50
  51 def make_tahoe_subdirectory(nodeurl, parent_writecap, name):
  52     url = nodeurl + "/".join(["uri",
  53                               urllib.quote(parent_writecap),
  54                               urllib.quote(unicode_to_url(name)),
  55                               ]) + "?t=mkdir"
  56     resp = do_http("POST", url)
  57     if resp.status in (200, 201):
  58         return resp.read().strip()
  59     raise HTTPError("Error during mkdir", resp)
  60
  61
  62 class LocalFileSource:
  63     def __init__(self, pathname):
  64         precondition(isinstance(pathname, unicode), pathname)
  65         self.pathname = pathname
  66
  67     def need_to_copy_bytes(self):
  68         return True
  69
  70     def open(self, caps_only):
  71         return open(os.path.expanduser(self.pathname), "rb")
  72
  73
  74 class LocalFileTarget:
  75     def __init__(self, pathname):
  76         precondition(isinstance(pathname, unicode), pathname)
  77         self.pathname = pathname
  78
  79     def put_file(self, inf):
  80         fileutil.put_file(self.pathname, inf)
  81
  82
  83 class LocalMissingTarget:
  84     def __init__(self, pathname):
  85         precondition(isinstance(pathname, unicode), pathname)
  86         self.pathname = pathname
  87
  88     def put_file(self, inf):
  89         fileutil.put_file(self.pathname, inf)
  90
  91
  92 class LocalDirectorySource:
  93     def __init__(self, progressfunc, pathname):
  94         precondition(isinstance(pathname, unicode), pathname)
  95
  96         self.progressfunc = progressfunc
  97         self.pathname = pathname
  98         self.children = None
  99
 100     def populate(self, recurse):
 101         if self.children is not None:
 102             return
 103         self.children = {}
 104         children = listdir_unicode(self.pathname)
 105         for i,n in enumerate(children):
 106             self.progressfunc("examining %d of %d" % (i+1, len(children)))
 107             pn = os.path.join(self.pathname, n)
 108             if os.path.isdir(pn):
 109                 child = LocalDirectorySource(self.progressfunc, pn)
 110                 self.children[n] = child
 111                 if recurse:
 112                     child.populate(True)
 113             elif os.path.isfile(pn):
 114                 self.children[n] = LocalFileSource(pn)
 115             else:
 116                 # Could be dangling symlink; probably not copy-able.
 117                 # TODO: output a warning
 118                 pass
 119
 120 class LocalDirectoryTarget:
 121     def __init__(self, progressfunc, pathname):
 122         precondition(isinstance(pathname, unicode), pathname)
 123
 124         self.progressfunc = progressfunc
 125         self.pathname = pathname
 126         self.children = None
 127
 128     def populate(self, recurse):
 129         if self.children is not None:
 130             return
 131         self.children = {}
 132         children = listdir_unicode(self.pathname)
 133         for i,n in enumerate(children):
 134             self.progressfunc("examining %d of %d" % (i+1, len(children)))
 135             n = unicode(n)
 136             pn = os.path.join(self.pathname, n)
 137             if os.path.isdir(pn):
 138                 child = LocalDirectoryTarget(self.progressfunc, pn)
 139                 self.children[n] = child
 140                 if recurse:
 141                     child.populate(True)
 142             else:
 143                 assert os.path.isfile(pn)
 144                 self.children[n] = LocalFileTarget(pn)
 145
 146     def get_child_target(self, name):
 147         if self.children is None:
 148             self.populate(False)
 149         if name in self.children:
 150             return self.children[name]
 151         pathname = os.path.join(self.pathname, name)
 152         os.makedirs(pathname)
 153         return LocalDirectoryTarget(self.progressfunc, pathname)
 154
 155     def put_file(self, name, inf):
 156         precondition(isinstance(name, unicode), name)
 157         pathname = os.path.join(self.pathname, name)
 158         fileutil.put_file(pathname, inf)
 159
 160     def set_children(self):
 161         pass
 162
 163 class TahoeFileSource:
 164     def __init__(self, nodeurl, mutable, writecap, readcap):
 165         self.nodeurl = nodeurl
 166         self.mutable = mutable
 167         self.writecap = writecap
 168         self.readcap = readcap
 169
 170     def need_to_copy_bytes(self):
 171         if self.mutable:
 172             return True
 173         return False
 174
 175     def open(self, caps_only):
 176         if caps_only:
 177             return StringIO(self.readcap)
 178         url = self.nodeurl + "uri/" + urllib.quote(self.readcap)
 179         return GET_to_file(url)
 180
 181     def bestcap(self):
 182         return self.writecap or self.readcap
 183
 184 class TahoeFileTarget:
 185     def __init__(self, nodeurl, mutable, writecap, readcap, url):
 186         self.nodeurl = nodeurl
 187         self.mutable = mutable
 188         self.writecap = writecap
 189         self.readcap = readcap
 190         self.url = url
 191
 192     def put_file(self, inf):
 193         # We want to replace this object in-place.
 194         assert self.url
 195         # our do_http() call currently requires a string or a filehandle with
 196         # a real .seek
 197         if not hasattr(inf, "seek"):
 198             inf = inf.read()
 199         PUT(self.url, inf)
 200         # TODO: this always creates immutable files. We might want an option
 201         # to always create mutable files, or to copy mutable files into new
 202         # mutable files. ticket #835
 203
 204 class TahoeDirectorySource:
 205     def __init__(self, nodeurl, cache, progressfunc):
 206         self.nodeurl = nodeurl
 207         self.cache = cache
 208         self.progressfunc = progressfunc
 209
 210     def init_from_grid(self, writecap, readcap):
 211         self.writecap = writecap
 212         self.readcap = readcap
 213         bestcap = writecap or readcap
 214         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 215         resp = do_http("GET", url + "?t=json")
 216         if resp.status != 200:
 217             raise HTTPError("Error examining source directory", resp)
 218         parsed = simplejson.loads(resp.read())
 219         nodetype, d = parsed
 220         assert nodetype == "dirnode"
 221         self.mutable = d.get("mutable", False) # older nodes don't provide it
 222         self.children_d = dict( [(unicode(name),value)
 223                                  for (name,value)
 224                                  in d["children"].iteritems()] )
 225         self.children = None
 226
 227     def init_from_parsed(self, parsed):
 228         nodetype, d = parsed
 229         self.writecap = to_str(d.get("rw_uri"))
 230         self.readcap = to_str(d.get("ro_uri"))
 231         self.mutable = d.get("mutable", False) # older nodes don't provide it
 232         self.children_d = dict( [(unicode(name),value)
 233                                  for (name,value)
 234                                  in d["children"].iteritems()] )
 235         self.children = None
 236
 237     def populate(self, recurse):
 238         if self.children is not None:
 239             return
 240         self.children = {}
 241         for i,(name, data) in enumerate(self.children_d.items()):
 242             self.progressfunc("examining %d of %d" % (i+1, len(self.children_d)))
 243             if data[0] == "filenode":
 244                 mutable = data[1].get("mutable", False)
 245                 writecap = to_str(data[1].get("rw_uri"))
 246                 readcap = to_str(data[1].get("ro_uri"))
 247                 self.children[name] = TahoeFileSource(self.nodeurl, mutable,
 248                                                       writecap, readcap)
 249             elif data[0] == "dirnode":
 250                 writecap = to_str(data[1].get("rw_uri"))
 251                 readcap = to_str(data[1].get("ro_uri"))
 252                 if writecap and writecap in self.cache:
 253                     child = self.cache[writecap]
 254                 elif readcap and readcap in self.cache:
 255                     child = self.cache[readcap]
 256                 else:
 257                     child = TahoeDirectorySource(self.nodeurl, self.cache,
 258                                                  self.progressfunc)
 259                     child.init_from_grid(writecap, readcap)
 260                     if writecap:
 261                         self.cache[writecap] = child
 262                     if readcap:
 263                         self.cache[readcap] = child
 264                     if recurse:
 265                         child.populate(True)
 266                 self.children[name] = child
 267             else:
 268                 # TODO: there should be an option to skip unknown nodes.
 269                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 270                                  "You probably need to use a later version of "
 271                                  "Tahoe-LAFS to copy this directory.")
 272
 273 class TahoeMissingTarget:
 274     def __init__(self, url):
 275         self.url = url
 276
 277     def put_file(self, inf):
 278         # We want to replace this object in-place.
 279         if not hasattr(inf, "seek"):
 280             inf = inf.read()
 281         PUT(self.url, inf)
 282         # TODO: this always creates immutable files. We might want an option
 283         # to always create mutable files, or to copy mutable files into new
 284         # mutable files.
 285
 286     def put_uri(self, filecap):
 287         # I'm not sure this will always work
 288         return PUT(self.url + "?t=uri", filecap)
 289
 290 class TahoeDirectoryTarget:
 291     def __init__(self, nodeurl, cache, progressfunc):
 292         self.nodeurl = nodeurl
 293         self.cache = cache
 294         self.progressfunc = progressfunc
 295         self.new_children = {}
 296
 297     def init_from_parsed(self, parsed):
 298         nodetype, d = parsed
 299         self.writecap = to_str(d.get("rw_uri"))
 300         self.readcap = to_str(d.get("ro_uri"))
 301         self.mutable = d.get("mutable", False) # older nodes don't provide it
 302         self.children_d = dict( [(unicode(name),value)
 303                                  for (name,value)
 304                                  in d["children"].iteritems()] )
 305         self.children = None
 306
 307     def init_from_grid(self, writecap, readcap):
 308         self.writecap = writecap
 309         self.readcap = readcap
 310         bestcap = writecap or readcap
 311         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 312         resp = do_http("GET", url + "?t=json")
 313         if resp.status != 200:
 314             raise HTTPError("Error examining target directory", resp)
 315         parsed = simplejson.loads(resp.read())
 316         nodetype, d = parsed
 317         assert nodetype == "dirnode"
 318         self.mutable = d.get("mutable", False) # older nodes don't provide it
 319         self.children_d = dict( [(unicode(name),value)
 320                                  for (name,value)
 321                                  in d["children"].iteritems()] )
 322         self.children = None
 323
 324     def just_created(self, writecap):
 325         self.writecap = writecap
 326         self.readcap = uri.from_string(writecap).get_readonly().to_string()
 327         self.mutable = True
 328         self.children_d = {}
 329         self.children = {}
 330
 331     def populate(self, recurse):
 332         if self.children is not None:
 333             return
 334         self.children = {}
 335         for i,(name, data) in enumerate(self.children_d.items()):
 336             self.progressfunc("examining %d of %d" % (i+1, len(self.children_d)))
 337             if data[0] == "filenode":
 338                 mutable = data[1].get("mutable", False)
 339                 writecap = to_str(data[1].get("rw_uri"))
 340                 readcap = to_str(data[1].get("ro_uri"))
 341                 url = None
 342                 if self.writecap:
 343                     url = self.nodeurl + "/".join(["uri",
 344                                                    urllib.quote(self.writecap),
 345                                                    urllib.quote(unicode_to_url(name))])
 346                 self.children[name] = TahoeFileTarget(self.nodeurl, mutable,
 347                                                       writecap, readcap, url)
 348             elif data[0] == "dirnode":
 349                 writecap = to_str(data[1].get("rw_uri"))
 350                 readcap = to_str(data[1].get("ro_uri"))
 351                 if writecap and writecap in self.cache:
 352                     child = self.cache[writecap]
 353                 elif readcap and readcap in self.cache:
 354                     child = self.cache[readcap]
 355                 else:
 356                     child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 357                                                  self.progressfunc)
 358                     child.init_from_grid(writecap, readcap)
 359                     if writecap:
 360                         self.cache[writecap] = child
 361                     if readcap:
 362                         self.cache[readcap] = child
 363                     if recurse:
 364                         child.populate(True)
 365                 self.children[name] = child
 366             else:
 367                 # TODO: there should be an option to skip unknown nodes.
 368                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 369                                  "You probably need to use a later version of "
 370                                  "Tahoe-LAFS to copy this directory.")
 371
 372     def get_child_target(self, name):
 373         # return a new target for a named subdirectory of this dir
 374         if self.children is None:
 375             self.populate(False)
 376         if name in self.children:
 377             return self.children[name]
 378         writecap = make_tahoe_subdirectory(self.nodeurl, self.writecap, name)
 379         child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 380                                      self.progressfunc)
 381         child.just_created(writecap)
 382         self.children[name] = child
 383         return child
 384
 385     def put_file(self, name, inf):
 386         url = self.nodeurl + "uri"
 387         if not hasattr(inf, "seek"):
 388             inf = inf.read()
 389
 390         if self.children is None:
 391             self.populate(False)
 392
 393         # Check to see if we already have a mutable file by this name.
 394         # If so, overwrite that file in place.
 395         if name in self.children and self.children[name].mutable:
 396             self.children[name].put_file(inf)
 397         else:
 398             filecap = PUT(url, inf)
 399             # TODO: this always creates immutable files. We might want an option
 400             # to always create mutable files, or to copy mutable files into new
 401             # mutable files.
 402             self.new_children[name] = filecap
 403
 404     def put_uri(self, name, filecap):
 405         self.new_children[name] = filecap
 406
 407     def set_children(self):
 408         if not self.new_children:
 409             return
 410         url = (self.nodeurl + "uri/" + urllib.quote(self.writecap)
 411                + "?t=set_children")
 412         set_data = {}
 413         for (name, filecap) in self.new_children.items():
 414             # it just so happens that ?t=set_children will accept both file
 415             # read-caps and write-caps as ['rw_uri'], and will handle either
 416             # correctly. So don't bother trying to figure out whether the one
 417             # we have is read-only or read-write.
 418             # TODO: think about how this affects forward-compatibility for
 419             # unknown caps
 420             set_data[name] = ["filenode", {"rw_uri": filecap}]
 421         body = simplejson.dumps(set_data)
 422         POST(url, body)
 423
 424 class Copier:
 425
 426     def do_copy(self, options, progressfunc=None):
 427         if options['quiet']:
 428             verbosity = 0
 429         elif options['verbose']:
 430             verbosity = 2
 431         else:
 432             verbosity = 1
 433
 434         nodeurl = options['node-url']
 435         if nodeurl[-1] != "/":
 436             nodeurl += "/"
 437         self.nodeurl = nodeurl
 438         self.progressfunc = progressfunc
 439         self.options = options
 440         self.aliases = options.aliases
 441         self.verbosity = verbosity
 442         self.stdout = options.stdout
 443         self.stderr = options.stderr
 444         if verbosity >= 2 and not self.progressfunc:
 445             def progress(message):
 446                 print >>self.stderr, message
 447             self.progressfunc = progress
 448         self.caps_only = options["caps-only"]
 449         self.cache = {}
 450         try:
 451             status = self.try_copy()
 452             return status
 453         except TahoeError, te:
 454             if verbosity >= 2:
 455                 Failure().printTraceback(self.stderr)
 456                 print >>self.stderr
 457             te.display(self.stderr)
 458             return 1
 459
 460     def try_copy(self):
 461         source_specs = self.options.sources
 462         destination_spec = self.options.destination
 463         recursive = self.options["recursive"]
 464
 465         target = self.get_target_info(destination_spec)
 466
 467         sources = [] # list of (name, source object)
 468         for ss in source_specs:
 469             name, source = self.get_source_info(ss)
 470             sources.append( (name, source) )
 471
 472         have_source_dirs = bool([s for (name,s) in sources
 473                                  if isinstance(s, (LocalDirectorySource,
 474                                                    TahoeDirectorySource))])
 475
 476         if have_source_dirs and not recursive:
 477             self.to_stderr("cannot copy directories without --recursive")
 478             return 1
 479
 480         if isinstance(target, (LocalFileTarget, TahoeFileTarget)):
 481             # cp STUFF foo.txt, where foo.txt already exists. This limits the
 482             # possibilities considerably.
 483             if len(sources) > 1:
 484                 self.to_stderr("target %s is not a directory" % quote_output(destination_spec))
 485                 return 1
 486             if have_source_dirs:
 487                 self.to_stderr("cannot copy directory into a file")
 488                 return 1
 489             name, source = sources[0]
 490             return self.copy_file(source, target)
 491
 492         if isinstance(target, (LocalMissingTarget, TahoeMissingTarget)):
 493             if recursive:
 494                 return self.copy_to_directory(sources, target)
 495             if len(sources) > 1:
 496                 # if we have -r, we'll auto-create the target directory. Without
 497                 # it, we'll only create a file.
 498                 self.to_stderr("cannot copy multiple files into a file without -r")
 499                 return 1
 500             # cp file1 newfile
 501             name, source = sources[0]
 502             return self.copy_file(source, target)
 503
 504         if isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)):
 505             # We're copying to an existing directory -- make sure that we
 506             # have target names for everything
 507             for (name, source) in sources:
 508                 if name is None and isinstance(source, TahoeFileSource):
 509                     self.to_stderr(
 510                         "error: you must specify a destination filename")
 511                     return 1
 512             return self.copy_to_directory(sources, target)
 513
 514         self.to_stderr("unknown target")
 515         return 1
 516
 517     def to_stderr(self, text):
 518         print >>self.stderr, text
 519
 520     def get_target_info(self, destination_spec):
 521         rootcap, path = get_alias(self.aliases, destination_spec, None)
 522         if rootcap == DefaultAliasMarker:
 523             # no alias, so this is a local file
 524             pathname = abspath_expanduser_unicode(path.decode('utf-8'))
 525             if not os.path.exists(pathname):
 526                 t = LocalMissingTarget(pathname)
 527             elif os.path.isdir(pathname):
 528                 t = LocalDirectoryTarget(self.progress, pathname)
 529             else:
 530                 assert os.path.isfile(pathname), pathname
 531                 t = LocalFileTarget(pathname) # non-empty
 532         else:
 533             # this is a tahoe object
 534             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 535             if path:
 536                 url += "/" + escape_path(path)
 537
 538             resp = do_http("GET", url + "?t=json")
 539             if resp.status == 404:
 540                 # doesn't exist yet
 541                 t = TahoeMissingTarget(url)
 542             elif resp.status == 200:
 543                 parsed = simplejson.loads(resp.read())
 544                 nodetype, d = parsed
 545                 if nodetype == "dirnode":
 546                     t = TahoeDirectoryTarget(self.nodeurl, self.cache,
 547                                              self.progress)
 548                     t.init_from_parsed(parsed)
 549                 else:
 550                     writecap = to_str(d.get("rw_uri"))
 551                     readcap = to_str(d.get("ro_uri"))
 552                     mutable = d.get("mutable", False)
 553                     t = TahoeFileTarget(self.nodeurl, mutable,
 554                                         writecap, readcap, url)
 555             else:
 556                 raise HTTPError("Error examining target %s"
 557                                  % quote_output(destination_spec), resp)
 558         return t
 559
 560     def get_source_info(self, source_spec):
 561         rootcap, path = get_alias(self.aliases, source_spec, None)
 562         if rootcap == DefaultAliasMarker:
 563             # no alias, so this is a local file
 564             pathname = abspath_expanduser_unicode(path.decode('utf-8'))
 565             name = os.path.basename(pathname)
 566             if not os.path.exists(pathname):
 567                 raise MissingSourceError(source_spec)
 568             if os.path.isdir(pathname):
 569                 t = LocalDirectorySource(self.progress, pathname)
 570             else:
 571                 assert os.path.isfile(pathname)
 572                 t = LocalFileSource(pathname) # non-empty
 573         else:
 574             # this is a tahoe object
 575             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 576             name = None
 577             if path:
 578                 url += "/" + escape_path(path)
 579                 last_slash = path.rfind("/")
 580                 name = path
 581                 if last_slash:
 582                     name = path[last_slash+1:]
 583
 584             resp = do_http("GET", url + "?t=json")
 585             if resp.status == 404:
 586                 raise MissingSourceError(source_spec)
 587             elif resp.status != 200:
 588                 raise HTTPError("Error examining source %s" % quote_output(source_spec),
 589                                 resp)
 590             parsed = simplejson.loads(resp.read())
 591             nodetype, d = parsed
 592             if nodetype == "dirnode":
 593                 t = TahoeDirectorySource(self.nodeurl, self.cache,
 594                                          self.progress)
 595                 t.init_from_parsed(parsed)
 596             else:
 597                 writecap = to_str(d.get("rw_uri"))
 598                 readcap = to_str(d.get("ro_uri"))
 599                 mutable = d.get("mutable", False) # older nodes don't provide it
 600                 if source_spec.rfind('/') != -1:
 601                     name = source_spec[source_spec.rfind('/')+1:]
 602                 t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap)
 603         return name, t
 604
 605
 606     def dump_graph(self, s, indent=" "):
 607         for name, child in s.children.items():
 608             print "%s%s: %r" % (indent, quote_output(name), child)
 609             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 610                 self.dump_graph(child, indent+"  ")
 611
 612     def copy_to_directory(self, source_infos, target):
 613         # step one: build a recursive graph of the source tree. This returns
 614         # a dictionary, with child names as keys, and values that are either
 615         # Directory or File instances (local or tahoe).
 616         source_dirs = self.build_graphs(source_infos)
 617         source_files = [source for source in source_infos
 618                         if isinstance(source[1], (LocalFileSource,
 619                                                   TahoeFileSource))]
 620
 621         #print "graphs"
 622         #for s in source_dirs:
 623         #    self.dump_graph(s)
 624
 625         # step two: create the top-level target directory object
 626         if isinstance(target, LocalMissingTarget):
 627             os.makedirs(target.pathname)
 628             target = LocalDirectoryTarget(self.progress, target.pathname)
 629         elif isinstance(target, TahoeMissingTarget):
 630             writecap = mkdir(target.url)
 631             target = TahoeDirectoryTarget(self.nodeurl, self.cache,
 632                                           self.progress)
 633             target.just_created(writecap)
 634         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 635         target.populate(False)
 636
 637         # step three: find a target for each source node, creating
 638         # directories as necessary. 'targetmap' is a dictionary that uses
 639         # target Directory instances as keys, and has values of
 640         # (name->sourceobject) dicts for all the files that need to wind up
 641         # there.
 642
 643         # sources are all LocalFile/LocalDirectory/TahoeFile/TahoeDirectory
 644         # target is LocalDirectory/TahoeDirectory
 645
 646         self.progress("attaching sources to targets, "
 647                       "%d files / %d dirs in root" %
 648                       (len(source_files), len(source_dirs)))
 649
 650         self.targetmap = {}
 651         self.files_to_copy = 0
 652
 653         for (name,s) in source_files:
 654             self.attach_to_target(s, name, target)
 655
 656         for source in source_dirs:
 657             self.assign_targets(source, target)
 658
 659         self.progress("targets assigned, %s dirs, %s files" %
 660                       (len(self.targetmap), self.files_to_copy))
 661
 662         self.progress("starting copy, %d files, %d directories" %
 663                       (self.files_to_copy, len(self.targetmap)))
 664         self.files_copied = 0
 665         self.targets_finished = 0
 666
 667         # step four: walk through the list of targets. For each one, copy all
 668         # the files. If the target is a TahoeDirectory, upload and create
 669         # read-caps, then do a set_children to the target directory.
 670
 671         for target in self.targetmap:
 672             self.copy_files_to_target(self.targetmap[target], target)
 673             self.targets_finished += 1
 674             self.progress("%d/%d directories" %
 675                           (self.targets_finished, len(self.targetmap)))
 676
 677         return self.announce_success("files copied")
 678
 679     def attach_to_target(self, source, name, target):
 680         if target not in self.targetmap:
 681             self.targetmap[target] = {}
 682         self.targetmap[target][name] = source
 683         self.files_to_copy += 1
 684
 685     def assign_targets(self, source, target):
 686         # copy everything in the source into the target
 687         assert isinstance(source, (LocalDirectorySource, TahoeDirectorySource))
 688
 689         for name, child in source.children.items():
 690             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 691                 # we will need a target directory for this one
 692                 subtarget = target.get_child_target(name)
 693                 self.assign_targets(child, subtarget)
 694             else:
 695                 assert isinstance(child, (LocalFileSource, TahoeFileSource))
 696                 self.attach_to_target(child, name, target)
 697
 698
 699
 700     def copy_files_to_target(self, targetmap, target):
 701         for name, source in targetmap.items():
 702             assert isinstance(source, (LocalFileSource, TahoeFileSource))
 703             self.copy_file_into(source, name, target)
 704             self.files_copied += 1
 705             self.progress("%d/%d files, %d/%d directories" %
 706                           (self.files_copied, self.files_to_copy,
 707                            self.targets_finished, len(self.targetmap)))
 708         target.set_children()
 709
 710     def need_to_copy_bytes(self, source, target):
 711         if source.need_to_copy_bytes:
 712             # mutable tahoe files, and local files
 713             return True
 714         if isinstance(target, (LocalFileTarget, LocalDirectoryTarget)):
 715             return True
 716         return False
 717
 718     def announce_success(self, msg):
 719         if self.verbosity >= 1:
 720             print >>self.stdout, "Success: %s" % msg
 721         return 0
 722
 723     def copy_file(self, source, target):
 724         assert isinstance(source, (LocalFileSource, TahoeFileSource))
 725         assert isinstance(target, (LocalFileTarget, TahoeFileTarget,
 726                                    LocalMissingTarget, TahoeMissingTarget))
 727         if self.need_to_copy_bytes(source, target):
 728             # if the target is a local directory, this will just write the
 729             # bytes to disk. If it is a tahoe directory, it will upload the
 730             # data, and stash the new filecap for a later set_children call.
 731             f = source.open(self.caps_only)
 732             target.put_file(f)
 733             return self.announce_success("file copied")
 734         # otherwise we're copying tahoe to tahoe, and using immutable files,
 735         # so we can just make a link. TODO: this probably won't always work:
 736         # need to enumerate the cases and analyze them.
 737         target.put_uri(source.bestcap())
 738         return self.announce_success("file linked")
 739
 740     def copy_file_into(self, source, name, target):
 741         assert isinstance(source, (LocalFileSource, TahoeFileSource))
 742         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 743         if self.need_to_copy_bytes(source, target):
 744             # if the target is a local directory, this will just write the
 745             # bytes to disk. If it is a tahoe directory, it will upload the
 746             # data, and stash the new filecap for a later set_children call.
 747             f = source.open(self.caps_only)
 748             target.put_file(name, f)
 749             return
 750         # otherwise we're copying tahoe to tahoe, and using immutable files,
 751         # so we can just make a link
 752         target.put_uri(name, source.bestcap())
 753
 754
 755     def progress(self, message):
 756         #print message
 757         if self.progressfunc:
 758             self.progressfunc(message)
 759
 760     def build_graphs(self, source_infos):
 761         graphs = []
 762         for name,source in source_infos:
 763             if isinstance(source, (LocalDirectorySource, TahoeDirectorySource)):
 764                 source.populate(True)
 765                 graphs.append(source)
 766         return graphs
 767
 768
 769 def copy(options):
 770     return Copier().do_copy(options)
 771
 772 # error cases that need improvement:
 773 #  local-file-in-the-way
 774 #   touch proposed
 775 #   tahoe cp -r my:docs/proposed/denver.txt proposed/denver.txt
 776 #  handling of unknown nodes
 777
 778 # things that maybe should be errors but aren't
 779 #  local-dir-in-the-way
 780 #   mkdir denver.txt
 781 #   tahoe cp -r my:docs/proposed/denver.txt denver.txt
 782 #   (creates denver.txt/denver.txt)
 783
 784 # error cases that look good:
 785 #  tahoe cp -r my:docs/missing missing
 786 #  disconnect servers
 787 #   tahoe cp -r my:docs/missing missing  -> No JSON object could be decoded
 788 #  tahoe-file-in-the-way (when we want to make a directory)
 789 #   tahoe put README my:docs
 790 #   tahoe cp -r docs/proposed my:docs/proposed