src/allmydata/scripts/tahoe_cp.py

   1
   2 import os.path
   3 import urllib
   4 import simplejson
   5 from cStringIO import StringIO
   6 from twisted.python.failure import Failure
   7 from allmydata.scripts.common import get_alias, escape_path, \
   8                                      DefaultAliasMarker, TahoeError
   9 from allmydata.scripts.common_http import do_http, HTTPError
  10 from allmydata import uri
  11 from allmydata.util import fileutil
  12 from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath
  13 from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \
  14     quote_local_unicode_path, to_str
  15 from allmydata.util.assertutil import precondition
  16
  17
  18 class MissingSourceError(TahoeError):
  19     def __init__(self, name, quotefn=quote_output):
  20         TahoeError.__init__(self, "No such file or directory %s" % quotefn(name))
  21
  22
  23 def GET_to_file(url):
  24     resp = do_http("GET", url)
  25     if resp.status == 200:
  26         return resp
  27     raise HTTPError("Error during GET", resp)
  28
  29 def GET_to_string(url):
  30     f = GET_to_file(url)
  31     return f.read()
  32
  33 def PUT(url, data):
  34     resp = do_http("PUT", url, data)
  35     if resp.status in (200, 201):
  36         return resp.read()
  37     raise HTTPError("Error during PUT", resp)
  38
  39 def POST(url, data):
  40     resp = do_http("POST", url, data)
  41     if resp.status in (200, 201):
  42         return resp.read()
  43     raise HTTPError("Error during POST", resp)
  44
  45 def mkdir(targeturl):
  46     url = targeturl + "?t=mkdir"
  47     resp = do_http("POST", url)
  48     if resp.status in (200, 201):
  49         return resp.read().strip()
  50     raise HTTPError("Error during mkdir", resp)
  51
  52 def make_tahoe_subdirectory(nodeurl, parent_writecap, name):
  53     url = nodeurl + "/".join(["uri",
  54                               urllib.quote(parent_writecap),
  55                               urllib.quote(unicode_to_url(name)),
  56                               ]) + "?t=mkdir"
  57     resp = do_http("POST", url)
  58     if resp.status in (200, 201):
  59         return resp.read().strip()
  60     raise HTTPError("Error during mkdir", resp)
  61
  62
  63 class LocalFileSource:
  64     def __init__(self, pathname):
  65         precondition_abspath(pathname)
  66         self.pathname = pathname
  67
  68     def need_to_copy_bytes(self):
  69         return True
  70
  71     def open(self, caps_only):
  72         return open(self.pathname, "rb")
  73
  74 class LocalFileTarget:
  75     def __init__(self, pathname):
  76         precondition_abspath(pathname)
  77         self.pathname = pathname
  78
  79     def put_file(self, inf):
  80         fileutil.put_file(self.pathname, inf)
  81
  82 class LocalMissingTarget:
  83     def __init__(self, pathname):
  84         precondition_abspath(pathname)
  85         self.pathname = pathname
  86
  87     def put_file(self, inf):
  88         fileutil.put_file(self.pathname, inf)
  89
  90 class LocalDirectorySource:
  91     def __init__(self, progressfunc, pathname):
  92         precondition_abspath(pathname)
  93
  94         self.progressfunc = progressfunc
  95         self.pathname = pathname
  96         self.children = None
  97
  98     def populate(self, recurse):
  99         if self.children is not None:
 100             return
 101         self.children = {}
 102         children = listdir_unicode(self.pathname)
 103         for i,n in enumerate(children):
 104             self.progressfunc("examining %d of %d" % (i+1, len(children)))
 105             pn = os.path.join(self.pathname, n)
 106             if os.path.isdir(pn):
 107                 child = LocalDirectorySource(self.progressfunc, pn)
 108                 self.children[n] = child
 109                 if recurse:
 110                     child.populate(True)
 111             elif os.path.isfile(pn):
 112                 self.children[n] = LocalFileSource(pn)
 113             else:
 114                 # Could be dangling symlink; probably not copy-able.
 115                 # TODO: output a warning
 116                 pass
 117
 118 class LocalDirectoryTarget:
 119     def __init__(self, progressfunc, pathname):
 120         precondition_abspath(pathname)
 121
 122         self.progressfunc = progressfunc
 123         self.pathname = pathname
 124         self.children = None
 125
 126     def populate(self, recurse):
 127         if self.children is not None:
 128             return
 129         self.children = {}
 130         children = listdir_unicode(self.pathname)
 131         for i,n in enumerate(children):
 132             self.progressfunc("examining %d of %d" % (i+1, len(children)))
 133             n = unicode(n)
 134             pn = os.path.join(self.pathname, n)
 135             if os.path.isdir(pn):
 136                 child = LocalDirectoryTarget(self.progressfunc, pn)
 137                 self.children[n] = child
 138                 if recurse:
 139                     child.populate(True)
 140             else:
 141                 assert os.path.isfile(pn)
 142                 self.children[n] = LocalFileTarget(pn)
 143
 144     def get_child_target(self, name):
 145         if self.children is None:
 146             self.populate(False)
 147         if name in self.children:
 148             return self.children[name]
 149         pathname = os.path.join(self.pathname, name)
 150         os.makedirs(pathname)
 151         return LocalDirectoryTarget(self.progressfunc, pathname)
 152
 153     def put_file(self, name, inf):
 154         precondition(isinstance(name, unicode), name)
 155         pathname = os.path.join(self.pathname, name)
 156         fileutil.put_file(pathname, inf)
 157
 158     def set_children(self):
 159         pass
 160
 161
 162 class TahoeFileSource:
 163     def __init__(self, nodeurl, mutable, writecap, readcap):
 164         self.nodeurl = nodeurl
 165         self.mutable = mutable
 166         self.writecap = writecap
 167         self.readcap = readcap
 168
 169     def need_to_copy_bytes(self):
 170         if self.mutable:
 171             return True
 172         return False
 173
 174     def open(self, caps_only):
 175         if caps_only:
 176             return StringIO(self.readcap)
 177         url = self.nodeurl + "uri/" + urllib.quote(self.readcap)
 178         return GET_to_file(url)
 179
 180     def bestcap(self):
 181         return self.writecap or self.readcap
 182
 183 class TahoeFileTarget:
 184     def __init__(self, nodeurl, mutable, writecap, readcap, url):
 185         self.nodeurl = nodeurl
 186         self.mutable = mutable
 187         self.writecap = writecap
 188         self.readcap = readcap
 189         self.url = url
 190
 191     def put_file(self, inf):
 192         # We want to replace this object in-place.
 193         assert self.url
 194         # our do_http() call currently requires a string or a filehandle with
 195         # a real .seek
 196         if not hasattr(inf, "seek"):
 197             inf = inf.read()
 198         PUT(self.url, inf)
 199         # TODO: this always creates immutable files. We might want an option
 200         # to always create mutable files, or to copy mutable files into new
 201         # mutable files. ticket #835
 202
 203 class TahoeDirectorySource:
 204     def __init__(self, nodeurl, cache, progressfunc):
 205         self.nodeurl = nodeurl
 206         self.cache = cache
 207         self.progressfunc = progressfunc
 208
 209     def init_from_grid(self, writecap, readcap):
 210         self.writecap = writecap
 211         self.readcap = readcap
 212         bestcap = writecap or readcap
 213         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 214         resp = do_http("GET", url + "?t=json")
 215         if resp.status != 200:
 216             raise HTTPError("Error examining source directory", resp)
 217         parsed = simplejson.loads(resp.read())
 218         nodetype, d = parsed
 219         assert nodetype == "dirnode"
 220         self.mutable = d.get("mutable", False) # older nodes don't provide it
 221         self.children_d = dict( [(unicode(name),value)
 222                                  for (name,value)
 223                                  in d["children"].iteritems()] )
 224         self.children = None
 225
 226     def init_from_parsed(self, parsed):
 227         nodetype, d = parsed
 228         self.writecap = to_str(d.get("rw_uri"))
 229         self.readcap = to_str(d.get("ro_uri"))
 230         self.mutable = d.get("mutable", False) # older nodes don't provide it
 231         self.children_d = dict( [(unicode(name),value)
 232                                  for (name,value)
 233                                  in d["children"].iteritems()] )
 234         self.children = None
 235
 236     def populate(self, recurse):
 237         if self.children is not None:
 238             return
 239         self.children = {}
 240         for i,(name, data) in enumerate(self.children_d.items()):
 241             self.progressfunc("examining %d of %d" % (i+1, len(self.children_d)))
 242             if data[0] == "filenode":
 243                 mutable = data[1].get("mutable", False)
 244                 writecap = to_str(data[1].get("rw_uri"))
 245                 readcap = to_str(data[1].get("ro_uri"))
 246                 self.children[name] = TahoeFileSource(self.nodeurl, mutable,
 247                                                       writecap, readcap)
 248             elif data[0] == "dirnode":
 249                 writecap = to_str(data[1].get("rw_uri"))
 250                 readcap = to_str(data[1].get("ro_uri"))
 251                 if writecap and writecap in self.cache:
 252                     child = self.cache[writecap]
 253                 elif readcap and readcap in self.cache:
 254                     child = self.cache[readcap]
 255                 else:
 256                     child = TahoeDirectorySource(self.nodeurl, self.cache,
 257                                                  self.progressfunc)
 258                     child.init_from_grid(writecap, readcap)
 259                     if writecap:
 260                         self.cache[writecap] = child
 261                     if readcap:
 262                         self.cache[readcap] = child
 263                     if recurse:
 264                         child.populate(True)
 265                 self.children[name] = child
 266             else:
 267                 # TODO: there should be an option to skip unknown nodes.
 268                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 269                                  "You probably need to use a later version of "
 270                                  "Tahoe-LAFS to copy this directory.")
 271
 272 class TahoeMissingTarget:
 273     def __init__(self, url):
 274         self.url = url
 275
 276     def put_file(self, inf):
 277         # We want to replace this object in-place.
 278         if not hasattr(inf, "seek"):
 279             inf = inf.read()
 280         PUT(self.url, inf)
 281         # TODO: this always creates immutable files. We might want an option
 282         # to always create mutable files, or to copy mutable files into new
 283         # mutable files.
 284
 285     def put_uri(self, filecap):
 286         # I'm not sure this will always work
 287         return PUT(self.url + "?t=uri", filecap)
 288
 289 class TahoeDirectoryTarget:
 290     def __init__(self, nodeurl, cache, progressfunc):
 291         self.nodeurl = nodeurl
 292         self.cache = cache
 293         self.progressfunc = progressfunc
 294         self.new_children = {}
 295
 296     def init_from_parsed(self, parsed):
 297         nodetype, d = parsed
 298         self.writecap = to_str(d.get("rw_uri"))
 299         self.readcap = to_str(d.get("ro_uri"))
 300         self.mutable = d.get("mutable", False) # older nodes don't provide it
 301         self.children_d = dict( [(unicode(name),value)
 302                                  for (name,value)
 303                                  in d["children"].iteritems()] )
 304         self.children = None
 305
 306     def init_from_grid(self, writecap, readcap):
 307         self.writecap = writecap
 308         self.readcap = readcap
 309         bestcap = writecap or readcap
 310         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 311         resp = do_http("GET", url + "?t=json")
 312         if resp.status != 200:
 313             raise HTTPError("Error examining target directory", resp)
 314         parsed = simplejson.loads(resp.read())
 315         nodetype, d = parsed
 316         assert nodetype == "dirnode"
 317         self.mutable = d.get("mutable", False) # older nodes don't provide it
 318         self.children_d = dict( [(unicode(name),value)
 319                                  for (name,value)
 320                                  in d["children"].iteritems()] )
 321         self.children = None
 322
 323     def just_created(self, writecap):
 324         self.writecap = writecap
 325         self.readcap = uri.from_string(writecap).get_readonly().to_string()
 326         self.mutable = True
 327         self.children_d = {}
 328         self.children = {}
 329
 330     def populate(self, recurse):
 331         if self.children is not None:
 332             return
 333         self.children = {}
 334         for i,(name, data) in enumerate(self.children_d.items()):
 335             self.progressfunc("examining %d of %d" % (i+1, len(self.children_d)))
 336             if data[0] == "filenode":
 337                 mutable = data[1].get("mutable", False)
 338                 writecap = to_str(data[1].get("rw_uri"))
 339                 readcap = to_str(data[1].get("ro_uri"))
 340                 url = None
 341                 if self.writecap:
 342                     url = self.nodeurl + "/".join(["uri",
 343                                                    urllib.quote(self.writecap),
 344                                                    urllib.quote(unicode_to_url(name))])
 345                 self.children[name] = TahoeFileTarget(self.nodeurl, mutable,
 346                                                       writecap, readcap, url)
 347             elif data[0] == "dirnode":
 348                 writecap = to_str(data[1].get("rw_uri"))
 349                 readcap = to_str(data[1].get("ro_uri"))
 350                 if writecap and writecap in self.cache:
 351                     child = self.cache[writecap]
 352                 elif readcap and readcap in self.cache:
 353                     child = self.cache[readcap]
 354                 else:
 355                     child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 356                                                  self.progressfunc)
 357                     child.init_from_grid(writecap, readcap)
 358                     if writecap:
 359                         self.cache[writecap] = child
 360                     if readcap:
 361                         self.cache[readcap] = child
 362                     if recurse:
 363                         child.populate(True)
 364                 self.children[name] = child
 365             else:
 366                 # TODO: there should be an option to skip unknown nodes.
 367                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 368                                  "You probably need to use a later version of "
 369                                  "Tahoe-LAFS to copy this directory.")
 370
 371     def get_child_target(self, name):
 372         # return a new target for a named subdirectory of this dir
 373         if self.children is None:
 374             self.populate(False)
 375         if name in self.children:
 376             return self.children[name]
 377         writecap = make_tahoe_subdirectory(self.nodeurl, self.writecap, name)
 378         child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 379                                      self.progressfunc)
 380         child.just_created(writecap)
 381         self.children[name] = child
 382         return child
 383
 384     def put_file(self, name, inf):
 385         url = self.nodeurl + "uri"
 386         if not hasattr(inf, "seek"):
 387             inf = inf.read()
 388
 389         if self.children is None:
 390             self.populate(False)
 391
 392         # Check to see if we already have a mutable file by this name.
 393         # If so, overwrite that file in place.
 394         if name in self.children and self.children[name].mutable:
 395             self.children[name].put_file(inf)
 396         else:
 397             filecap = PUT(url, inf)
 398             # TODO: this always creates immutable files. We might want an option
 399             # to always create mutable files, or to copy mutable files into new
 400             # mutable files.
 401             self.new_children[name] = filecap
 402
 403     def put_uri(self, name, filecap):
 404         self.new_children[name] = filecap
 405
 406     def set_children(self):
 407         if not self.new_children:
 408             return
 409         url = (self.nodeurl + "uri/" + urllib.quote(self.writecap)
 410                + "?t=set_children")
 411         set_data = {}
 412         for (name, filecap) in self.new_children.items():
 413             # it just so happens that ?t=set_children will accept both file
 414             # read-caps and write-caps as ['rw_uri'], and will handle either
 415             # correctly. So don't bother trying to figure out whether the one
 416             # we have is read-only or read-write.
 417             # TODO: think about how this affects forward-compatibility for
 418             # unknown caps
 419             set_data[name] = ["filenode", {"rw_uri": filecap}]
 420         body = simplejson.dumps(set_data)
 421         POST(url, body)
 422
 423 class Copier:
 424
 425     def do_copy(self, options, progressfunc=None):
 426         if options['quiet']:
 427             verbosity = 0
 428         elif options['verbose']:
 429             verbosity = 2
 430         else:
 431             verbosity = 1
 432
 433         nodeurl = options['node-url']
 434         if nodeurl[-1] != "/":
 435             nodeurl += "/"
 436         self.nodeurl = nodeurl
 437         self.progressfunc = progressfunc
 438         self.options = options
 439         self.aliases = options.aliases
 440         self.verbosity = verbosity
 441         self.stdout = options.stdout
 442         self.stderr = options.stderr
 443         if verbosity >= 2 and not self.progressfunc:
 444             def progress(message):
 445                 print >>self.stderr, message
 446             self.progressfunc = progress
 447         self.caps_only = options["caps-only"]
 448         self.cache = {}
 449         try:
 450             status = self.try_copy()
 451             return status
 452         except TahoeError, te:
 453             if verbosity >= 2:
 454                 Failure().printTraceback(self.stderr)
 455                 print >>self.stderr
 456             te.display(self.stderr)
 457             return 1
 458
 459     def try_copy(self):
 460         source_specs = self.options.sources
 461         destination_spec = self.options.destination
 462         recursive = self.options["recursive"]
 463
 464         target = self.get_target_info(destination_spec)
 465
 466         sources = [] # list of (name, source object)
 467         for ss in source_specs:
 468             name, source = self.get_source_info(ss)
 469             sources.append( (name, source) )
 470
 471         del name
 472         have_source_dirs = bool([s for (name,s) in sources
 473                                  if isinstance(s, (LocalDirectorySource,
 474                                                    TahoeDirectorySource))])
 475
 476         if have_source_dirs and not recursive:
 477             self.to_stderr("cannot copy directories without --recursive")
 478             return 1
 479
 480         if isinstance(target, (LocalFileTarget, TahoeFileTarget)):
 481             # cp STUFF foo.txt, where foo.txt already exists. This limits the
 482             # possibilities considerably.
 483             if len(sources) > 1:
 484                 self.to_stderr("target %s is not a directory" % quote_output(destination_spec))
 485                 return 1
 486             if have_source_dirs:
 487                 self.to_stderr("cannot copy directory into a file")
 488                 return 1
 489             name, source = sources[0]
 490             return self.copy_file(source, target)
 491
 492         if isinstance(target, (LocalMissingTarget, TahoeMissingTarget)):
 493             if recursive:
 494                 return self.copy_to_directory(sources, target)
 495             if len(sources) > 1:
 496                 # if we have -r, we'll auto-create the target directory. Without
 497                 # it, we'll only create a file.
 498                 self.to_stderr("cannot copy multiple files into a file without -r")
 499                 return 1
 500             # cp file1 newfile
 501             name, source = sources[0]
 502             return self.copy_file(source, target)
 503
 504         if isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)):
 505             # We're copying to an existing directory -- make sure that we
 506             # have target names for everything
 507             for (name, source) in sources:
 508                 if name is None and isinstance(source, TahoeFileSource):
 509                     self.to_stderr(
 510                         "error: you must specify a destination filename")
 511                     return 1
 512             return self.copy_to_directory(sources, target)
 513
 514         self.to_stderr("unknown target")
 515         return 1
 516
 517     def to_stderr(self, text):
 518         print >>self.stderr, text
 519
 520     # FIXME reduce the amount of near-duplicate code between get_target_info and get_source_info.
 521
 522     def get_target_info(self, destination_spec):
 523         rootcap, path = get_alias(self.aliases, destination_spec, None)
 524         if rootcap == DefaultAliasMarker:
 525             # no alias, so this is a local file
 526             pathname = abspath_expanduser_unicode(path.decode('utf-8'))
 527             if not os.path.exists(pathname):
 528                 t = LocalMissingTarget(pathname)
 529             elif os.path.isdir(pathname):
 530                 t = LocalDirectoryTarget(self.progress, pathname)
 531             else:
 532                 assert os.path.isfile(pathname), pathname
 533                 t = LocalFileTarget(pathname) # non-empty
 534         else:
 535             # this is a tahoe object
 536             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 537             if path:
 538                 url += "/" + escape_path(path)
 539
 540             resp = do_http("GET", url + "?t=json")
 541             if resp.status == 404:
 542                 # doesn't exist yet
 543                 t = TahoeMissingTarget(url)
 544             elif resp.status == 200:
 545                 parsed = simplejson.loads(resp.read())
 546                 nodetype, d = parsed
 547                 if nodetype == "dirnode":
 548                     t = TahoeDirectoryTarget(self.nodeurl, self.cache,
 549                                              self.progress)
 550                     t.init_from_parsed(parsed)
 551                 else:
 552                     writecap = to_str(d.get("rw_uri"))
 553                     readcap = to_str(d.get("ro_uri"))
 554                     mutable = d.get("mutable", False)
 555                     t = TahoeFileTarget(self.nodeurl, mutable,
 556                                         writecap, readcap, url)
 557             else:
 558                 raise HTTPError("Error examining target %s"
 559                                  % quote_output(destination_spec), resp)
 560         return t
 561
 562     def get_source_info(self, source_spec):
 563         rootcap, path = get_alias(self.aliases, source_spec, None)
 564         if rootcap == DefaultAliasMarker:
 565             # no alias, so this is a local file
 566             pathname = abspath_expanduser_unicode(path.decode('utf-8'))
 567             name = os.path.basename(pathname)
 568             if not os.path.exists(pathname):
 569                 raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path)
 570             if os.path.isdir(pathname):
 571                 t = LocalDirectorySource(self.progress, pathname)
 572             else:
 573                 assert os.path.isfile(pathname)
 574                 t = LocalFileSource(pathname) # non-empty
 575         else:
 576             # this is a tahoe object
 577             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 578             name = None
 579             if path:
 580                 url += "/" + escape_path(path)
 581                 last_slash = path.rfind("/")
 582                 name = path
 583                 if last_slash:
 584                     name = path[last_slash+1:]
 585
 586             resp = do_http("GET", url + "?t=json")
 587             if resp.status == 404:
 588                 raise MissingSourceError(source_spec)
 589             elif resp.status != 200:
 590                 raise HTTPError("Error examining source %s" % quote_output(source_spec),
 591                                 resp)
 592             parsed = simplejson.loads(resp.read())
 593             nodetype, d = parsed
 594             if nodetype == "dirnode":
 595                 t = TahoeDirectorySource(self.nodeurl, self.cache,
 596                                          self.progress)
 597                 t.init_from_parsed(parsed)
 598             else:
 599                 writecap = to_str(d.get("rw_uri"))
 600                 readcap = to_str(d.get("ro_uri"))
 601                 mutable = d.get("mutable", False) # older nodes don't provide it
 602                 if source_spec.rfind('/') != -1:
 603                     name = source_spec[source_spec.rfind('/')+1:]
 604                 t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap)
 605         return name, t
 606
 607
 608     def dump_graph(self, s, indent=" "):
 609         for name, child in s.children.items():
 610             print "%s%s: %r" % (indent, quote_output(name), child)
 611             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 612                 self.dump_graph(child, indent+"  ")
 613
 614     def copy_to_directory(self, source_infos, target):
 615         # step one: build a recursive graph of the source tree. This returns
 616         # a dictionary, with child names as keys, and values that are either
 617         # Directory or File instances (local or tahoe).
 618         source_dirs = self.build_graphs(source_infos)
 619         source_files = [source for source in source_infos
 620                         if isinstance(source[1], (LocalFileSource,
 621                                                   TahoeFileSource))]
 622
 623         #print "graphs"
 624         #for s in source_dirs:
 625         #    self.dump_graph(s)
 626
 627         # step two: create the top-level target directory object
 628         if isinstance(target, LocalMissingTarget):
 629             os.makedirs(target.pathname)
 630             target = LocalDirectoryTarget(self.progress, target.pathname)
 631         elif isinstance(target, TahoeMissingTarget):
 632             writecap = mkdir(target.url)
 633             target = TahoeDirectoryTarget(self.nodeurl, self.cache,
 634                                           self.progress)
 635             target.just_created(writecap)
 636         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 637         target.populate(False)
 638
 639         # step three: find a target for each source node, creating
 640         # directories as necessary. 'targetmap' is a dictionary that uses
 641         # target Directory instances as keys, and has values of
 642         # (name->sourceobject) dicts for all the files that need to wind up
 643         # there.
 644
 645         # sources are all LocalFile/LocalDirectory/TahoeFile/TahoeDirectory
 646         # target is LocalDirectory/TahoeDirectory
 647
 648         self.progress("attaching sources to targets, "
 649                       "%d files / %d dirs in root" %
 650                       (len(source_files), len(source_dirs)))
 651
 652         self.targetmap = {}
 653         self.files_to_copy = 0
 654
 655         for (name,s) in source_files:
 656             self.attach_to_target(s, name, target)
 657
 658         for (name, source) in source_dirs:
 659             new_target = target.get_child_target(name)
 660             self.assign_targets(source, new_target)
 661
 662         self.progress("targets assigned, %s dirs, %s files" %
 663                       (len(self.targetmap), self.files_to_copy))
 664
 665         self.progress("starting copy, %d files, %d directories" %
 666                       (self.files_to_copy, len(self.targetmap)))
 667         self.files_copied = 0
 668         self.targets_finished = 0
 669
 670         # step four: walk through the list of targets. For each one, copy all
 671         # the files. If the target is a TahoeDirectory, upload and create
 672         # read-caps, then do a set_children to the target directory.
 673
 674         for target in self.targetmap:
 675             self.copy_files_to_target(self.targetmap[target], target)
 676             self.targets_finished += 1
 677             self.progress("%d/%d directories" %
 678                           (self.targets_finished, len(self.targetmap)))
 679
 680         return self.announce_success("files copied")
 681
 682     def attach_to_target(self, source, name, target):
 683         if target not in self.targetmap:
 684             self.targetmap[target] = {}
 685         self.targetmap[target][name] = source
 686         self.files_to_copy += 1
 687
 688     def assign_targets(self, source, target):
 689         # copy everything in the source into the target
 690         assert isinstance(source, (LocalDirectorySource, TahoeDirectorySource))
 691
 692         for name, child in source.children.items():
 693             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 694                 # we will need a target directory for this one
 695                 subtarget = target.get_child_target(name)
 696                 self.assign_targets(child, subtarget)
 697             else:
 698                 assert isinstance(child, (LocalFileSource, TahoeFileSource))
 699                 self.attach_to_target(child, name, target)
 700
 701
 702
 703     def copy_files_to_target(self, targetmap, target):
 704         for name, source in targetmap.items():
 705             assert isinstance(source, (LocalFileSource, TahoeFileSource))
 706             self.copy_file_into(source, name, target)
 707             self.files_copied += 1
 708             self.progress("%d/%d files, %d/%d directories" %
 709                           (self.files_copied, self.files_to_copy,
 710                            self.targets_finished, len(self.targetmap)))
 711         target.set_children()
 712
 713     def need_to_copy_bytes(self, source, target):
 714         if source.need_to_copy_bytes:
 715             # mutable tahoe files, and local files
 716             return True
 717         if isinstance(target, (LocalFileTarget, LocalDirectoryTarget)):
 718             return True
 719         return False
 720
 721     def announce_success(self, msg):
 722         if self.verbosity >= 1:
 723             print >>self.stdout, "Success: %s" % msg
 724         return 0
 725
 726     def copy_file(self, source, target):
 727         assert isinstance(source, (LocalFileSource, TahoeFileSource))
 728         assert isinstance(target, (LocalFileTarget, TahoeFileTarget,
 729                                    LocalMissingTarget, TahoeMissingTarget))
 730         if self.need_to_copy_bytes(source, target):
 731             # if the target is a local directory, this will just write the
 732             # bytes to disk. If it is a tahoe directory, it will upload the
 733             # data, and stash the new filecap for a later set_children call.
 734             f = source.open(self.caps_only)
 735             target.put_file(f)
 736             return self.announce_success("file copied")
 737         # otherwise we're copying tahoe to tahoe, and using immutable files,
 738         # so we can just make a link. TODO: this probably won't always work:
 739         # need to enumerate the cases and analyze them.
 740         target.put_uri(source.bestcap())
 741         return self.announce_success("file linked")
 742
 743     def copy_file_into(self, source, name, target):
 744         assert isinstance(source, (LocalFileSource, TahoeFileSource))
 745         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 746         if self.need_to_copy_bytes(source, target):
 747             # if the target is a local directory, this will just write the
 748             # bytes to disk. If it is a tahoe directory, it will upload the
 749             # data, and stash the new filecap for a later set_children call.
 750             f = source.open(self.caps_only)
 751             target.put_file(name, f)
 752             return
 753         # otherwise we're copying tahoe to tahoe, and using immutable files,
 754         # so we can just make a link
 755         target.put_uri(name, source.bestcap())
 756
 757
 758     def progress(self, message):
 759         #print message
 760         if self.progressfunc:
 761             self.progressfunc(message)
 762
 763     def build_graphs(self, source_infos):
 764         graphs = []
 765         for name,source in source_infos:
 766             if isinstance(source, (LocalDirectorySource, TahoeDirectorySource)):
 767                 source.populate(True)
 768                 # Remove trailing slash (if applicable) and get dir name
 769                 name = os.path.basename(os.path.normpath(name))
 770                 graphs.append((name, source))
 771         return graphs
 772
 773
 774 def copy(options):
 775     return Copier().do_copy(options)
 776
 777 # error cases that need improvement:
 778 #  local-file-in-the-way
 779 #   touch proposed
 780 #   tahoe cp -r my:docs/proposed/denver.txt proposed/denver.txt
 781 #  handling of unknown nodes
 782
 783 # things that maybe should be errors but aren't
 784 #  local-dir-in-the-way
 785 #   mkdir denver.txt
 786 #   tahoe cp -r my:docs/proposed/denver.txt denver.txt
 787 #   (creates denver.txt/denver.txt)
 788
 789 # error cases that look good:
 790 #  tahoe cp -r my:docs/missing missing
 791 #  disconnect servers
 792 #   tahoe cp -r my:docs/missing missing  -> No JSON object could be decoded
 793 #  tahoe-file-in-the-way (when we want to make a directory)
 794 #   tahoe put README my:docs
 795 #   tahoe cp -r docs/proposed my:docs/proposed