src/allmydata/scripts/tahoe_cp.py

   1
   2 import os.path
   3 import urllib
   4 import simplejson
   5 from cStringIO import StringIO
   6 from twisted.python.failure import Failure
   7 from allmydata.scripts.common import get_alias, escape_path, \
   8                                      DefaultAliasMarker, UnknownAliasError
   9 from allmydata.scripts.common_http import do_http
  10 from allmydata import uri
  11
  12 def ascii_or_none(s):
  13     if s is None:
  14         return s
  15     return str(s)
  16
  17 class TahoeError(Exception):
  18     def __init__(self, msg, resp):
  19         self.msg = msg
  20         self.status = resp.status
  21         self.reason = resp.reason
  22         self.body = resp.read()
  23
  24     def display(self, err):
  25         print >>err, "%s: %s %s" % (self.msg, self.status, self.reason)
  26         print >>err, self.body
  27
  28 class MissingSourceError(Exception):
  29     pass
  30
  31 def GET_to_file(url):
  32     resp = do_http("GET", url)
  33     if resp.status == 200:
  34         return resp
  35     raise TahoeError("Error during GET", resp)
  36
  37 def GET_to_string(url):
  38     f = GET_to_file(url)
  39     return f.read()
  40
  41 def PUT(url, data):
  42     resp = do_http("PUT", url, data)
  43     if resp.status in (200, 201):
  44         return resp.read()
  45     raise TahoeError("Error during PUT", resp)
  46
  47 def POST(url, data):
  48     resp = do_http("POST", url, data)
  49     if resp.status in (200, 201):
  50         return resp.read()
  51     raise TahoeError("Error during POST", resp)
  52
  53 def mkdir(targeturl):
  54     url = targeturl + "?t=mkdir"
  55     resp = do_http("POST", url)
  56     if resp.status in (200, 201):
  57         return resp.read().strip()
  58     raise TahoeError("Error during mkdir", resp)
  59
  60 def make_tahoe_subdirectory(nodeurl, parent_writecap, name):
  61     url = nodeurl + "/".join(["uri",
  62                               urllib.quote(parent_writecap),
  63                               urllib.quote(name),
  64                               ]) + "?t=mkdir"
  65     resp = do_http("POST", url)
  66     if resp.status in (200, 201):
  67         return resp.read().strip()
  68     raise TahoeError("Error during mkdir", resp)
  69
  70
  71 class LocalFileSource:
  72     def __init__(self, pathname):
  73         self.pathname = pathname
  74
  75     def need_to_copy_bytes(self):
  76         return True
  77
  78     def open(self, caps_only):
  79         return open(self.pathname, "rb")
  80
  81 class LocalFileTarget:
  82     def __init__(self, pathname):
  83         self.pathname = pathname
  84     def put_file(self, inf):
  85         outf = open(self.pathname, "wb")
  86         while True:
  87             data = inf.read(32768)
  88             if not data:
  89                 break
  90             outf.write(data)
  91         outf.close()
  92
  93 class LocalMissingTarget:
  94     def __init__(self, pathname):
  95         self.pathname = pathname
  96
  97     def put_file(self, inf):
  98         outf = open(self.pathname, "wb")
  99         while True:
 100             data = inf.read(32768)
 101             if not data:
 102                 break
 103             outf.write(data)
 104         outf.close()
 105
 106 class LocalDirectorySource:
 107     def __init__(self, progressfunc, pathname):
 108         self.progressfunc = progressfunc
 109         self.pathname = pathname
 110         self.children = None
 111
 112     def populate(self, recurse):
 113         if self.children is not None:
 114             return
 115         self.children = {}
 116         children = os.listdir(self.pathname)
 117         for i,n in enumerate(children):
 118             self.progressfunc("examining %d of %d" % (i, len(children)))
 119             pn = os.path.join(self.pathname, n)
 120             if os.path.isdir(pn):
 121                 child = LocalDirectorySource(self.progressfunc, pn)
 122                 self.children[n] = child
 123                 if recurse:
 124                     child.populate(True)
 125             elif os.path.isfile(pn):
 126                 self.children[n] = LocalFileSource(pn)
 127             else:
 128                 # Could be dangling symlink; probably not copy-able.
 129                 pass
 130
 131 class LocalDirectoryTarget:
 132     def __init__(self, progressfunc, pathname):
 133         self.progressfunc = progressfunc
 134         self.pathname = pathname
 135         self.children = None
 136
 137     def populate(self, recurse):
 138         if self.children is not None:
 139             return
 140         self.children = {}
 141         children = os.listdir(self.pathname)
 142         for i,n in enumerate(children):
 143             self.progressfunc("examining %d of %d" % (i, len(children)))
 144             pn = os.path.join(self.pathname, n)
 145             if os.path.isdir(pn):
 146                 child = LocalDirectoryTarget(self.progressfunc, pn)
 147                 self.children[n] = child
 148                 if recurse:
 149                     child.populate(True)
 150             else:
 151                 assert os.path.isfile(pn)
 152                 self.children[n] = LocalFileTarget(pn)
 153
 154     def get_child_target(self, name):
 155         if self.children is None:
 156             self.populate(False)
 157         if name in self.children:
 158             return self.children[name]
 159         pathname = os.path.join(self.pathname, name)
 160         os.makedirs(pathname)
 161         return LocalDirectoryTarget(self.progressfunc, pathname)
 162
 163     def put_file(self, name, inf):
 164         pathname = os.path.join(self.pathname, name)
 165         outf = open(pathname, "wb")
 166         while True:
 167             data = inf.read(32768)
 168             if not data:
 169                 break
 170             outf.write(data)
 171         outf.close()
 172
 173     def set_children(self):
 174         pass
 175
 176 class TahoeFileSource:
 177     def __init__(self, nodeurl, mutable, writecap, readcap):
 178         self.nodeurl = nodeurl
 179         self.mutable = mutable
 180         self.writecap = writecap
 181         self.readcap = readcap
 182
 183     def need_to_copy_bytes(self):
 184         if self.mutable:
 185             return True
 186         return False
 187
 188     def open(self, caps_only):
 189         if caps_only:
 190             return StringIO(self.readcap)
 191         url = self.nodeurl + "uri/" + urllib.quote(self.readcap)
 192         return GET_to_file(url)
 193
 194     def bestcap(self):
 195         return self.writecap or self.readcap
 196
 197 class TahoeFileTarget:
 198     def __init__(self, nodeurl, mutable, writecap, readcap, url):
 199         self.nodeurl = nodeurl
 200         self.mutable = mutable
 201         self.writecap = writecap
 202         self.readcap = readcap
 203         self.url = url
 204
 205     def put_file(self, inf):
 206         # We want to replace this object in-place.
 207         assert self.url
 208         # our do_http() call currently requires a string or a filehandle with
 209         # a real .seek
 210         if not hasattr(inf, "seek"):
 211             inf = inf.read()
 212         PUT(self.url, inf)
 213         # TODO: this always creates immutable files. We might want an option
 214         # to always create mutable files, or to copy mutable files into new
 215         # mutable files. ticket #835
 216
 217 class TahoeDirectorySource:
 218     def __init__(self, nodeurl, cache, progressfunc):
 219         self.nodeurl = nodeurl
 220         self.cache = cache
 221         self.progressfunc = progressfunc
 222
 223     def init_from_grid(self, writecap, readcap):
 224         self.writecap = writecap
 225         self.readcap = readcap
 226         bestcap = writecap or readcap
 227         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 228         resp = do_http("GET", url + "?t=json")
 229         if resp.status != 200:
 230             raise TahoeError("Error examining source directory", resp)
 231         parsed = simplejson.loads(resp.read())
 232         nodetype, d = parsed
 233         assert nodetype == "dirnode"
 234         self.mutable = d.get("mutable", False) # older nodes don't provide it
 235         self.children_d = dict( [(unicode(name),value)
 236                                  for (name,value)
 237                                  in d["children"].iteritems()] )
 238         self.children = None
 239
 240     def init_from_parsed(self, parsed):
 241         nodetype, d = parsed
 242         self.writecap = ascii_or_none(d.get("rw_uri"))
 243         self.readcap = ascii_or_none(d.get("ro_uri"))
 244         self.mutable = d.get("mutable", False) # older nodes don't provide it
 245         self.children_d = dict( [(unicode(name),value)
 246                                  for (name,value)
 247                                  in d["children"].iteritems()] )
 248         self.children = None
 249
 250     def populate(self, recurse):
 251         if self.children is not None:
 252             return
 253         self.children = {}
 254         for i,(name, data) in enumerate(self.children_d.items()):
 255             self.progressfunc("examining %d of %d" % (i, len(self.children_d)))
 256             if data[0] == "filenode":
 257                 mutable = data[1].get("mutable", False)
 258                 writecap = ascii_or_none(data[1].get("rw_uri"))
 259                 readcap = ascii_or_none(data[1].get("ro_uri"))
 260                 self.children[name] = TahoeFileSource(self.nodeurl, mutable,
 261                                                       writecap, readcap)
 262             elif data[0] == "dirnode":
 263                 writecap = ascii_or_none(data[1].get("rw_uri"))
 264                 readcap = ascii_or_none(data[1].get("ro_uri"))
 265                 if writecap and writecap in self.cache:
 266                     child = self.cache[writecap]
 267                 elif readcap and readcap in self.cache:
 268                     child = self.cache[readcap]
 269                 else:
 270                     child = TahoeDirectorySource(self.nodeurl, self.cache,
 271                                                  self.progressfunc)
 272                     child.init_from_grid(writecap, readcap)
 273                     if writecap:
 274                         self.cache[writecap] = child
 275                     if readcap:
 276                         self.cache[readcap] = child
 277                     if recurse:
 278                         child.populate(True)
 279                 self.children[name] = child
 280             else:
 281                 # TODO: there should be an option to skip unknown nodes.
 282                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 283                                  "You probably need to use a later version of "
 284                                  "Tahoe-LAFS to copy this directory.")
 285
 286 class TahoeMissingTarget:
 287     def __init__(self, url):
 288         self.url = url
 289
 290     def put_file(self, inf):
 291         # We want to replace this object in-place.
 292         if not hasattr(inf, "seek"):
 293             inf = inf.read()
 294         PUT(self.url, inf)
 295         # TODO: this always creates immutable files. We might want an option
 296         # to always create mutable files, or to copy mutable files into new
 297         # mutable files.
 298
 299     def put_uri(self, filecap):
 300         # I'm not sure this will always work
 301         return PUT(self.url + "?t=uri", filecap)
 302
 303 class TahoeDirectoryTarget:
 304     def __init__(self, nodeurl, cache, progressfunc):
 305         self.nodeurl = nodeurl
 306         self.cache = cache
 307         self.progressfunc = progressfunc
 308         self.new_children = {}
 309
 310     def init_from_parsed(self, parsed):
 311         nodetype, d = parsed
 312         self.writecap = ascii_or_none(d.get("rw_uri"))
 313         self.readcap = ascii_or_none(d.get("ro_uri"))
 314         self.mutable = d.get("mutable", False) # older nodes don't provide it
 315         self.children_d = dict( [(unicode(name),value)
 316                                  for (name,value)
 317                                  in d["children"].iteritems()] )
 318         self.children = None
 319
 320     def init_from_grid(self, writecap, readcap):
 321         self.writecap = writecap
 322         self.readcap = readcap
 323         bestcap = writecap or readcap
 324         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 325         resp = do_http("GET", url + "?t=json")
 326         if resp.status != 200:
 327             raise TahoeError("Error examining target directory", resp)
 328         parsed = simplejson.loads(resp.read())
 329         nodetype, d = parsed
 330         assert nodetype == "dirnode"
 331         self.mutable = d.get("mutable", False) # older nodes don't provide it
 332         self.children_d = dict( [(unicode(name),value)
 333                                  for (name,value)
 334                                  in d["children"].iteritems()] )
 335         self.children = None
 336
 337     def just_created(self, writecap):
 338         self.writecap = writecap
 339         self.readcap = uri.from_string(writecap).get_readonly().to_string()
 340         self.mutable = True
 341         self.children_d = {}
 342         self.children = {}
 343
 344     def populate(self, recurse):
 345         if self.children is not None:
 346             return
 347         self.children = {}
 348         for i,(name, data) in enumerate(self.children_d.items()):
 349             self.progressfunc("examining %d of %d" % (i, len(self.children_d)))
 350             if data[0] == "filenode":
 351                 mutable = data[1].get("mutable", False)
 352                 writecap = ascii_or_none(data[1].get("rw_uri"))
 353                 readcap = ascii_or_none(data[1].get("ro_uri"))
 354                 url = None
 355                 if self.writecap:
 356                     url = self.nodeurl + "/".join(["uri",
 357                                                    urllib.quote(self.writecap),
 358                                                    urllib.quote(name.encode('utf-8'))])
 359                 self.children[name] = TahoeFileTarget(self.nodeurl, mutable,
 360                                                       writecap, readcap, url)
 361             elif data[0] == "dirnode":
 362                 writecap = ascii_or_none(data[1].get("rw_uri"))
 363                 readcap = ascii_or_none(data[1].get("ro_uri"))
 364                 if writecap and writecap in self.cache:
 365                     child = self.cache[writecap]
 366                 elif readcap and readcap in self.cache:
 367                     child = self.cache[readcap]
 368                 else:
 369                     child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 370                                                  self.progressfunc)
 371                     child.init_from_grid(writecap, readcap)
 372                     if writecap:
 373                         self.cache[writecap] = child
 374                     if readcap:
 375                         self.cache[readcap] = child
 376                     if recurse:
 377                         child.populate(True)
 378                 self.children[name] = child
 379             else:
 380                 # TODO: there should be an option to skip unknown nodes.
 381                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 382                                  "You probably need to use a later version of "
 383                                  "Tahoe-LAFS to copy this directory.")
 384
 385     def get_child_target(self, name):
 386         # return a new target for a named subdirectory of this dir
 387         if self.children is None:
 388             self.populate(False)
 389         if name in self.children:
 390             return self.children[name]
 391         writecap = make_tahoe_subdirectory(self.nodeurl, self.writecap, name)
 392         child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 393                                      self.progressfunc)
 394         child.just_created(writecap)
 395         self.children[name] = child
 396         return child
 397
 398     def put_file(self, name, inf):
 399         url = self.nodeurl + "uri"
 400         if not hasattr(inf, "seek"):
 401             inf = inf.read()
 402         filecap = PUT(url, inf)
 403         # TODO: this always creates immutable files. We might want an option
 404         # to always create mutable files, or to copy mutable files into new
 405         # mutable files.
 406         self.new_children[name] = filecap
 407
 408     def put_uri(self, name, filecap):
 409         self.new_children[name] = filecap
 410
 411     def set_children(self):
 412         if not self.new_children:
 413             return
 414         url = (self.nodeurl + "uri/" + urllib.quote(self.writecap)
 415                + "?t=set_children")
 416         set_data = {}
 417         for (name, filecap) in self.new_children.items():
 418             # it just so happens that ?t=set_children will accept both file
 419             # read-caps and write-caps as ['rw_uri'], and will handle either
 420             # correctly. So don't bother trying to figure out whether the one
 421             # we have is read-only or read-write.
 422             # TODO: think about how this affects forward-compatibility for
 423             # unknown caps
 424             set_data[name] = ["filenode", {"rw_uri": filecap}]
 425         body = simplejson.dumps(set_data)
 426         POST(url, body)
 427
 428 class Copier:
 429
 430     def do_copy(self, options, progressfunc=None):
 431         if options['quiet']:
 432             verbosity = 0
 433         elif options['verbose']:
 434             verbosity = 2
 435         else:
 436             verbosity = 1
 437
 438         nodeurl = options['node-url']
 439         if nodeurl[-1] != "/":
 440             nodeurl += "/"
 441         self.nodeurl = nodeurl
 442         self.progressfunc = progressfunc
 443         self.options = options
 444         self.aliases = options.aliases
 445         self.verbosity = verbosity
 446         self.stdout = options.stdout
 447         self.stderr = options.stderr
 448         if verbosity >= 2 and not self.progressfunc:
 449             def progress(message):
 450                 print >>self.stderr, message
 451             self.progressfunc = progress
 452         self.caps_only = options["caps-only"]
 453         self.cache = {}
 454         try:
 455             status = self.try_copy()
 456             return status
 457         except TahoeError, te:
 458             Failure().printTraceback(self.stderr)
 459             print >>self.stderr
 460             te.display(self.stderr)
 461             return 1
 462
 463     def try_copy(self):
 464         source_specs = self.options.sources
 465         destination_spec = self.options.destination
 466         recursive = self.options["recursive"]
 467
 468         try:
 469             target = self.get_target_info(destination_spec)
 470         except UnknownAliasError, e:
 471             self.to_stderr("error: %s" % e.args[0])
 472             return 1
 473
 474         try:
 475             sources = [] # list of (name, source object)
 476             for ss in source_specs:
 477                 name, source = self.get_source_info(ss)
 478                 sources.append( (name, source) )
 479         except MissingSourceError, e:
 480             self.to_stderr("No such file or directory %s" % e.args[0])
 481             return 1
 482         except UnknownAliasError, e:
 483             self.to_stderr("error: %s" % e.args[0])
 484             return 1
 485
 486         have_source_dirs = bool([s for (name,s) in sources
 487                                  if isinstance(s, (LocalDirectorySource,
 488                                                    TahoeDirectorySource))])
 489
 490         if have_source_dirs and not recursive:
 491             self.to_stderr("cannot copy directories without --recursive")
 492             return 1
 493
 494         if isinstance(target, (LocalFileTarget, TahoeFileTarget)):
 495             # cp STUFF foo.txt, where foo.txt already exists. This limits the
 496             # possibilities considerably.
 497             if len(sources) > 1:
 498                 self.to_stderr("target '%s' is not a directory" % destination_spec)
 499                 return 1
 500             if have_source_dirs:
 501                 self.to_stderr("cannot copy directory into a file")
 502                 return 1
 503             name, source = sources[0]
 504             return self.copy_file(source, target)
 505
 506         if isinstance(target, (LocalMissingTarget, TahoeMissingTarget)):
 507             if recursive:
 508                 return self.copy_to_directory(sources, target)
 509             if len(sources) > 1:
 510                 # if we have -r, we'll auto-create the target directory. Without
 511                 # it, we'll only create a file.
 512                 self.to_stderr("cannot copy multiple files into a file without -r")
 513                 return 1
 514             # cp file1 newfile
 515             name, source = sources[0]
 516             return self.copy_file(source, target)
 517
 518         if isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)):
 519             # We're copying to an existing directory -- make sure that we
 520             # have target names for everything
 521             for (name, source) in sources:
 522                 if name is None and isinstance(source, TahoeFileSource):
 523                     self.to_stderr(
 524                         "error: you must specify a destination filename")
 525                     return 1
 526             return self.copy_to_directory(sources, target)
 527
 528         self.to_stderr("unknown target")
 529         return 1
 530
 531     def to_stderr(self, text):
 532         print >>self.stderr, text
 533
 534     def get_target_info(self, destination_spec):
 535         rootcap, path = get_alias(self.aliases, destination_spec, None)
 536         if rootcap == DefaultAliasMarker:
 537             # no alias, so this is a local file
 538             pathname = os.path.abspath(os.path.expanduser(path))
 539             if not os.path.exists(pathname):
 540                 t = LocalMissingTarget(pathname)
 541             elif os.path.isdir(pathname):
 542                 t = LocalDirectoryTarget(self.progress, pathname)
 543             else:
 544                 assert os.path.isfile(pathname), pathname
 545                 t = LocalFileTarget(pathname) # non-empty
 546         else:
 547             # this is a tahoe object
 548             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 549             if path:
 550                 url += "/" + escape_path(path)
 551
 552             resp = do_http("GET", url + "?t=json")
 553             if resp.status == 404:
 554                 # doesn't exist yet
 555                 t = TahoeMissingTarget(url)
 556             elif resp.status == 200:
 557                 parsed = simplejson.loads(resp.read())
 558                 nodetype, d = parsed
 559                 if nodetype == "dirnode":
 560                     t = TahoeDirectoryTarget(self.nodeurl, self.cache,
 561                                              self.progress)
 562                     t.init_from_parsed(parsed)
 563                 else:
 564                     writecap = ascii_or_none(d.get("rw_uri"))
 565                     readcap = ascii_or_none(d.get("ro_uri"))
 566                     mutable = d.get("mutable", False)
 567                     t = TahoeFileTarget(self.nodeurl, mutable,
 568                                         writecap, readcap, url)
 569             else:
 570                 raise TahoeError("Error examining target '%s'"
 571                                  % destination_spec, resp)
 572         return t
 573
 574     def get_source_info(self, source_spec):
 575         rootcap, path = get_alias(self.aliases, source_spec, None)
 576         if rootcap == DefaultAliasMarker:
 577             # no alias, so this is a local file
 578             pathname = os.path.abspath(os.path.expanduser(path))
 579             name = os.path.basename(pathname)
 580             if not os.path.exists(pathname):
 581                 raise MissingSourceError(source_spec)
 582             if os.path.isdir(pathname):
 583                 t = LocalDirectorySource(self.progress, pathname)
 584             else:
 585                 assert os.path.isfile(pathname)
 586                 t = LocalFileSource(pathname) # non-empty
 587         else:
 588             # this is a tahoe object
 589             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 590             name = None
 591             if path:
 592                 url += "/" + escape_path(path)
 593                 last_slash = path.rfind("/")
 594                 name = path
 595                 if last_slash:
 596                     name = path[last_slash+1:]
 597
 598             resp = do_http("GET", url + "?t=json")
 599             if resp.status == 404:
 600                 raise MissingSourceError(source_spec)
 601             elif resp.status != 200:
 602                 raise TahoeError("Error examining source '%s'" % source_spec,
 603                                  resp)
 604             parsed = simplejson.loads(resp.read())
 605             nodetype, d = parsed
 606             if nodetype == "dirnode":
 607                 t = TahoeDirectorySource(self.nodeurl, self.cache,
 608                                          self.progress)
 609                 t.init_from_parsed(parsed)
 610             else:
 611                 writecap = ascii_or_none(d.get("rw_uri"))
 612                 readcap = ascii_or_none(d.get("ro_uri"))
 613                 mutable = d.get("mutable", False) # older nodes don't provide it
 614                 if source_spec.rfind('/') != -1:
 615                     name = source_spec[source_spec.rfind('/')+1:]
 616                 t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap)
 617         return name, t
 618
 619
 620     def dump_graph(self, s, indent=" "):
 621         for name, child in s.children.items():
 622             print indent + name + ":" + str(child)
 623             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 624                 self.dump_graph(child, indent+"  ")
 625
 626     def copy_to_directory(self, source_infos, target):
 627         # step one: build a recursive graph of the source tree. This returns
 628         # a dictionary, with child names as keys, and values that are either
 629         # Directory or File instances (local or tahoe).
 630         source_dirs = self.build_graphs(source_infos)
 631         source_files = [source for source in source_infos
 632                         if isinstance(source[1], (LocalFileSource,
 633                                                   TahoeFileSource))]
 634
 635         #print "graphs"
 636         #for s in source_dirs:
 637         #    self.dump_graph(s)
 638
 639         # step two: create the top-level target directory object
 640         if isinstance(target, LocalMissingTarget):
 641             os.makedirs(target.pathname)
 642             target = LocalDirectoryTarget(self.progress, target.pathname)
 643         elif isinstance(target, TahoeMissingTarget):
 644             writecap = mkdir(target.url)
 645             target = TahoeDirectoryTarget(self.nodeurl, self.cache,
 646                                           self.progress)
 647             target.just_created(writecap)
 648         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 649         target.populate(False)
 650
 651         # step three: find a target for each source node, creating
 652         # directories as necessary. 'targetmap' is a dictionary that uses
 653         # target Directory instances as keys, and has values of
 654         # (name->sourceobject) dicts for all the files that need to wind up
 655         # there.
 656
 657         # sources are all LocalFile/LocalDirectory/TahoeFile/TahoeDirectory
 658         # target is LocalDirectory/TahoeDirectory
 659
 660         self.progress("attaching sources to targets, "
 661                       "%d files / %d dirs in root" %
 662                       (len(source_files), len(source_dirs)))
 663
 664         self.targetmap = {}
 665         self.files_to_copy = 0
 666
 667         for (name,s) in source_files:
 668             self.attach_to_target(s, name, target)
 669             self.files_to_copy += 1
 670
 671         for source in source_dirs:
 672             self.assign_targets(source, target)
 673
 674         self.progress("targets assigned, %s dirs, %s files" %
 675                       (len(self.targetmap), self.files_to_copy))
 676
 677         self.progress("starting copy, %d files, %d directories" %
 678                       (self.files_to_copy, len(self.targetmap)))
 679         self.files_copied = 0
 680         self.targets_finished = 0
 681
 682         # step four: walk through the list of targets. For each one, copy all
 683         # the files. If the target is a TahoeDirectory, upload and create
 684         # read-caps, then do a set_children to the target directory.
 685
 686         for target in self.targetmap:
 687             self.copy_files_to_target(self.targetmap[target], target)
 688             self.targets_finished += 1
 689             self.progress("%d/%d directories" %
 690                           (self.targets_finished, len(self.targetmap)))
 691
 692         return self.announce_success("files copied")
 693
 694     def attach_to_target(self, source, name, target):
 695         if target not in self.targetmap:
 696             self.targetmap[target] = {}
 697         self.targetmap[target][name] = source
 698         self.files_to_copy += 1
 699
 700     def assign_targets(self, source, target):
 701         # copy everything in the source into the target
 702         assert isinstance(source, (LocalDirectorySource, TahoeDirectorySource))
 703
 704         for name, child in source.children.items():
 705             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 706                 # we will need a target directory for this one
 707                 subtarget = target.get_child_target(name)
 708                 self.assign_targets(child, subtarget)
 709             else:
 710                 assert isinstance(child, (LocalFileSource, TahoeFileSource))
 711                 self.attach_to_target(child, name, target)
 712
 713
 714
 715     def copy_files_to_target(self, targetmap, target):
 716         for name, source in targetmap.items():
 717             assert isinstance(source, (LocalFileSource, TahoeFileSource))
 718             self.copy_file_into(source, name, target)
 719             self.files_copied += 1
 720             self.progress("%d/%d files, %d/%d directories" %
 721                           (self.files_copied, self.files_to_copy,
 722                            self.targets_finished, len(self.targetmap)))
 723         target.set_children()
 724
 725     def need_to_copy_bytes(self, source, target):
 726         if source.need_to_copy_bytes:
 727             # mutable tahoe files, and local files
 728             return True
 729         if isinstance(target, (LocalFileTarget, LocalDirectoryTarget)):
 730             return True
 731         return False
 732
 733     def announce_success(self, msg):
 734         if self.verbosity >= 1:
 735             print >>self.stdout, "Success: %s" % msg
 736         return 0
 737
 738     def copy_file(self, source, target):
 739         assert isinstance(source, (LocalFileSource, TahoeFileSource))
 740         assert isinstance(target, (LocalFileTarget, TahoeFileTarget,
 741                                    LocalMissingTarget, TahoeMissingTarget))
 742         if self.need_to_copy_bytes(source, target):
 743             # if the target is a local directory, this will just write the
 744             # bytes to disk. If it is a tahoe directory, it will upload the
 745             # data, and stash the new filecap for a later set_children call.
 746             f = source.open(self.caps_only)
 747             target.put_file(f)
 748             return self.announce_success("file copied")
 749         # otherwise we're copying tahoe to tahoe, and using immutable files,
 750         # so we can just make a link. TODO: this probably won't always work:
 751         # need to enumerate the cases and analyze them.
 752         target.put_uri(source.bestcap())
 753         return self.announce_success("file linked")
 754
 755     def copy_file_into(self, source, name, target):
 756         assert isinstance(source, (LocalFileSource, TahoeFileSource))
 757         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 758         if self.need_to_copy_bytes(source, target):
 759             # if the target is a local directory, this will just write the
 760             # bytes to disk. If it is a tahoe directory, it will upload the
 761             # data, and stash the new filecap for a later set_children call.
 762             f = source.open(self.caps_only)
 763             target.put_file(name, f)
 764             return
 765         # otherwise we're copying tahoe to tahoe, and using immutable files,
 766         # so we can just make a link
 767         target.put_uri(name, source.bestcap())
 768
 769
 770     def progress(self, message):
 771         #print message
 772         if self.progressfunc:
 773             self.progressfunc(message)
 774
 775     def build_graphs(self, source_infos):
 776         graphs = []
 777         for name,source in source_infos:
 778             if isinstance(source, (LocalDirectorySource, TahoeDirectorySource)):
 779                 source.populate(True)
 780                 graphs.append(source)
 781         return graphs
 782
 783
 784 def copy(options):
 785     return Copier().do_copy(options)
 786
 787 # error cases that need improvement:
 788 #  local-file-in-the-way
 789 #   touch proposed
 790 #   tahoe cp -r my:docs/proposed/denver.txt proposed/denver.txt
 791 #  handling of unknown nodes
 792
 793 # things that maybe should be errors but aren't
 794 #  local-dir-in-the-way
 795 #   mkdir denver.txt
 796 #   tahoe cp -r my:docs/proposed/denver.txt denver.txt
 797 #   (creates denver.txt/denver.txt)
 798
 799 # error cases that look good:
 800 #  tahoe cp -r my:docs/missing missing
 801 #  disconnect servers
 802 #   tahoe cp -r my:docs/missing missing  -> No JSON object could be decoded
 803 #  tahoe-file-in-the-way (when we want to make a directory)
 804 #   tahoe put README my:docs
 805 #   tahoe cp -r docs/proposed my:docs/proposed