src/allmydata/scripts/tahoe_cp.py

   1
   2 import os.path
   3 import urllib
   4 import simplejson
   5 import sys
   6 from cStringIO import StringIO
   7 from twisted.python.failure import Failure
   8 from allmydata.scripts.common import get_alias, escape_path, \
   9                                      DefaultAliasMarker, UnknownAliasError
  10 from allmydata.scripts.common_http import do_http
  11 from allmydata import uri
  12 from twisted.python import usage
  13 from allmydata.util.stringutils import unicode_to_url, listdir_unicode, open_unicode
  14 from allmydata.util.assertutil import precondition
  15
  16
  17 def ascii_or_none(s):
  18     if s is None:
  19         return s
  20     return str(s)
  21
  22 class TahoeError(Exception):
  23     def __init__(self, msg, resp):
  24         self.msg = msg
  25         self.status = resp.status
  26         self.reason = resp.reason
  27         self.body = resp.read()
  28
  29     def display(self, err):
  30         print >>err, "%s: %s %s" % (self.msg, self.status, self.reason)
  31         print >>err, self.body
  32
  33 class MissingSourceError(Exception):
  34     pass
  35
  36 def GET_to_file(url):
  37     resp = do_http("GET", url)
  38     if resp.status == 200:
  39         return resp
  40     raise TahoeError("Error during GET", resp)
  41
  42 def GET_to_string(url):
  43     f = GET_to_file(url)
  44     return f.read()
  45
  46 def PUT(url, data):
  47     resp = do_http("PUT", url, data)
  48     if resp.status in (200, 201):
  49         return resp.read()
  50     raise TahoeError("Error during PUT", resp)
  51
  52 def POST(url, data):
  53     resp = do_http("POST", url, data)
  54     if resp.status in (200, 201):
  55         return resp.read()
  56     raise TahoeError("Error during POST", resp)
  57
  58 def mkdir(targeturl):
  59     url = targeturl + "?t=mkdir"
  60     resp = do_http("POST", url)
  61     if resp.status in (200, 201):
  62         return resp.read().strip()
  63     raise TahoeError("Error during mkdir", resp)
  64
  65 def make_tahoe_subdirectory(nodeurl, parent_writecap, name):
  66     url = nodeurl + "/".join(["uri",
  67                               urllib.quote(parent_writecap),
  68                               urllib.quote(name),
  69                               ]) + "?t=mkdir"
  70     resp = do_http("POST", url)
  71     if resp.status in (200, 201):
  72         return resp.read().strip()
  73     raise TahoeError("Error during mkdir", resp)
  74
  75
  76 class LocalFileSource:
  77     def __init__(self, pathname):
  78         precondition(isinstance(pathname, unicode), pathname)
  79         self.pathname = pathname
  80
  81     def need_to_copy_bytes(self):
  82         return True
  83
  84     def open(self, caps_only):
  85         return open(self.pathname, "rb")
  86
  87 class LocalFileTarget:
  88     def __init__(self, pathname):
  89         precondition(isinstance(pathname, unicode), pathname)
  90         self.pathname = pathname
  91     def put_file(self, inf):
  92         outf = open(self.pathname, "wb")
  93         while True:
  94             data = inf.read(32768)
  95             if not data:
  96                 break
  97             outf.write(data)
  98         outf.close()
  99
 100 class LocalMissingTarget:
 101     def __init__(self, pathname):
 102         precondition(isinstance(pathname, unicode), pathname)
 103         self.pathname = pathname
 104
 105     def put_file(self, inf):
 106         outf = open(self.pathname, "wb")
 107         while True:
 108             data = inf.read(32768)
 109             if not data:
 110                 break
 111             outf.write(data)
 112         outf.close()
 113
 114 class LocalDirectorySource:
 115     def __init__(self, progressfunc, pathname):
 116         precondition(isinstance(pathname, unicode), pathname)
 117
 118         self.progressfunc = progressfunc
 119         self.pathname = pathname
 120         self.children = None
 121
 122     def populate(self, recurse):
 123         if self.children is not None:
 124             return
 125         self.children = {}
 126         children = listdir_unicode(self.pathname)
 127         for i,n in enumerate(children):
 128             self.progressfunc("examining %d of %d" % (i, len(children)))
 129             pn = os.path.join(self.pathname, n)
 130             if os.path.isdir(pn):
 131                 child = LocalDirectorySource(self.progressfunc, pn)
 132                 self.children[n] = child
 133                 if recurse:
 134                     child.populate(True)
 135             elif os.path.isfile(pn):
 136                 self.children[n] = LocalFileSource(pn)
 137             else:
 138                 # Could be dangling symlink; probably not copy-able.
 139                 pass
 140
 141 class LocalDirectoryTarget:
 142     def __init__(self, progressfunc, pathname):
 143         precondition(isinstance(pathname, unicode), pathname)
 144
 145         self.progressfunc = progressfunc
 146         self.pathname = pathname
 147         self.children = None
 148
 149     def populate(self, recurse):
 150         if self.children is not None:
 151             return
 152         self.children = {}
 153         children = listdir_unicode(self.pathname)
 154         for i,n in enumerate(children):
 155             self.progressfunc("examining %d of %d" % (i, len(children)))
 156             pn = os.path.join(self.pathname, n)
 157             if os.path.isdir(pn):
 158                 child = LocalDirectoryTarget(self.progressfunc, pn)
 159                 self.children[n] = child
 160                 if recurse:
 161                     child.populate(True)
 162             else:
 163                 assert os.path.isfile(pn)
 164                 self.children[n] = LocalFileTarget(pn)
 165
 166     def get_child_target(self, name):
 167         if self.children is None:
 168             self.populate(False)
 169         if name in self.children:
 170             return self.children[name]
 171         pathname = os.path.join(self.pathname, name)
 172         os.makedirs(pathname)
 173         return LocalDirectoryTarget(self.progressfunc, pathname)
 174
 175     def put_file(self, name, inf):
 176         precondition(isinstance(name, unicode), name)
 177         pathname = os.path.join(self.pathname, name)
 178         outf = open_unicode(pathname, "wb")
 179         while True:
 180             data = inf.read(32768)
 181             if not data:
 182                 break
 183             outf.write(data)
 184         outf.close()
 185
 186     def set_children(self):
 187         pass
 188
 189 class TahoeFileSource:
 190     def __init__(self, nodeurl, mutable, writecap, readcap):
 191         self.nodeurl = nodeurl
 192         self.mutable = mutable
 193         self.writecap = writecap
 194         self.readcap = readcap
 195
 196     def need_to_copy_bytes(self):
 197         if self.mutable:
 198             return True
 199         return False
 200
 201     def open(self, caps_only):
 202         if caps_only:
 203             return StringIO(self.readcap)
 204         url = self.nodeurl + "uri/" + urllib.quote(self.readcap)
 205         return GET_to_file(url)
 206
 207     def bestcap(self):
 208         return self.writecap or self.readcap
 209
 210 class TahoeFileTarget:
 211     def __init__(self, nodeurl, mutable, writecap, readcap, url):
 212         self.nodeurl = nodeurl
 213         self.mutable = mutable
 214         self.writecap = writecap
 215         self.readcap = readcap
 216         self.url = url
 217
 218     def put_file(self, inf):
 219         # We want to replace this object in-place.
 220         assert self.url
 221         # our do_http() call currently requires a string or a filehandle with
 222         # a real .seek
 223         if not hasattr(inf, "seek"):
 224             inf = inf.read()
 225         PUT(self.url, inf)
 226         # TODO: this always creates immutable files. We might want an option
 227         # to always create mutable files, or to copy mutable files into new
 228         # mutable files. ticket #835
 229
 230 class TahoeDirectorySource:
 231     def __init__(self, nodeurl, cache, progressfunc):
 232         self.nodeurl = nodeurl
 233         self.cache = cache
 234         self.progressfunc = progressfunc
 235
 236     def init_from_grid(self, writecap, readcap):
 237         self.writecap = writecap
 238         self.readcap = readcap
 239         bestcap = writecap or readcap
 240         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 241         resp = do_http("GET", url + "?t=json")
 242         if resp.status != 200:
 243             raise TahoeError("Error examining source directory", resp)
 244         parsed = simplejson.loads(resp.read())
 245         nodetype, d = parsed
 246         assert nodetype == "dirnode"
 247         self.mutable = d.get("mutable", False) # older nodes don't provide it
 248         self.children_d = dict( [(unicode(name),value)
 249                                  for (name,value)
 250                                  in d["children"].iteritems()] )
 251         self.children = None
 252
 253     def init_from_parsed(self, parsed):
 254         nodetype, d = parsed
 255         self.writecap = ascii_or_none(d.get("rw_uri"))
 256         self.readcap = ascii_or_none(d.get("ro_uri"))
 257         self.mutable = d.get("mutable", False) # older nodes don't provide it
 258         self.children_d = dict( [(unicode(name),value)
 259                                  for (name,value)
 260                                  in d["children"].iteritems()] )
 261         self.children = None
 262
 263     def populate(self, recurse):
 264         if self.children is not None:
 265             return
 266         self.children = {}
 267         for i,(name, data) in enumerate(self.children_d.items()):
 268             self.progressfunc("examining %d of %d" % (i, len(self.children_d)))
 269             if data[0] == "filenode":
 270                 mutable = data[1].get("mutable", False)
 271                 writecap = ascii_or_none(data[1].get("rw_uri"))
 272                 readcap = ascii_or_none(data[1].get("ro_uri"))
 273                 self.children[name] = TahoeFileSource(self.nodeurl, mutable,
 274                                                       writecap, readcap)
 275             elif data[0] == "dirnode":
 276                 writecap = ascii_or_none(data[1].get("rw_uri"))
 277                 readcap = ascii_or_none(data[1].get("ro_uri"))
 278                 if writecap and writecap in self.cache:
 279                     child = self.cache[writecap]
 280                 elif readcap and readcap in self.cache:
 281                     child = self.cache[readcap]
 282                 else:
 283                     child = TahoeDirectorySource(self.nodeurl, self.cache,
 284                                                  self.progressfunc)
 285                     child.init_from_grid(writecap, readcap)
 286                     if writecap:
 287                         self.cache[writecap] = child
 288                     if readcap:
 289                         self.cache[readcap] = child
 290                     if recurse:
 291                         child.populate(True)
 292                 self.children[name] = child
 293             else:
 294                 # TODO: there should be an option to skip unknown nodes.
 295                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 296                                  "You probably need to use a later version of "
 297                                  "Tahoe-LAFS to copy this directory.")
 298
 299 class TahoeMissingTarget:
 300     def __init__(self, url):
 301         self.url = url
 302
 303     def put_file(self, inf):
 304         # We want to replace this object in-place.
 305         if not hasattr(inf, "seek"):
 306             inf = inf.read()
 307         PUT(self.url, inf)
 308         # TODO: this always creates immutable files. We might want an option
 309         # to always create mutable files, or to copy mutable files into new
 310         # mutable files.
 311
 312     def put_uri(self, filecap):
 313         # I'm not sure this will always work
 314         return PUT(self.url + "?t=uri", filecap)
 315
 316 class TahoeDirectoryTarget:
 317     def __init__(self, nodeurl, cache, progressfunc):
 318         self.nodeurl = nodeurl
 319         self.cache = cache
 320         self.progressfunc = progressfunc
 321         self.new_children = {}
 322
 323     def init_from_parsed(self, parsed):
 324         nodetype, d = parsed
 325         self.writecap = ascii_or_none(d.get("rw_uri"))
 326         self.readcap = ascii_or_none(d.get("ro_uri"))
 327         self.mutable = d.get("mutable", False) # older nodes don't provide it
 328         self.children_d = dict( [(unicode(name),value)
 329                                  for (name,value)
 330                                  in d["children"].iteritems()] )
 331         self.children = None
 332
 333     def init_from_grid(self, writecap, readcap):
 334         self.writecap = writecap
 335         self.readcap = readcap
 336         bestcap = writecap or readcap
 337         url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
 338         resp = do_http("GET", url + "?t=json")
 339         if resp.status != 200:
 340             raise TahoeError("Error examining target directory", resp)
 341         parsed = simplejson.loads(resp.read())
 342         nodetype, d = parsed
 343         assert nodetype == "dirnode"
 344         self.mutable = d.get("mutable", False) # older nodes don't provide it
 345         self.children_d = dict( [(unicode(name),value)
 346                                  for (name,value)
 347                                  in d["children"].iteritems()] )
 348         self.children = None
 349
 350     def just_created(self, writecap):
 351         self.writecap = writecap
 352         self.readcap = uri.from_string(writecap).get_readonly().to_string()
 353         self.mutable = True
 354         self.children_d = {}
 355         self.children = {}
 356
 357     def populate(self, recurse):
 358         if self.children is not None:
 359             return
 360         self.children = {}
 361         for i,(name, data) in enumerate(self.children_d.items()):
 362             self.progressfunc("examining %d of %d" % (i, len(self.children_d)))
 363             if data[0] == "filenode":
 364                 mutable = data[1].get("mutable", False)
 365                 writecap = ascii_or_none(data[1].get("rw_uri"))
 366                 readcap = ascii_or_none(data[1].get("ro_uri"))
 367                 url = None
 368                 if self.writecap:
 369                     url = self.nodeurl + "/".join(["uri",
 370                                                    urllib.quote(self.writecap),
 371                                                    urllib.quote(unicode_to_url(name))])
 372                 self.children[name] = TahoeFileTarget(self.nodeurl, mutable,
 373                                                       writecap, readcap, url)
 374             elif data[0] == "dirnode":
 375                 writecap = ascii_or_none(data[1].get("rw_uri"))
 376                 readcap = ascii_or_none(data[1].get("ro_uri"))
 377                 if writecap and writecap in self.cache:
 378                     child = self.cache[writecap]
 379                 elif readcap and readcap in self.cache:
 380                     child = self.cache[readcap]
 381                 else:
 382                     child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 383                                                  self.progressfunc)
 384                     child.init_from_grid(writecap, readcap)
 385                     if writecap:
 386                         self.cache[writecap] = child
 387                     if readcap:
 388                         self.cache[readcap] = child
 389                     if recurse:
 390                         child.populate(True)
 391                 self.children[name] = child
 392             else:
 393                 # TODO: there should be an option to skip unknown nodes.
 394                 raise TahoeError("Cannot copy unknown nodes (ticket #839). "
 395                                  "You probably need to use a later version of "
 396                                  "Tahoe-LAFS to copy this directory.")
 397
 398     def get_child_target(self, name):
 399         # return a new target for a named subdirectory of this dir
 400         if self.children is None:
 401             self.populate(False)
 402         if name in self.children:
 403             return self.children[name]
 404         writecap = make_tahoe_subdirectory(self.nodeurl, self.writecap, name)
 405         child = TahoeDirectoryTarget(self.nodeurl, self.cache,
 406                                      self.progressfunc)
 407         child.just_created(writecap)
 408         self.children[name] = child
 409         return child
 410
 411     def put_file(self, name, inf):
 412         url = self.nodeurl + "uri"
 413         if not hasattr(inf, "seek"):
 414             inf = inf.read()
 415         filecap = PUT(url, inf)
 416         # TODO: this always creates immutable files. We might want an option
 417         # to always create mutable files, or to copy mutable files into new
 418         # mutable files.
 419         self.new_children[name] = filecap
 420
 421     def put_uri(self, name, filecap):
 422         self.new_children[name] = filecap
 423
 424     def set_children(self):
 425         if not self.new_children:
 426             return
 427         url = (self.nodeurl + "uri/" + urllib.quote(self.writecap)
 428                + "?t=set_children")
 429         set_data = {}
 430         for (name, filecap) in self.new_children.items():
 431             # it just so happens that ?t=set_children will accept both file
 432             # read-caps and write-caps as ['rw_uri'], and will handle either
 433             # correctly. So don't bother trying to figure out whether the one
 434             # we have is read-only or read-write.
 435             # TODO: think about how this affects forward-compatibility for
 436             # unknown caps
 437             set_data[name] = ["filenode", {"rw_uri": filecap}]
 438         body = simplejson.dumps(set_data)
 439         POST(url, body)
 440
 441 class Copier:
 442
 443     def do_copy(self, options, progressfunc=None):
 444         if options['quiet']:
 445             verbosity = 0
 446         elif options['verbose']:
 447             verbosity = 2
 448         else:
 449             verbosity = 1
 450
 451         nodeurl = options['node-url']
 452         if nodeurl[-1] != "/":
 453             nodeurl += "/"
 454         self.nodeurl = nodeurl
 455         self.progressfunc = progressfunc
 456         self.options = options
 457         self.aliases = options.aliases
 458         self.verbosity = verbosity
 459         self.stdout = options.stdout
 460         self.stderr = options.stderr
 461         if verbosity >= 2 and not self.progressfunc:
 462             def progress(message):
 463                 print >>self.stderr, message
 464             self.progressfunc = progress
 465         self.caps_only = options["caps-only"]
 466         self.cache = {}
 467         try:
 468             status = self.try_copy()
 469             return status
 470         except TahoeError, te:
 471             Failure().printTraceback(self.stderr)
 472             print >>self.stderr
 473             te.display(self.stderr)
 474             return 1
 475
 476     def try_copy(self):
 477         source_specs = self.options.sources
 478         destination_spec = self.options.destination
 479         recursive = self.options["recursive"]
 480
 481         try:
 482             target = self.get_target_info(destination_spec)
 483         except UnknownAliasError, e:
 484             self.to_stderr("error: %s" % e.args[0])
 485             return 1
 486
 487         try:
 488             sources = [] # list of (name, source object)
 489             for ss in source_specs:
 490                 name, source = self.get_source_info(ss)
 491                 sources.append( (name, source) )
 492         except MissingSourceError, e:
 493             self.to_stderr("No such file or directory %s" % e.args[0])
 494             return 1
 495         except UnknownAliasError, e:
 496             self.to_stderr("error: %s" % e.args[0])
 497             return 1
 498
 499         have_source_dirs = bool([s for (name,s) in sources
 500                                  if isinstance(s, (LocalDirectorySource,
 501                                                    TahoeDirectorySource))])
 502
 503         if have_source_dirs and not recursive:
 504             self.to_stderr("cannot copy directories without --recursive")
 505             return 1
 506
 507         if isinstance(target, (LocalFileTarget, TahoeFileTarget)):
 508             # cp STUFF foo.txt, where foo.txt already exists. This limits the
 509             # possibilities considerably.
 510             if len(sources) > 1:
 511                 self.to_stderr("target '%s' is not a directory" % destination_spec)
 512                 return 1
 513             if have_source_dirs:
 514                 self.to_stderr("cannot copy directory into a file")
 515                 return 1
 516             name, source = sources[0]
 517             return self.copy_file(source, target)
 518
 519         if isinstance(target, (LocalMissingTarget, TahoeMissingTarget)):
 520             if recursive:
 521                 return self.copy_to_directory(sources, target)
 522             if len(sources) > 1:
 523                 # if we have -r, we'll auto-create the target directory. Without
 524                 # it, we'll only create a file.
 525                 self.to_stderr("cannot copy multiple files into a file without -r")
 526                 return 1
 527             # cp file1 newfile
 528             name, source = sources[0]
 529             return self.copy_file(source, target)
 530
 531         if isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)):
 532             # We're copying to an existing directory -- make sure that we
 533             # have target names for everything
 534             for (name, source) in sources:
 535                 if name is None and isinstance(source, TahoeFileSource):
 536                     self.to_stderr(
 537                         "error: you must specify a destination filename")
 538                     return 1
 539             return self.copy_to_directory(sources, target)
 540
 541         self.to_stderr("unknown target")
 542         return 1
 543
 544     def to_stderr(self, text):
 545         print >>self.stderr, text
 546
 547     def get_target_info(self, destination_spec):
 548         rootcap, path = get_alias(self.aliases, destination_spec, None)
 549         if rootcap == DefaultAliasMarker:
 550             # no alias, so this is a local file
 551             pathname = os.path.abspath(os.path.expanduser(path))
 552             if not os.path.exists(pathname):
 553                 t = LocalMissingTarget(pathname)
 554             elif os.path.isdir(pathname):
 555                 t = LocalDirectoryTarget(self.progress, pathname)
 556             else:
 557                 assert os.path.isfile(pathname), pathname
 558                 t = LocalFileTarget(pathname) # non-empty
 559         else:
 560             # this is a tahoe object
 561             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 562             if path:
 563                 url += "/" + escape_path(path)
 564
 565             resp = do_http("GET", url + "?t=json")
 566             if resp.status == 404:
 567                 # doesn't exist yet
 568                 t = TahoeMissingTarget(url)
 569             elif resp.status == 200:
 570                 parsed = simplejson.loads(resp.read())
 571                 nodetype, d = parsed
 572                 if nodetype == "dirnode":
 573                     t = TahoeDirectoryTarget(self.nodeurl, self.cache,
 574                                              self.progress)
 575                     t.init_from_parsed(parsed)
 576                 else:
 577                     writecap = ascii_or_none(d.get("rw_uri"))
 578                     readcap = ascii_or_none(d.get("ro_uri"))
 579                     mutable = d.get("mutable", False)
 580                     t = TahoeFileTarget(self.nodeurl, mutable,
 581                                         writecap, readcap, url)
 582             else:
 583                 raise TahoeError("Error examining target '%s'"
 584                                  % destination_spec, resp)
 585         return t
 586
 587     def get_source_info(self, source_spec):
 588         rootcap, path = get_alias(self.aliases, source_spec, None)
 589         if rootcap == DefaultAliasMarker:
 590             # no alias, so this is a local file
 591             pathname = os.path.abspath(os.path.expanduser(path))
 592             name = os.path.basename(pathname)
 593             if not os.path.exists(pathname):
 594                 raise MissingSourceError(source_spec)
 595             if os.path.isdir(pathname):
 596                 t = LocalDirectorySource(self.progress, pathname)
 597             else:
 598                 assert os.path.isfile(pathname)
 599                 t = LocalFileSource(pathname) # non-empty
 600         else:
 601             # this is a tahoe object
 602             url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
 603             name = None
 604             if path:
 605                 url += "/" + escape_path(path)
 606                 last_slash = path.rfind("/")
 607                 name = path
 608                 if last_slash:
 609                     name = path[last_slash+1:]
 610
 611             resp = do_http("GET", url + "?t=json")
 612             if resp.status == 404:
 613                 raise MissingSourceError(source_spec)
 614             elif resp.status != 200:
 615                 raise TahoeError("Error examining source '%s'" % source_spec,
 616                                  resp)
 617             parsed = simplejson.loads(resp.read())
 618             nodetype, d = parsed
 619             if nodetype == "dirnode":
 620                 t = TahoeDirectorySource(self.nodeurl, self.cache,
 621                                          self.progress)
 622                 t.init_from_parsed(parsed)
 623             else:
 624                 writecap = ascii_or_none(d.get("rw_uri"))
 625                 readcap = ascii_or_none(d.get("ro_uri"))
 626                 mutable = d.get("mutable", False) # older nodes don't provide it
 627                 if source_spec.rfind('/') != -1:
 628                     name = source_spec[source_spec.rfind('/')+1:]
 629                 t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap)
 630         return name, t
 631
 632
 633     def dump_graph(self, s, indent=" "):
 634         for name, child in s.children.items():
 635             print indent + name + ":" + str(child)
 636             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 637                 self.dump_graph(child, indent+"  ")
 638
 639     def copy_to_directory(self, source_infos, target):
 640         # step one: build a recursive graph of the source tree. This returns
 641         # a dictionary, with child names as keys, and values that are either
 642         # Directory or File instances (local or tahoe).
 643         source_dirs = self.build_graphs(source_infos)
 644         source_files = [source for source in source_infos
 645                         if isinstance(source[1], (LocalFileSource,
 646                                                   TahoeFileSource))]
 647
 648         #print "graphs"
 649         #for s in source_dirs:
 650         #    self.dump_graph(s)
 651
 652         # step two: create the top-level target directory object
 653         if isinstance(target, LocalMissingTarget):
 654             os.makedirs(target.pathname)
 655             target = LocalDirectoryTarget(self.progress, target.pathname)
 656         elif isinstance(target, TahoeMissingTarget):
 657             writecap = mkdir(target.url)
 658             target = TahoeDirectoryTarget(self.nodeurl, self.cache,
 659                                           self.progress)
 660             target.just_created(writecap)
 661         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 662         target.populate(False)
 663
 664         # step three: find a target for each source node, creating
 665         # directories as necessary. 'targetmap' is a dictionary that uses
 666         # target Directory instances as keys, and has values of
 667         # (name->sourceobject) dicts for all the files that need to wind up
 668         # there.
 669
 670         # sources are all LocalFile/LocalDirectory/TahoeFile/TahoeDirectory
 671         # target is LocalDirectory/TahoeDirectory
 672
 673         self.progress("attaching sources to targets, "
 674                       "%d files / %d dirs in root" %
 675                       (len(source_files), len(source_dirs)))
 676
 677         self.targetmap = {}
 678         self.files_to_copy = 0
 679
 680         for (name,s) in source_files:
 681             self.attach_to_target(s, name, target)
 682             self.files_to_copy += 1
 683
 684         for source in source_dirs:
 685             self.assign_targets(source, target)
 686
 687         self.progress("targets assigned, %s dirs, %s files" %
 688                       (len(self.targetmap), self.files_to_copy))
 689
 690         self.progress("starting copy, %d files, %d directories" %
 691                       (self.files_to_copy, len(self.targetmap)))
 692         self.files_copied = 0
 693         self.targets_finished = 0
 694
 695         # step four: walk through the list of targets. For each one, copy all
 696         # the files. If the target is a TahoeDirectory, upload and create
 697         # read-caps, then do a set_children to the target directory.
 698
 699         for target in self.targetmap:
 700             self.copy_files_to_target(self.targetmap[target], target)
 701             self.targets_finished += 1
 702             self.progress("%d/%d directories" %
 703                           (self.targets_finished, len(self.targetmap)))
 704
 705         return self.announce_success("files copied")
 706
 707     def attach_to_target(self, source, name, target):
 708         if target not in self.targetmap:
 709             self.targetmap[target] = {}
 710         self.targetmap[target][name] = source
 711         self.files_to_copy += 1
 712
 713     def assign_targets(self, source, target):
 714         # copy everything in the source into the target
 715         assert isinstance(source, (LocalDirectorySource, TahoeDirectorySource))
 716
 717         for name, child in source.children.items():
 718             if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
 719                 # we will need a target directory for this one
 720                 subtarget = target.get_child_target(name)
 721                 self.assign_targets(child, subtarget)
 722             else:
 723                 assert isinstance(child, (LocalFileSource, TahoeFileSource))
 724                 self.attach_to_target(child, name, target)
 725
 726
 727
 728     def copy_files_to_target(self, targetmap, target):
 729         for name, source in targetmap.items():
 730             assert isinstance(source, (LocalFileSource, TahoeFileSource))
 731             self.copy_file_into(source, name, target)
 732             self.files_copied += 1
 733             self.progress("%d/%d files, %d/%d directories" %
 734                           (self.files_copied, self.files_to_copy,
 735                            self.targets_finished, len(self.targetmap)))
 736         target.set_children()
 737
 738     def need_to_copy_bytes(self, source, target):
 739         if source.need_to_copy_bytes:
 740             # mutable tahoe files, and local files
 741             return True
 742         if isinstance(target, (LocalFileTarget, LocalDirectoryTarget)):
 743             return True
 744         return False
 745
 746     def announce_success(self, msg):
 747         if self.verbosity >= 1:
 748             print >>self.stdout, "Success: %s" % msg
 749         return 0
 750
 751     def copy_file(self, source, target):
 752         assert isinstance(source, (LocalFileSource, TahoeFileSource))
 753         assert isinstance(target, (LocalFileTarget, TahoeFileTarget,
 754                                    LocalMissingTarget, TahoeMissingTarget))
 755         if self.need_to_copy_bytes(source, target):
 756             # if the target is a local directory, this will just write the
 757             # bytes to disk. If it is a tahoe directory, it will upload the
 758             # data, and stash the new filecap for a later set_children call.
 759             f = source.open(self.caps_only)
 760             target.put_file(f)
 761             return self.announce_success("file copied")
 762         # otherwise we're copying tahoe to tahoe, and using immutable files,
 763         # so we can just make a link. TODO: this probably won't always work:
 764         # need to enumerate the cases and analyze them.
 765         target.put_uri(source.bestcap())
 766         return self.announce_success("file linked")
 767
 768     def copy_file_into(self, source, name, target):
 769         assert isinstance(source, (LocalFileSource, TahoeFileSource))
 770         assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
 771         if self.need_to_copy_bytes(source, target):
 772             # if the target is a local directory, this will just write the
 773             # bytes to disk. If it is a tahoe directory, it will upload the
 774             # data, and stash the new filecap for a later set_children call.
 775             f = source.open(self.caps_only)
 776             target.put_file(name, f)
 777             return
 778         # otherwise we're copying tahoe to tahoe, and using immutable files,
 779         # so we can just make a link
 780         target.put_uri(name, source.bestcap())
 781
 782
 783     def progress(self, message):
 784         #print message
 785         if self.progressfunc:
 786             self.progressfunc(message)
 787
 788     def build_graphs(self, source_infos):
 789         graphs = []
 790         for name,source in source_infos:
 791             if isinstance(source, (LocalDirectorySource, TahoeDirectorySource)):
 792                 source.populate(True)
 793                 graphs.append(source)
 794         return graphs
 795
 796
 797 def copy(options):
 798     return Copier().do_copy(options)
 799
 800 # error cases that need improvement:
 801 #  local-file-in-the-way
 802 #   touch proposed
 803 #   tahoe cp -r my:docs/proposed/denver.txt proposed/denver.txt
 804 #  handling of unknown nodes
 805
 806 # things that maybe should be errors but aren't
 807 #  local-dir-in-the-way
 808 #   mkdir denver.txt
 809 #   tahoe cp -r my:docs/proposed/denver.txt denver.txt
 810 #   (creates denver.txt/denver.txt)
 811
 812 # error cases that look good:
 813 #  tahoe cp -r my:docs/missing missing
 814 #  disconnect servers
 815 #   tahoe cp -r my:docs/missing missing  -> No JSON object could be decoded
 816 #  tahoe-file-in-the-way (when we want to make a directory)
 817 #   tahoe put README my:docs
 818 #   tahoe cp -r docs/proposed my:docs/proposed