1 import os.path, re, sys, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import BaseOptions, get_aliases, get_default_nodedir, DEFAULT_ALIAS
4 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_output
6 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
8 _default_nodedir = get_default_nodedir()
10 class VDriveOptions(BaseOptions):
12 ["node-directory", "d", None,
13 "Specify which Tahoe node directory should be used. The directory "
14 "should either contain a full Tahoe node, or a file named node.url "
15 "that points to some other Tahoe node. It should also contain a file "
16 "named '" + os.path.join('private', 'aliases') + "' which contains the "
17 "mapping from alias name to root dirnode URI." + (
18 _default_nodedir and (" [default: " + quote_output(_default_nodedir) + "]") or "")],
19 ["node-url", "u", None,
20 "Specify the URL of the Tahoe gateway node, such as 'http://127.0.0.1:3456'. "
21 "This overrides the URL found in the --node-directory ."],
22 ["dir-cap", None, None,
23 "Specify which dirnode URI should be used as the 'tahoe' alias."]
26 def postOptions(self):
27 if self['node-directory']:
28 self['node-directory'] = argv_to_abspath(self['node-directory'])
30 self['node-directory'] = _default_nodedir
32 # compute a node-url from the existing options, put in self['node-url']
34 if (not isinstance(self['node-url'], basestring)
35 or not NODEURL_RE.match(self['node-url'])):
36 msg = ("--node-url is required to be a string and look like "
37 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
39 raise usage.UsageError(msg)
41 node_url_file = os.path.join(self['node-directory'], "node.url")
42 self['node-url'] = open(node_url_file, "r").read().strip()
43 if self['node-url'][-1] != "/":
44 self['node-url'] += "/"
46 aliases = get_aliases(self['node-directory'])
48 aliases[DEFAULT_ALIAS] = self['dir-cap']
49 self.aliases = aliases # maps alias name to dircap
52 class MakeDirectoryOptions(VDriveOptions):
53 def parseArgs(self, where=""):
54 self.where = argv_to_unicode(where)
55 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
57 class AddAliasOptions(VDriveOptions):
58 def parseArgs(self, alias, cap):
59 self.alias = argv_to_unicode(alias)
62 def getSynopsis(self):
63 return "%s add-alias ALIAS DIRCAP" % (os.path.basename(sys.argv[0]),)
65 longdesc = """Add a new alias for an existing directory."""
67 class CreateAliasOptions(VDriveOptions):
68 def parseArgs(self, alias):
69 self.alias = argv_to_unicode(alias)
71 def getSynopsis(self):
72 return "%s create-alias ALIAS" % (os.path.basename(sys.argv[0]),)
74 longdesc = """Create a new directory and add an alias for it."""
76 class ListAliasOptions(VDriveOptions):
77 longdesc = """Display a table of all configured aliases."""
79 class ListOptions(VDriveOptions):
81 ("long", "l", "Use long format: show file sizes, and timestamps."),
82 ("uri", "u", "Show file/directory URIs."),
83 ("readonly-uri", None, "Show read-only file/directory URIs."),
84 ("classify", "F", "Append '/' to directory names, and '*' to mutable."),
85 ("json", None, "Show the raw JSON output."),
87 def parseArgs(self, where=""):
88 self.where = argv_to_unicode(where)
91 List the contents of some portion of the grid.
93 When the -l or --long option is used, each line is shown in the
96 drwx <size> <date/time> <name in this directory>
98 where each of the letters on the left may be replaced by '-'.
99 If 'd' is present, it indicates that the object is a directory.
100 If the 'd' is replaced by a '?', the object type is unknown.
101 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
102 then the object is writeable through its link in this directory
103 (note that the link might be replaceable even if the object is
104 not writeable through the current link).
105 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
106 only to indicate that the contents of a directory can be listed.
108 Directories have no size, so their size field is shown as '-'.
109 Otherwise the size of the file, when known, is given in bytes.
110 The size of mutable files or unknown objects is shown as '?'.
112 The date/time shows when this link in the Tahoe filesystem was
116 class GetOptions(VDriveOptions):
117 def parseArgs(self, arg1, arg2=None):
118 # tahoe get FOO |less # write to stdout
119 # tahoe get tahoe:FOO |less # same
120 # tahoe get FOO bar # write to local file
121 # tahoe get tahoe:FOO bar # same
123 self.from_file = argv_to_unicode(arg1)
126 self.to_file = argv_to_unicode(arg2)
130 if self.to_file == "-":
133 def getSynopsis(self):
134 return "%s get REMOTE_FILE LOCAL_FILE" % (os.path.basename(sys.argv[0]),)
137 Retrieve a file from the grid and write it to the local filesystem. If
138 LOCAL_FILE is omitted or '-', the contents of the file will be written to
141 def getUsage(self, width=None):
142 t = VDriveOptions.getUsage(self, width)
145 % tahoe get FOO |less # write to stdout
146 % tahoe get tahoe:FOO |less # same
147 % tahoe get FOO bar # write to local file
148 % tahoe get tahoe:FOO bar # same
152 class PutOptions(VDriveOptions):
154 ("mutable", "m", "Create a mutable file instead of an immutable one."),
157 def parseArgs(self, arg1=None, arg2=None):
160 if arg1 is not None and arg2 is not None:
161 self.from_file = argv_to_unicode(arg1)
162 self.to_file = argv_to_unicode(arg2)
163 elif arg1 is not None and arg2 is None:
164 self.from_file = argv_to_unicode(arg1) # might be "-"
167 self.from_file = None
169 if self.from_file == u"-":
170 self.from_file = None
172 def getSynopsis(self):
173 return "%s put LOCAL_FILE REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
176 Put a file into the grid, copying its contents from the local filesystem.
177 If REMOTE_FILE is missing, upload the file but do not link it into a
178 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
179 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
180 with tahoe: unless otherwise specified."""
182 def getUsage(self, width=None):
183 t = VDriveOptions.getUsage(self, width)
186 % cat FILE | tahoe put # create unlinked file from stdin
187 % cat FILE | tahoe put - # same
188 % tahoe put bar # create unlinked file from local 'bar'
189 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
190 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
191 % tahoe put bar tahoe:FOO # same
192 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
196 class CpOptions(VDriveOptions):
198 ("recursive", "r", "Copy source directory recursively."),
199 ("verbose", "v", "Be noisy about what is happening."),
201 "When copying to local files, write out filecaps instead of actual "
202 "data (only useful for debugging and tree-comparison purposes)."),
204 def parseArgs(self, *args):
206 raise usage.UsageError("cp requires at least two arguments")
207 self.sources = map(argv_to_unicode, args[:-1])
208 self.destination = argv_to_unicode(args[-1])
209 def getSynopsis(self):
210 return "Usage: tahoe [options] cp FROM.. TO"
212 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
213 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
214 files or non-file arguments. Directories will be copied recursively.
215 New Tahoe-side directories will be created when necessary. Assuming that
216 you have previously set up an alias 'home' with 'tahoe create-alias home',
217 here are some examples:
219 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
221 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
223 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
225 You can also use a dircap as either FROM or TO target:
227 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
229 This command still has some limitations: symlinks and special files
230 (device nodes, named pipes) are not handled very well. Arguments should
231 probably not have trailing slashes. 'tahoe cp' does not behave as much
232 like /bin/cp as you would wish, especially with respect to trailing
236 class RmOptions(VDriveOptions):
237 def parseArgs(self, where):
238 self.where = argv_to_unicode(where)
240 def getSynopsis(self):
241 return "%s rm REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
243 class UnlinkOptions(RmOptions):
244 def getSynopsis(self):
245 return "%s unlink REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
247 class MvOptions(VDriveOptions):
248 def parseArgs(self, frompath, topath):
249 self.from_file = argv_to_unicode(frompath)
250 self.to_file = argv_to_unicode(topath)
252 def getSynopsis(self):
253 return "%s mv FROM TO" % (os.path.basename(sys.argv[0]),)
255 Use 'tahoe mv' to move files that are already on the grid elsewhere on
256 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
258 If moving a remote file into a remote directory, you'll need to append a
259 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
260 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
262 Note that it is not possible to use this command to move local files to
263 the grid -- use 'tahoe cp' for that.
266 class LnOptions(VDriveOptions):
267 def parseArgs(self, frompath, topath):
268 self.from_file = argv_to_unicode(frompath)
269 self.to_file = argv_to_unicode(topath)
271 def getSynopsis(self):
272 return "%s ln FROM_LINK TO_LINK" % (os.path.basename(sys.argv[0]),)
275 Use 'tahoe ln' to duplicate a link (directory entry) already on the grid
276 to elsewhere on the grid. For example 'tahoe ln alias:some_file
277 alias:new_file'. causes 'alias:new_file' to point to the same object that
278 'alias:some_file' points to.
280 (The argument order is the same as Unix ln. To remember the order, you
281 can think of this command as copying a link, rather than copying a file
282 as 'tahoe cp' does. Then the argument order is consistent with that of
285 When linking a remote file into a remote directory, you'll need to append
286 a '/' to the name of the remote directory, e.g. 'tahoe ln tahoe:file1
287 tahoe:dir/' (which is shorthand for 'tahoe ln tahoe:file1
288 tahoe:dir/file1'). If you forget the '/', e.g. 'tahoe ln tahoe:file1
289 tahoe:dir', the 'ln' command will refuse to overwrite the 'tahoe:dir'
290 directory, and will exit with an error.
292 Note that it is not possible to use this command to create links between
293 local and remote files.
296 class BackupConfigurationError(Exception):
299 class BackupOptions(VDriveOptions):
301 ("verbose", "v", "Be noisy about what is happening."),
302 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
305 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
306 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
307 '=meta-update', '=update', '.bzr', '.bzrignore',
308 '.bzrtags', '.hg', '.hgignore', '_darcs')
311 super(BackupOptions, self).__init__()
312 self['exclude'] = set()
314 def parseArgs(self, localdir, topath):
315 self.from_dir = argv_to_unicode(localdir)
316 self.to_dir = argv_to_unicode(topath)
318 def getSynopsis(Self):
319 return "%s backup FROM ALIAS:TO" % os.path.basename(sys.argv[0])
321 def opt_exclude(self, pattern):
322 """Ignore files matching a glob pattern. You may give multiple
323 '--exclude' options."""
324 g = argv_to_unicode(pattern).strip()
326 exclude = self['exclude']
329 def opt_exclude_from(self, filepath):
330 """Ignore file matching glob patterns listed in file, one per
331 line. The file is assumed to be in the argv encoding."""
332 abs_filepath = argv_to_abspath(filepath)
334 exclude_file = file(abs_filepath)
336 raise BackupConfigurationError('Error opening exclude file %s.' % quote_output(abs_filepath))
338 for line in exclude_file:
339 self.opt_exclude(line)
343 def opt_exclude_vcs(self):
344 """Exclude files and directories used by following version control
345 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
347 for pattern in self.vcs_patterns:
348 self.opt_exclude(pattern)
350 def filter_listdir(self, listdir):
351 """Yields non-excluded childpaths in path."""
352 exclude = self['exclude']
353 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
354 for filename in listdir:
355 for regexp in exclude_regexps:
356 if regexp.match(filename):
362 Add a versioned backup of the local FROM directory to a timestamped
363 subdirectory of the TO/Archives directory on the grid, sharing as many
364 files and directories as possible with earlier backups. Create TO/Latest
365 as a reference to the latest backup. Behaves somewhat like 'rsync -a
366 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
367 TO/Archives/(new) TO/Latest'."""
369 class WebopenOptions(VDriveOptions):
371 ("info", "i", "Open the t=info page for the file"),
373 def parseArgs(self, where=''):
374 self.where = argv_to_unicode(where)
376 def getSynopsis(self):
377 return "%s webopen [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
379 longdesc = """Open a web browser to the contents of some file or
380 directory on the grid. When run without arguments, open the Welcome
383 class ManifestOptions(VDriveOptions):
385 ("storage-index", "s", "Only print storage index strings, not pathname+cap."),
386 ("verify-cap", None, "Only print verifycap, not pathname+cap."),
387 ("repair-cap", None, "Only print repaircap, not pathname+cap."),
388 ("raw", "r", "Display raw JSON data instead of parsed."),
390 def parseArgs(self, where=''):
391 self.where = argv_to_unicode(where)
393 def getSynopsis(self):
394 return "%s manifest [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
396 longdesc = """Print a list of all files and directories reachable from
397 the given starting point."""
399 class StatsOptions(VDriveOptions):
401 ("raw", "r", "Display raw JSON data instead of parsed"),
403 def parseArgs(self, where=''):
404 self.where = argv_to_unicode(where)
406 def getSynopsis(self):
407 return "%s stats [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
409 longdesc = """Print statistics about of all files and directories
410 reachable from the given starting point."""
412 class CheckOptions(VDriveOptions):
414 ("raw", None, "Display raw JSON data instead of parsed."),
415 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
416 ("repair", None, "Automatically repair any problems found."),
417 ("add-lease", None, "Add/renew lease on all shares."),
419 def parseArgs(self, where=''):
420 self.where = argv_to_unicode(where)
422 def getSynopsis(self):
423 return "%s check [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
426 Check a single file or directory: count how many shares are available and
427 verify their hashes. Optionally repair the file if any problems were
430 class DeepCheckOptions(VDriveOptions):
432 ("raw", None, "Display raw JSON data instead of parsed."),
433 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
434 ("repair", None, "Automatically repair any problems found."),
435 ("add-lease", None, "Add/renew lease on all shares."),
436 ("verbose", "v", "Be noisy about what is happening."),
438 def parseArgs(self, where=''):
439 self.where = argv_to_unicode(where)
441 def getSynopsis(self):
442 return "%s deep-check [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
445 Check all files and directories reachable from the given starting point
446 (which must be a directory), like 'tahoe check' but for multiple files.
447 Optionally repair any problems found."""
450 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
451 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
452 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
453 ["list-aliases", None, ListAliasOptions, "List all alias caps."],
454 ["ls", None, ListOptions, "List a directory."],
455 ["get", None, GetOptions, "Retrieve a file from the grid."],
456 ["put", None, PutOptions, "Upload a file into the grid."],
457 ["cp", None, CpOptions, "Copy one or more files or directories."],
458 ["rm", None, RmOptions, "Unlink a file or directory on the grid."],
459 ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid (same as rm)."],
460 ["mv", None, MvOptions, "Move a file within the grid."],
461 ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
462 ["backup", None, BackupOptions, "Make target dir look like local dir."],
463 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
464 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
465 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
466 ["check", None, CheckOptions, "Check a single file or directory."],
467 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
471 from allmydata.scripts import tahoe_mkdir
472 rc = tahoe_mkdir.mkdir(options)
475 def add_alias(options):
476 from allmydata.scripts import tahoe_add_alias
477 rc = tahoe_add_alias.add_alias(options)
480 def create_alias(options):
481 from allmydata.scripts import tahoe_add_alias
482 rc = tahoe_add_alias.create_alias(options)
485 def list_aliases(options):
486 from allmydata.scripts import tahoe_add_alias
487 rc = tahoe_add_alias.list_aliases(options)
491 from allmydata.scripts import tahoe_ls
492 rc = tahoe_ls.list(options)
496 from allmydata.scripts import tahoe_get
497 rc = tahoe_get.get(options)
499 if options.to_file is None:
500 # be quiet, since the file being written to stdout should be
501 # proof enough that it worked, unless the user is unlucky
502 # enough to have picked an empty file
505 print >>options.stderr, "%s retrieved and written to %s" % \
506 (options.from_file, options.to_file)
510 from allmydata.scripts import tahoe_put
511 rc = tahoe_put.put(options)
515 from allmydata.scripts import tahoe_cp
516 rc = tahoe_cp.copy(options)
520 from allmydata.scripts import tahoe_rm
521 rc = tahoe_rm.rm(options)
525 from allmydata.scripts import tahoe_mv
526 rc = tahoe_mv.mv(options, mode="move")
530 from allmydata.scripts import tahoe_mv
531 rc = tahoe_mv.mv(options, mode="link")
535 from allmydata.scripts import tahoe_backup
536 rc = tahoe_backup.backup(options)
539 def webopen(options, opener=None):
540 from allmydata.scripts import tahoe_webopen
541 rc = tahoe_webopen.webopen(options, opener=opener)
544 def manifest(options):
545 from allmydata.scripts import tahoe_manifest
546 rc = tahoe_manifest.manifest(options)
550 from allmydata.scripts import tahoe_manifest
551 rc = tahoe_manifest.stats(options)
555 from allmydata.scripts import tahoe_check
556 rc = tahoe_check.check(options)
559 def deepcheck(options):
560 from allmydata.scripts import tahoe_check
561 rc = tahoe_check.deepcheck(options)
566 "add-alias": add_alias,
567 "create-alias": create_alias,
568 "list-aliases": list_aliases,
579 "manifest": manifest,
582 "deep-check": deepcheck,