1 import os.path, re, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import BaseOptions, get_aliases, get_default_nodedir, DEFAULT_ALIAS
4 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_output
6 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
8 _default_nodedir = get_default_nodedir()
10 class VDriveOptions(BaseOptions):
12 ["node-directory", "d", None,
13 "Specify which Tahoe node directory should be used. The directory "
14 "should either contain a full Tahoe node, or a file named node.url "
15 "that points to some other Tahoe node. It should also contain a file "
16 "named '" + os.path.join('private', 'aliases') + "' which contains the "
17 "mapping from alias name to root dirnode URI." + (
18 _default_nodedir and (" [default: " + quote_output(_default_nodedir) + "]") or "")],
19 ["node-url", "u", None,
20 "Specify the URL of the Tahoe gateway node, such as 'http://127.0.0.1:3456'. "
21 "This overrides the URL found in the --node-directory ."],
22 ["dir-cap", None, None,
23 "Specify which dirnode URI should be used as the 'tahoe' alias."]
26 def postOptions(self):
27 if self['node-directory']:
28 self['node-directory'] = argv_to_abspath(self['node-directory'])
30 self['node-directory'] = _default_nodedir
32 # compute a node-url from the existing options, put in self['node-url']
34 if (not isinstance(self['node-url'], basestring)
35 or not NODEURL_RE.match(self['node-url'])):
36 msg = ("--node-url is required to be a string and look like "
37 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
39 raise usage.UsageError(msg)
41 node_url_file = os.path.join(self['node-directory'], "node.url")
42 self['node-url'] = open(node_url_file, "r").read().strip()
43 if self['node-url'][-1] != "/":
44 self['node-url'] += "/"
46 aliases = get_aliases(self['node-directory'])
48 aliases[DEFAULT_ALIAS] = self['dir-cap']
49 self.aliases = aliases # maps alias name to dircap
52 class MakeDirectoryOptions(VDriveOptions):
53 def parseArgs(self, where=""):
54 self.where = argv_to_unicode(where)
56 def getSynopsis(self):
57 return "Usage: %s mkdir [options] [REMOTE_DIR]" % (self.command_name,)
59 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
61 class AddAliasOptions(VDriveOptions):
62 def parseArgs(self, alias, cap):
63 self.alias = argv_to_unicode(alias)
64 if self.alias.endswith(u':'):
65 self.alias = self.alias[:-1]
68 def getSynopsis(self):
69 return "Usage: %s add-alias [options] ALIAS[:] DIRCAP" % (self.command_name,)
71 longdesc = """Add a new alias for an existing directory."""
73 class CreateAliasOptions(VDriveOptions):
74 def parseArgs(self, alias):
75 self.alias = argv_to_unicode(alias)
76 if self.alias.endswith(u':'):
77 self.alias = self.alias[:-1]
79 def getSynopsis(self):
80 return "Usage: %s create-alias [options] ALIAS[:]" % (self.command_name,)
82 longdesc = """Create a new directory and add an alias for it."""
84 class ListAliasesOptions(VDriveOptions):
85 def getSynopsis(self):
86 return "Usage: %s list-aliases [options]" % (self.command_name,)
88 longdesc = """Display a table of all configured aliases."""
90 class ListOptions(VDriveOptions):
92 ("long", "l", "Use long format: show file sizes, and timestamps."),
93 ("uri", "u", "Show file/directory URIs."),
94 ("readonly-uri", None, "Show read-only file/directory URIs."),
95 ("classify", "F", "Append '/' to directory names, and '*' to mutable."),
96 ("json", None, "Show the raw JSON output."),
98 def parseArgs(self, where=""):
99 self.where = argv_to_unicode(where)
102 List the contents of some portion of the grid.
104 When the -l or --long option is used, each line is shown in the
107 drwx <size> <date/time> <name in this directory>
109 where each of the letters on the left may be replaced by '-'.
110 If 'd' is present, it indicates that the object is a directory.
111 If the 'd' is replaced by a '?', the object type is unknown.
112 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
113 then the object is writeable through its link in this directory
114 (note that the link might be replaceable even if the object is
115 not writeable through the current link).
116 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
117 only to indicate that the contents of a directory can be listed.
119 Directories have no size, so their size field is shown as '-'.
120 Otherwise the size of the file, when known, is given in bytes.
121 The size of mutable files or unknown objects is shown as '?'.
123 The date/time shows when this link in the Tahoe filesystem was
127 class GetOptions(VDriveOptions):
128 def parseArgs(self, arg1, arg2=None):
129 # tahoe get FOO |less # write to stdout
130 # tahoe get tahoe:FOO |less # same
131 # tahoe get FOO bar # write to local file
132 # tahoe get tahoe:FOO bar # same
134 self.from_file = argv_to_unicode(arg1)
137 self.to_file = argv_to_unicode(arg2)
141 if self.to_file == "-":
144 def getSynopsis(self):
145 return "Usage: %s get [options] REMOTE_FILE LOCAL_FILE" % (self.command_name,)
148 Retrieve a file from the grid and write it to the local filesystem. If
149 LOCAL_FILE is omitted or '-', the contents of the file will be written to
152 def getUsage(self, width=None):
153 t = VDriveOptions.getUsage(self, width)
156 % tahoe get FOO |less # write to stdout
157 % tahoe get tahoe:FOO |less # same
158 % tahoe get FOO bar # write to local file
159 % tahoe get tahoe:FOO bar # same
163 class PutOptions(VDriveOptions):
165 ("mutable", "m", "Create a mutable file instead of an immutable one."),
168 def parseArgs(self, arg1=None, arg2=None):
171 if arg1 is not None and arg2 is not None:
172 self.from_file = argv_to_unicode(arg1)
173 self.to_file = argv_to_unicode(arg2)
174 elif arg1 is not None and arg2 is None:
175 self.from_file = argv_to_unicode(arg1) # might be "-"
178 self.from_file = None
180 if self.from_file == u"-":
181 self.from_file = None
183 def getSynopsis(self):
184 return "Usage: %s put [options] LOCAL_FILE REMOTE_FILE" % (self.command_name,)
187 Put a file into the grid, copying its contents from the local filesystem.
188 If REMOTE_FILE is missing, upload the file but do not link it into a
189 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
190 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
191 with tahoe: unless otherwise specified."""
193 def getUsage(self, width=None):
194 t = VDriveOptions.getUsage(self, width)
197 % cat FILE | tahoe put # create unlinked file from stdin
198 % cat FILE | tahoe put - # same
199 % tahoe put bar # create unlinked file from local 'bar'
200 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
201 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
202 % tahoe put bar tahoe:FOO # same
203 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
207 class CpOptions(VDriveOptions):
209 ("recursive", "r", "Copy source directory recursively."),
210 ("verbose", "v", "Be noisy about what is happening."),
212 "When copying to local files, write out filecaps instead of actual "
213 "data (only useful for debugging and tree-comparison purposes)."),
216 def parseArgs(self, *args):
218 raise usage.UsageError("cp requires at least two arguments")
219 self.sources = map(argv_to_unicode, args[:-1])
220 self.destination = argv_to_unicode(args[-1])
222 def getSynopsis(self):
223 return "Usage: tahoe cp [options] FROM.. TO"
226 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
227 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
228 files or non-file arguments. Directories will be copied recursively.
229 New Tahoe-side directories will be created when necessary. Assuming that
230 you have previously set up an alias 'home' with 'tahoe create-alias home',
231 here are some examples:
233 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
235 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
237 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
239 You can also use a dircap as either FROM or TO target:
241 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
243 This command still has some limitations: symlinks and special files
244 (device nodes, named pipes) are not handled very well. Arguments should
245 probably not have trailing slashes. 'tahoe cp' does not behave as much
246 like /bin/cp as you would wish, especially with respect to trailing
250 class RmOptions(VDriveOptions):
251 def parseArgs(self, where):
252 self.where = argv_to_unicode(where)
254 def getSynopsis(self):
255 return "Usage: %s rm [options] REMOTE_FILE" % (self.command_name,)
257 class UnlinkOptions(RmOptions):
258 def getSynopsis(self):
259 return "Usage: %s unlink [options] REMOTE_FILE" % (self.command_name,)
261 class MvOptions(VDriveOptions):
262 def parseArgs(self, frompath, topath):
263 self.from_file = argv_to_unicode(frompath)
264 self.to_file = argv_to_unicode(topath)
266 def getSynopsis(self):
267 return "Usage: %s mv [options] FROM TO" % (self.command_name,)
270 Use 'tahoe mv' to move files that are already on the grid elsewhere on
271 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
273 If moving a remote file into a remote directory, you'll need to append a
274 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
275 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
277 Note that it is not possible to use this command to move local files to
278 the grid -- use 'tahoe cp' for that.
281 class LnOptions(VDriveOptions):
282 def parseArgs(self, frompath, topath):
283 self.from_file = argv_to_unicode(frompath)
284 self.to_file = argv_to_unicode(topath)
286 def getSynopsis(self):
287 return "Usage: %s ln [options] FROM_LINK TO_LINK" % (self.command_name,)
290 Use 'tahoe ln' to duplicate a link (directory entry) already on the grid
291 to elsewhere on the grid. For example 'tahoe ln alias:some_file
292 alias:new_file'. causes 'alias:new_file' to point to the same object that
293 'alias:some_file' points to.
295 (The argument order is the same as Unix ln. To remember the order, you
296 can think of this command as copying a link, rather than copying a file
297 as 'tahoe cp' does. Then the argument order is consistent with that of
300 When linking a remote file into a remote directory, you'll need to append
301 a '/' to the name of the remote directory, e.g. 'tahoe ln tahoe:file1
302 tahoe:dir/' (which is shorthand for 'tahoe ln tahoe:file1
303 tahoe:dir/file1'). If you forget the '/', e.g. 'tahoe ln tahoe:file1
304 tahoe:dir', the 'ln' command will refuse to overwrite the 'tahoe:dir'
305 directory, and will exit with an error.
307 Note that it is not possible to use this command to create links between
308 local and remote files.
311 class BackupConfigurationError(Exception):
314 class BackupOptions(VDriveOptions):
316 ("verbose", "v", "Be noisy about what is happening."),
317 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
320 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
321 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
322 '=meta-update', '=update', '.bzr', '.bzrignore',
323 '.bzrtags', '.hg', '.hgignore', '_darcs')
326 super(BackupOptions, self).__init__()
327 self['exclude'] = set()
329 def parseArgs(self, localdir, topath):
330 self.from_dir = argv_to_unicode(localdir)
331 self.to_dir = argv_to_unicode(topath)
333 def getSynopsis(self):
334 return "Usage: %s backup [options] FROM ALIAS:TO" % (self.command_name,)
336 def opt_exclude(self, pattern):
337 """Ignore files matching a glob pattern. You may give multiple
338 '--exclude' options."""
339 g = argv_to_unicode(pattern).strip()
341 exclude = self['exclude']
344 def opt_exclude_from(self, filepath):
345 """Ignore file matching glob patterns listed in file, one per
346 line. The file is assumed to be in the argv encoding."""
347 abs_filepath = argv_to_abspath(filepath)
349 exclude_file = file(abs_filepath)
351 raise BackupConfigurationError('Error opening exclude file %s.' % quote_output(abs_filepath))
353 for line in exclude_file:
354 self.opt_exclude(line)
358 def opt_exclude_vcs(self):
359 """Exclude files and directories used by following version control
360 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
362 for pattern in self.vcs_patterns:
363 self.opt_exclude(pattern)
365 def filter_listdir(self, listdir):
366 """Yields non-excluded childpaths in path."""
367 exclude = self['exclude']
368 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
369 for filename in listdir:
370 for regexp in exclude_regexps:
371 if regexp.match(filename):
377 Add a versioned backup of the local FROM directory to a timestamped
378 subdirectory of the TO/Archives directory on the grid, sharing as many
379 files and directories as possible with earlier backups. Create TO/Latest
380 as a reference to the latest backup. Behaves somewhat like 'rsync -a
381 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
382 TO/Archives/(new) TO/Latest'."""
384 class WebopenOptions(VDriveOptions):
386 ("info", "i", "Open the t=info page for the file"),
388 def parseArgs(self, where=''):
389 self.where = argv_to_unicode(where)
391 def getSynopsis(self):
392 return "Usage: %s webopen [options] [ALIAS:PATH]" % (self.command_name,)
394 longdesc = """Open a web browser to the contents of some file or
395 directory on the grid. When run without arguments, open the Welcome
398 class ManifestOptions(VDriveOptions):
400 ("storage-index", "s", "Only print storage index strings, not pathname+cap."),
401 ("verify-cap", None, "Only print verifycap, not pathname+cap."),
402 ("repair-cap", None, "Only print repaircap, not pathname+cap."),
403 ("raw", "r", "Display raw JSON data instead of parsed."),
405 def parseArgs(self, where=''):
406 self.where = argv_to_unicode(where)
408 def getSynopsis(self):
409 return "Usage: %s manifest [options] [ALIAS:PATH]" % (self.command_name,)
411 longdesc = """Print a list of all files and directories reachable from
412 the given starting point."""
414 class StatsOptions(VDriveOptions):
416 ("raw", "r", "Display raw JSON data instead of parsed"),
418 def parseArgs(self, where=''):
419 self.where = argv_to_unicode(where)
421 def getSynopsis(self):
422 return "Usage: %s stats [options] [ALIAS:PATH]" % (self.command_name,)
424 longdesc = """Print statistics about of all files and directories
425 reachable from the given starting point."""
427 class CheckOptions(VDriveOptions):
429 ("raw", None, "Display raw JSON data instead of parsed."),
430 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
431 ("repair", None, "Automatically repair any problems found."),
432 ("add-lease", None, "Add/renew lease on all shares."),
434 def parseArgs(self, where=''):
435 self.where = argv_to_unicode(where)
437 def getSynopsis(self):
438 return "Usage: %s check [options] [ALIAS:PATH]" % (self.command_name,)
441 Check a single file or directory: count how many shares are available and
442 verify their hashes. Optionally repair the file if any problems were
445 class DeepCheckOptions(VDriveOptions):
447 ("raw", None, "Display raw JSON data instead of parsed."),
448 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
449 ("repair", None, "Automatically repair any problems found."),
450 ("add-lease", None, "Add/renew lease on all shares."),
451 ("verbose", "v", "Be noisy about what is happening."),
453 def parseArgs(self, where=''):
454 self.where = argv_to_unicode(where)
456 def getSynopsis(self):
457 return "Usage: %s deep-check [options] [ALIAS:PATH]" % (self.command_name,)
460 Check all files and directories reachable from the given starting point
461 (which must be a directory), like 'tahoe check' but for multiple files.
462 Optionally repair any problems found."""
465 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
466 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
467 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
468 ["list-aliases", None, ListAliasesOptions, "List all alias caps."],
469 ["ls", None, ListOptions, "List a directory."],
470 ["get", None, GetOptions, "Retrieve a file from the grid."],
471 ["put", None, PutOptions, "Upload a file into the grid."],
472 ["cp", None, CpOptions, "Copy one or more files or directories."],
473 ["rm", None, RmOptions, "Unlink a file or directory on the grid."],
474 ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid (same as rm)."],
475 ["mv", None, MvOptions, "Move a file within the grid."],
476 ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
477 ["backup", None, BackupOptions, "Make target dir look like local dir."],
478 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
479 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
480 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
481 ["check", None, CheckOptions, "Check a single file or directory."],
482 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
486 from allmydata.scripts import tahoe_mkdir
487 rc = tahoe_mkdir.mkdir(options)
490 def add_alias(options):
491 from allmydata.scripts import tahoe_add_alias
492 rc = tahoe_add_alias.add_alias(options)
495 def create_alias(options):
496 from allmydata.scripts import tahoe_add_alias
497 rc = tahoe_add_alias.create_alias(options)
500 def list_aliases(options):
501 from allmydata.scripts import tahoe_add_alias
502 rc = tahoe_add_alias.list_aliases(options)
506 from allmydata.scripts import tahoe_ls
507 rc = tahoe_ls.list(options)
511 from allmydata.scripts import tahoe_get
512 rc = tahoe_get.get(options)
514 if options.to_file is None:
515 # be quiet, since the file being written to stdout should be
516 # proof enough that it worked, unless the user is unlucky
517 # enough to have picked an empty file
520 print >>options.stderr, "%s retrieved and written to %s" % \
521 (options.from_file, options.to_file)
525 from allmydata.scripts import tahoe_put
526 rc = tahoe_put.put(options)
530 from allmydata.scripts import tahoe_cp
531 rc = tahoe_cp.copy(options)
535 from allmydata.scripts import tahoe_rm
536 rc = tahoe_rm.rm(options)
540 from allmydata.scripts import tahoe_mv
541 rc = tahoe_mv.mv(options, mode="move")
545 from allmydata.scripts import tahoe_mv
546 rc = tahoe_mv.mv(options, mode="link")
550 from allmydata.scripts import tahoe_backup
551 rc = tahoe_backup.backup(options)
554 def webopen(options, opener=None):
555 from allmydata.scripts import tahoe_webopen
556 rc = tahoe_webopen.webopen(options, opener=opener)
559 def manifest(options):
560 from allmydata.scripts import tahoe_manifest
561 rc = tahoe_manifest.manifest(options)
565 from allmydata.scripts import tahoe_manifest
566 rc = tahoe_manifest.stats(options)
570 from allmydata.scripts import tahoe_check
571 rc = tahoe_check.check(options)
574 def deepcheck(options):
575 from allmydata.scripts import tahoe_check
576 rc = tahoe_check.deepcheck(options)
581 "add-alias": add_alias,
582 "create-alias": create_alias,
583 "list-aliases": list_aliases,
594 "manifest": manifest,
597 "deep-check": deepcheck,