1 import os.path, re, sys, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import BaseOptions, get_aliases, get_default_nodedir, DEFAULT_ALIAS
4 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_output
6 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
8 _default_nodedir = get_default_nodedir()
10 class VDriveOptions(BaseOptions):
12 ["node-directory", "d", None,
13 "Specify which Tahoe node directory should be used. The directory "
14 "should either contain a full Tahoe node, or a file named node.url "
15 "that points to some other Tahoe node. It should also contain a file "
16 "named '" + os.path.join('private', 'aliases') + "' which contains the "
17 "mapping from alias name to root dirnode URI." + (
18 _default_nodedir and (" [default: " + quote_output(_default_nodedir) + "]") or "")],
19 ["node-url", "u", None,
20 "Specify the URL of the Tahoe gateway node, such as 'http://127.0.0.1:3456'. "
21 "This overrides the URL found in the --node-directory ."],
22 ["dir-cap", None, None,
23 "Specify which dirnode URI should be used as the 'tahoe' alias."]
26 def postOptions(self):
27 if self['node-directory']:
28 self['node-directory'] = argv_to_abspath(self['node-directory'])
30 self['node-directory'] = _default_nodedir
32 # compute a node-url from the existing options, put in self['node-url']
34 if (not isinstance(self['node-url'], basestring)
35 or not NODEURL_RE.match(self['node-url'])):
36 msg = ("--node-url is required to be a string and look like "
37 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
39 raise usage.UsageError(msg)
41 node_url_file = os.path.join(self['node-directory'], "node.url")
42 self['node-url'] = open(node_url_file, "r").read().strip()
43 if self['node-url'][-1] != "/":
44 self['node-url'] += "/"
46 aliases = get_aliases(self['node-directory'])
48 aliases[DEFAULT_ALIAS] = self['dir-cap']
49 self.aliases = aliases # maps alias name to dircap
52 class MakeDirectoryOptions(VDriveOptions):
53 def parseArgs(self, where=""):
54 self.where = argv_to_unicode(where)
55 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
57 class AddAliasOptions(VDriveOptions):
58 def parseArgs(self, alias, cap):
59 self.alias = argv_to_unicode(alias)
60 if self.alias.endswith(u':'):
61 self.alias = self.alias[:-1]
64 def getSynopsis(self):
65 return "%s add-alias ALIAS[:] DIRCAP" % (os.path.basename(sys.argv[0]),)
67 longdesc = """Add a new alias for an existing directory."""
69 class CreateAliasOptions(VDriveOptions):
70 def parseArgs(self, alias):
71 self.alias = argv_to_unicode(alias)
72 if self.alias.endswith(u':'):
73 self.alias = self.alias[:-1]
75 def getSynopsis(self):
76 return "%s create-alias ALIAS[:]" % (os.path.basename(sys.argv[0]),)
78 longdesc = """Create a new directory and add an alias for it."""
80 class ListAliasOptions(VDriveOptions):
81 longdesc = """Display a table of all configured aliases."""
83 class ListOptions(VDriveOptions):
85 ("long", "l", "Use long format: show file sizes, and timestamps."),
86 ("uri", "u", "Show file/directory URIs."),
87 ("readonly-uri", None, "Show read-only file/directory URIs."),
88 ("classify", "F", "Append '/' to directory names, and '*' to mutable."),
89 ("json", None, "Show the raw JSON output."),
91 def parseArgs(self, where=""):
92 self.where = argv_to_unicode(where)
95 List the contents of some portion of the grid.
97 When the -l or --long option is used, each line is shown in the
100 drwx <size> <date/time> <name in this directory>
102 where each of the letters on the left may be replaced by '-'.
103 If 'd' is present, it indicates that the object is a directory.
104 If the 'd' is replaced by a '?', the object type is unknown.
105 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
106 then the object is writeable through its link in this directory
107 (note that the link might be replaceable even if the object is
108 not writeable through the current link).
109 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
110 only to indicate that the contents of a directory can be listed.
112 Directories have no size, so their size field is shown as '-'.
113 Otherwise the size of the file, when known, is given in bytes.
114 The size of mutable files or unknown objects is shown as '?'.
116 The date/time shows when this link in the Tahoe filesystem was
120 class GetOptions(VDriveOptions):
121 def parseArgs(self, arg1, arg2=None):
122 # tahoe get FOO |less # write to stdout
123 # tahoe get tahoe:FOO |less # same
124 # tahoe get FOO bar # write to local file
125 # tahoe get tahoe:FOO bar # same
127 self.from_file = argv_to_unicode(arg1)
130 self.to_file = argv_to_unicode(arg2)
134 if self.to_file == "-":
137 def getSynopsis(self):
138 return "%s get REMOTE_FILE LOCAL_FILE" % (os.path.basename(sys.argv[0]),)
141 Retrieve a file from the grid and write it to the local filesystem. If
142 LOCAL_FILE is omitted or '-', the contents of the file will be written to
145 def getUsage(self, width=None):
146 t = VDriveOptions.getUsage(self, width)
149 % tahoe get FOO |less # write to stdout
150 % tahoe get tahoe:FOO |less # same
151 % tahoe get FOO bar # write to local file
152 % tahoe get tahoe:FOO bar # same
156 class PutOptions(VDriveOptions):
158 ("mutable", "m", "Create a mutable file instead of an immutable one."),
161 def parseArgs(self, arg1=None, arg2=None):
164 if arg1 is not None and arg2 is not None:
165 self.from_file = argv_to_unicode(arg1)
166 self.to_file = argv_to_unicode(arg2)
167 elif arg1 is not None and arg2 is None:
168 self.from_file = argv_to_unicode(arg1) # might be "-"
171 self.from_file = None
173 if self.from_file == u"-":
174 self.from_file = None
176 def getSynopsis(self):
177 return "%s put LOCAL_FILE REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
180 Put a file into the grid, copying its contents from the local filesystem.
181 If REMOTE_FILE is missing, upload the file but do not link it into a
182 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
183 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
184 with tahoe: unless otherwise specified."""
186 def getUsage(self, width=None):
187 t = VDriveOptions.getUsage(self, width)
190 % cat FILE | tahoe put # create unlinked file from stdin
191 % cat FILE | tahoe put - # same
192 % tahoe put bar # create unlinked file from local 'bar'
193 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
194 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
195 % tahoe put bar tahoe:FOO # same
196 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
200 class CpOptions(VDriveOptions):
202 ("recursive", "r", "Copy source directory recursively."),
203 ("verbose", "v", "Be noisy about what is happening."),
205 "When copying to local files, write out filecaps instead of actual "
206 "data (only useful for debugging and tree-comparison purposes)."),
208 def parseArgs(self, *args):
210 raise usage.UsageError("cp requires at least two arguments")
211 self.sources = map(argv_to_unicode, args[:-1])
212 self.destination = argv_to_unicode(args[-1])
213 def getSynopsis(self):
214 return "Usage: tahoe [options] cp FROM.. TO"
216 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
217 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
218 files or non-file arguments. Directories will be copied recursively.
219 New Tahoe-side directories will be created when necessary. Assuming that
220 you have previously set up an alias 'home' with 'tahoe create-alias home',
221 here are some examples:
223 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
225 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
227 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
229 You can also use a dircap as either FROM or TO target:
231 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
233 This command still has some limitations: symlinks and special files
234 (device nodes, named pipes) are not handled very well. Arguments should
235 probably not have trailing slashes. 'tahoe cp' does not behave as much
236 like /bin/cp as you would wish, especially with respect to trailing
240 class RmOptions(VDriveOptions):
241 def parseArgs(self, where):
242 self.where = argv_to_unicode(where)
244 def getSynopsis(self):
245 return "%s rm REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
247 class UnlinkOptions(RmOptions):
248 def getSynopsis(self):
249 return "%s unlink REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
251 class MvOptions(VDriveOptions):
252 def parseArgs(self, frompath, topath):
253 self.from_file = argv_to_unicode(frompath)
254 self.to_file = argv_to_unicode(topath)
256 def getSynopsis(self):
257 return "%s mv FROM TO" % (os.path.basename(sys.argv[0]),)
259 Use 'tahoe mv' to move files that are already on the grid elsewhere on
260 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
262 If moving a remote file into a remote directory, you'll need to append a
263 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
264 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
266 Note that it is not possible to use this command to move local files to
267 the grid -- use 'tahoe cp' for that.
270 class LnOptions(VDriveOptions):
271 def parseArgs(self, frompath, topath):
272 self.from_file = argv_to_unicode(frompath)
273 self.to_file = argv_to_unicode(topath)
275 def getSynopsis(self):
276 return "%s ln FROM_LINK TO_LINK" % (os.path.basename(sys.argv[0]),)
279 Use 'tahoe ln' to duplicate a link (directory entry) already on the grid
280 to elsewhere on the grid. For example 'tahoe ln alias:some_file
281 alias:new_file'. causes 'alias:new_file' to point to the same object that
282 'alias:some_file' points to.
284 (The argument order is the same as Unix ln. To remember the order, you
285 can think of this command as copying a link, rather than copying a file
286 as 'tahoe cp' does. Then the argument order is consistent with that of
289 When linking a remote file into a remote directory, you'll need to append
290 a '/' to the name of the remote directory, e.g. 'tahoe ln tahoe:file1
291 tahoe:dir/' (which is shorthand for 'tahoe ln tahoe:file1
292 tahoe:dir/file1'). If you forget the '/', e.g. 'tahoe ln tahoe:file1
293 tahoe:dir', the 'ln' command will refuse to overwrite the 'tahoe:dir'
294 directory, and will exit with an error.
296 Note that it is not possible to use this command to create links between
297 local and remote files.
300 class BackupConfigurationError(Exception):
303 class BackupOptions(VDriveOptions):
305 ("verbose", "v", "Be noisy about what is happening."),
306 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
309 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
310 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
311 '=meta-update', '=update', '.bzr', '.bzrignore',
312 '.bzrtags', '.hg', '.hgignore', '_darcs')
315 super(BackupOptions, self).__init__()
316 self['exclude'] = set()
318 def parseArgs(self, localdir, topath):
319 self.from_dir = argv_to_unicode(localdir)
320 self.to_dir = argv_to_unicode(topath)
322 def getSynopsis(Self):
323 return "%s backup FROM ALIAS:TO" % os.path.basename(sys.argv[0])
325 def opt_exclude(self, pattern):
326 """Ignore files matching a glob pattern. You may give multiple
327 '--exclude' options."""
328 g = argv_to_unicode(pattern).strip()
330 exclude = self['exclude']
333 def opt_exclude_from(self, filepath):
334 """Ignore file matching glob patterns listed in file, one per
335 line. The file is assumed to be in the argv encoding."""
336 abs_filepath = argv_to_abspath(filepath)
338 exclude_file = file(abs_filepath)
340 raise BackupConfigurationError('Error opening exclude file %s.' % quote_output(abs_filepath))
342 for line in exclude_file:
343 self.opt_exclude(line)
347 def opt_exclude_vcs(self):
348 """Exclude files and directories used by following version control
349 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
351 for pattern in self.vcs_patterns:
352 self.opt_exclude(pattern)
354 def filter_listdir(self, listdir):
355 """Yields non-excluded childpaths in path."""
356 exclude = self['exclude']
357 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
358 for filename in listdir:
359 for regexp in exclude_regexps:
360 if regexp.match(filename):
366 Add a versioned backup of the local FROM directory to a timestamped
367 subdirectory of the TO/Archives directory on the grid, sharing as many
368 files and directories as possible with earlier backups. Create TO/Latest
369 as a reference to the latest backup. Behaves somewhat like 'rsync -a
370 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
371 TO/Archives/(new) TO/Latest'."""
373 class WebopenOptions(VDriveOptions):
375 ("info", "i", "Open the t=info page for the file"),
377 def parseArgs(self, where=''):
378 self.where = argv_to_unicode(where)
380 def getSynopsis(self):
381 return "%s webopen [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
383 longdesc = """Open a web browser to the contents of some file or
384 directory on the grid. When run without arguments, open the Welcome
387 class ManifestOptions(VDriveOptions):
389 ("storage-index", "s", "Only print storage index strings, not pathname+cap."),
390 ("verify-cap", None, "Only print verifycap, not pathname+cap."),
391 ("repair-cap", None, "Only print repaircap, not pathname+cap."),
392 ("raw", "r", "Display raw JSON data instead of parsed."),
394 def parseArgs(self, where=''):
395 self.where = argv_to_unicode(where)
397 def getSynopsis(self):
398 return "%s manifest [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
400 longdesc = """Print a list of all files and directories reachable from
401 the given starting point."""
403 class StatsOptions(VDriveOptions):
405 ("raw", "r", "Display raw JSON data instead of parsed"),
407 def parseArgs(self, where=''):
408 self.where = argv_to_unicode(where)
410 def getSynopsis(self):
411 return "%s stats [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
413 longdesc = """Print statistics about of all files and directories
414 reachable from the given starting point."""
416 class CheckOptions(VDriveOptions):
418 ("raw", None, "Display raw JSON data instead of parsed."),
419 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
420 ("repair", None, "Automatically repair any problems found."),
421 ("add-lease", None, "Add/renew lease on all shares."),
423 def parseArgs(self, where=''):
424 self.where = argv_to_unicode(where)
426 def getSynopsis(self):
427 return "%s check [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
430 Check a single file or directory: count how many shares are available and
431 verify their hashes. Optionally repair the file if any problems were
434 class DeepCheckOptions(VDriveOptions):
436 ("raw", None, "Display raw JSON data instead of parsed."),
437 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
438 ("repair", None, "Automatically repair any problems found."),
439 ("add-lease", None, "Add/renew lease on all shares."),
440 ("verbose", "v", "Be noisy about what is happening."),
442 def parseArgs(self, where=''):
443 self.where = argv_to_unicode(where)
445 def getSynopsis(self):
446 return "%s deep-check [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
449 Check all files and directories reachable from the given starting point
450 (which must be a directory), like 'tahoe check' but for multiple files.
451 Optionally repair any problems found."""
454 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
455 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
456 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
457 ["list-aliases", None, ListAliasOptions, "List all alias caps."],
458 ["ls", None, ListOptions, "List a directory."],
459 ["get", None, GetOptions, "Retrieve a file from the grid."],
460 ["put", None, PutOptions, "Upload a file into the grid."],
461 ["cp", None, CpOptions, "Copy one or more files or directories."],
462 ["rm", None, RmOptions, "Unlink a file or directory on the grid."],
463 ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid (same as rm)."],
464 ["mv", None, MvOptions, "Move a file within the grid."],
465 ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
466 ["backup", None, BackupOptions, "Make target dir look like local dir."],
467 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
468 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
469 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
470 ["check", None, CheckOptions, "Check a single file or directory."],
471 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
475 from allmydata.scripts import tahoe_mkdir
476 rc = tahoe_mkdir.mkdir(options)
479 def add_alias(options):
480 from allmydata.scripts import tahoe_add_alias
481 rc = tahoe_add_alias.add_alias(options)
484 def create_alias(options):
485 from allmydata.scripts import tahoe_add_alias
486 rc = tahoe_add_alias.create_alias(options)
489 def list_aliases(options):
490 from allmydata.scripts import tahoe_add_alias
491 rc = tahoe_add_alias.list_aliases(options)
495 from allmydata.scripts import tahoe_ls
496 rc = tahoe_ls.list(options)
500 from allmydata.scripts import tahoe_get
501 rc = tahoe_get.get(options)
503 if options.to_file is None:
504 # be quiet, since the file being written to stdout should be
505 # proof enough that it worked, unless the user is unlucky
506 # enough to have picked an empty file
509 print >>options.stderr, "%s retrieved and written to %s" % \
510 (options.from_file, options.to_file)
514 from allmydata.scripts import tahoe_put
515 rc = tahoe_put.put(options)
519 from allmydata.scripts import tahoe_cp
520 rc = tahoe_cp.copy(options)
524 from allmydata.scripts import tahoe_rm
525 rc = tahoe_rm.rm(options)
529 from allmydata.scripts import tahoe_mv
530 rc = tahoe_mv.mv(options, mode="move")
534 from allmydata.scripts import tahoe_mv
535 rc = tahoe_mv.mv(options, mode="link")
539 from allmydata.scripts import tahoe_backup
540 rc = tahoe_backup.backup(options)
543 def webopen(options, opener=None):
544 from allmydata.scripts import tahoe_webopen
545 rc = tahoe_webopen.webopen(options, opener=opener)
548 def manifest(options):
549 from allmydata.scripts import tahoe_manifest
550 rc = tahoe_manifest.manifest(options)
554 from allmydata.scripts import tahoe_manifest
555 rc = tahoe_manifest.stats(options)
559 from allmydata.scripts import tahoe_check
560 rc = tahoe_check.check(options)
563 def deepcheck(options):
564 from allmydata.scripts import tahoe_check
565 rc = tahoe_check.deepcheck(options)
570 "add-alias": add_alias,
571 "create-alias": create_alias,
572 "list-aliases": list_aliases,
583 "manifest": manifest,
586 "deep-check": deepcheck,