1 import os.path, re, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import BaseOptions, get_aliases, get_default_nodedir, DEFAULT_ALIAS
4 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_output
6 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
8 _default_nodedir = get_default_nodedir()
10 class VDriveOptions(BaseOptions):
12 ["node-directory", "d", None,
13 "Specify which Tahoe node directory should be used. The directory "
14 "should either contain a full Tahoe node, or a file named node.url "
15 "that points to some other Tahoe node. It should also contain a file "
16 "named '" + os.path.join('private', 'aliases') + "' which contains the "
17 "mapping from alias name to root dirnode URI." + (
18 _default_nodedir and (" [default: " + quote_output(_default_nodedir) + "]") or "")],
19 ["node-url", "u", None,
20 "Specify the URL of the Tahoe gateway node, such as 'http://127.0.0.1:3456'. "
21 "This overrides the URL found in the --node-directory ."],
22 ["dir-cap", None, None,
23 "Specify which dirnode URI should be used as the 'tahoe' alias."]
26 def postOptions(self):
27 if self['node-directory']:
28 self['node-directory'] = argv_to_abspath(self['node-directory'])
30 self['node-directory'] = _default_nodedir
32 # compute a node-url from the existing options, put in self['node-url']
34 if (not isinstance(self['node-url'], basestring)
35 or not NODEURL_RE.match(self['node-url'])):
36 msg = ("--node-url is required to be a string and look like "
37 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
39 raise usage.UsageError(msg)
41 node_url_file = os.path.join(self['node-directory'], "node.url")
42 self['node-url'] = open(node_url_file, "r").read().strip()
43 if self['node-url'][-1] != "/":
44 self['node-url'] += "/"
46 aliases = get_aliases(self['node-directory'])
48 aliases[DEFAULT_ALIAS] = self['dir-cap']
49 self.aliases = aliases # maps alias name to dircap
52 class MakeDirectoryOptions(VDriveOptions):
54 ("format", None, None, "Create a directory with the given format: SDMF or MDMF (case-insensitive)"),
57 def parseArgs(self, where=""):
58 self.where = argv_to_unicode(where)
61 if self['format'].upper() not in ("SDMF", "MDMF"):
62 raise usage.UsageError("%s is an invalid format" % self['format'])
64 def getSynopsis(self):
65 return "Usage: %s mkdir [options] [REMOTE_DIR]" % (self.command_name,)
67 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
69 class AddAliasOptions(VDriveOptions):
70 def parseArgs(self, alias, cap):
71 self.alias = argv_to_unicode(alias)
72 if self.alias.endswith(u':'):
73 self.alias = self.alias[:-1]
76 def getSynopsis(self):
77 return "Usage: %s add-alias [options] ALIAS[:] DIRCAP" % (self.command_name,)
79 longdesc = """Add a new alias for an existing directory."""
81 class CreateAliasOptions(VDriveOptions):
82 def parseArgs(self, alias):
83 self.alias = argv_to_unicode(alias)
84 if self.alias.endswith(u':'):
85 self.alias = self.alias[:-1]
87 def getSynopsis(self):
88 return "Usage: %s create-alias [options] ALIAS[:]" % (self.command_name,)
90 longdesc = """Create a new directory and add an alias for it."""
92 class ListAliasesOptions(VDriveOptions):
93 def getSynopsis(self):
94 return "Usage: %s list-aliases [options]" % (self.command_name,)
96 longdesc = """Display a table of all configured aliases."""
98 class ListOptions(VDriveOptions):
100 ("long", "l", "Use long format: show file sizes, and timestamps."),
101 ("uri", "u", "Show file/directory URIs."),
102 ("readonly-uri", None, "Show read-only file/directory URIs."),
103 ("classify", "F", "Append '/' to directory names, and '*' to mutable."),
104 ("json", None, "Show the raw JSON output."),
106 def parseArgs(self, where=""):
107 self.where = argv_to_unicode(where)
109 def getSynopsis(self):
110 return "Usage: %s ls [options] [PATH]" % (self.command_name,)
113 List the contents of some portion of the grid.
115 If PATH is omitted, "tahoe:" is assumed.
117 When the -l or --long option is used, each line is shown in the
120 drwx <size> <date/time> <name in this directory>
122 where each of the letters on the left may be replaced by '-'.
123 If 'd' is present, it indicates that the object is a directory.
124 If the 'd' is replaced by a '?', the object type is unknown.
125 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
126 then the object is writeable through its link in this directory
127 (note that the link might be replaceable even if the object is
128 not writeable through the current link).
129 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
130 only to indicate that the contents of a directory can be listed.
132 Directories have no size, so their size field is shown as '-'.
133 Otherwise the size of the file, when known, is given in bytes.
134 The size of mutable files or unknown objects is shown as '?'.
136 The date/time shows when this link in the Tahoe filesystem was
140 class GetOptions(VDriveOptions):
141 def parseArgs(self, arg1, arg2=None):
142 # tahoe get FOO |less # write to stdout
143 # tahoe get tahoe:FOO |less # same
144 # tahoe get FOO bar # write to local file
145 # tahoe get tahoe:FOO bar # same
147 self.from_file = argv_to_unicode(arg1)
150 self.to_file = argv_to_unicode(arg2)
154 if self.to_file == "-":
157 def getSynopsis(self):
158 return "Usage: %s get [options] REMOTE_FILE LOCAL_FILE" % (self.command_name,)
161 Retrieve a file from the grid and write it to the local filesystem. If
162 LOCAL_FILE is omitted or '-', the contents of the file will be written to
165 def getUsage(self, width=None):
166 t = VDriveOptions.getUsage(self, width)
169 % tahoe get FOO |less # write to stdout
170 % tahoe get tahoe:FOO |less # same
171 % tahoe get FOO bar # write to local file
172 % tahoe get tahoe:FOO bar # same
176 class PutOptions(VDriveOptions):
178 ("mutable", "m", "Create a mutable file instead of an immutable one (like --format=SDMF)"),
181 ("format", None, None, "Create a file with the given format: SDMF and MDMF for mutable, CHK (default) for immutable. (case-insensitive)"),
184 def parseArgs(self, arg1=None, arg2=None):
187 if arg1 is not None and arg2 is not None:
188 self.from_file = argv_to_unicode(arg1)
189 self.to_file = argv_to_unicode(arg2)
190 elif arg1 is not None and arg2 is None:
191 self.from_file = argv_to_unicode(arg1) # might be "-"
194 self.from_file = None
196 if self.from_file == u"-":
197 self.from_file = None
200 if self['format'].upper() not in ("SDMF", "MDMF", "CHK"):
201 raise usage.UsageError("%s is an invalid format" % self['format'])
203 def getSynopsis(self):
204 return "Usage: %s put [options] LOCAL_FILE REMOTE_FILE" % (self.command_name,)
207 Put a file into the grid, copying its contents from the local filesystem.
208 If REMOTE_FILE is missing, upload the file but do not link it into a
209 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
210 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
211 with tahoe: unless otherwise specified.
213 If the destination file already exists and is mutable, it will be modified
214 in-place, whether or not --mutable is specified. (--mutable only affects
215 creation of new files.)"""
217 def getUsage(self, width=None):
218 t = VDriveOptions.getUsage(self, width)
221 % cat FILE | tahoe put # create unlinked file from stdin
222 % cat FILE | tahoe put - # same
223 % tahoe put bar # create unlinked file from local 'bar'
224 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
225 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
226 % tahoe put bar tahoe:FOO # same
227 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
231 class CpOptions(VDriveOptions):
233 ("recursive", "r", "Copy source directory recursively."),
234 ("verbose", "v", "Be noisy about what is happening."),
236 "When copying to local files, write out filecaps instead of actual "
237 "data (only useful for debugging and tree-comparison purposes)."),
240 def parseArgs(self, *args):
242 raise usage.UsageError("cp requires at least two arguments")
243 self.sources = map(argv_to_unicode, args[:-1])
244 self.destination = argv_to_unicode(args[-1])
246 def getSynopsis(self):
247 return "Usage: %s cp [options] FROM.. TO" % (self.command_name,)
250 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
251 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
252 files or non-file arguments. Directories will be copied recursively.
253 New Tahoe-side directories will be created when necessary. Assuming that
254 you have previously set up an alias 'home' with 'tahoe create-alias home',
255 here are some examples:
257 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
259 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
261 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
263 You can also use a dircap as either FROM or TO target:
265 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
267 This command still has some limitations: symlinks and special files
268 (device nodes, named pipes) are not handled very well. Arguments should
269 probably not have trailing slashes. 'tahoe cp' does not behave as much
270 like /bin/cp as you would wish, especially with respect to trailing
274 class UnlinkOptions(VDriveOptions):
275 def parseArgs(self, where):
276 self.where = argv_to_unicode(where)
278 def getSynopsis(self):
279 return "Usage: %s unlink [options] REMOTE_FILE" % (self.command_name,)
281 class RmOptions(UnlinkOptions):
282 def getSynopsis(self):
283 return "Usage: %s rm [options] REMOTE_FILE" % (self.command_name,)
285 class MvOptions(VDriveOptions):
286 def parseArgs(self, frompath, topath):
287 self.from_file = argv_to_unicode(frompath)
288 self.to_file = argv_to_unicode(topath)
290 def getSynopsis(self):
291 return "Usage: %s mv [options] FROM TO" % (self.command_name,)
294 Use 'tahoe mv' to move files that are already on the grid elsewhere on
295 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
297 If moving a remote file into a remote directory, you'll need to append a
298 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
299 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
301 Note that it is not possible to use this command to move local files to
302 the grid -- use 'tahoe cp' for that.
305 class LnOptions(VDriveOptions):
306 def parseArgs(self, frompath, topath):
307 self.from_file = argv_to_unicode(frompath)
308 self.to_file = argv_to_unicode(topath)
310 def getSynopsis(self):
311 return "Usage: %s ln [options] FROM_LINK TO_LINK" % (self.command_name,)
314 Use 'tahoe ln' to duplicate a link (directory entry) already on the grid
315 to elsewhere on the grid. For example 'tahoe ln alias:some_file
316 alias:new_file'. causes 'alias:new_file' to point to the same object that
317 'alias:some_file' points to.
319 (The argument order is the same as Unix ln. To remember the order, you
320 can think of this command as copying a link, rather than copying a file
321 as 'tahoe cp' does. Then the argument order is consistent with that of
324 When linking a remote file into a remote directory, you'll need to append
325 a '/' to the name of the remote directory, e.g. 'tahoe ln tahoe:file1
326 tahoe:dir/' (which is shorthand for 'tahoe ln tahoe:file1
327 tahoe:dir/file1'). If you forget the '/', e.g. 'tahoe ln tahoe:file1
328 tahoe:dir', the 'ln' command will refuse to overwrite the 'tahoe:dir'
329 directory, and will exit with an error.
331 Note that it is not possible to use this command to create links between
332 local and remote files.
335 class BackupConfigurationError(Exception):
338 class BackupOptions(VDriveOptions):
340 ("verbose", "v", "Be noisy about what is happening."),
341 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
344 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
345 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
346 '=meta-update', '=update', '.bzr', '.bzrignore',
347 '.bzrtags', '.hg', '.hgignore', '_darcs')
350 super(BackupOptions, self).__init__()
351 self['exclude'] = set()
353 def parseArgs(self, localdir, topath):
354 self.from_dir = argv_to_unicode(localdir)
355 self.to_dir = argv_to_unicode(topath)
357 def getSynopsis(self):
358 return "Usage: %s backup [options] FROM ALIAS:TO" % (self.command_name,)
360 def opt_exclude(self, pattern):
361 """Ignore files matching a glob pattern. You may give multiple
362 '--exclude' options."""
363 g = argv_to_unicode(pattern).strip()
365 exclude = self['exclude']
368 def opt_exclude_from(self, filepath):
369 """Ignore file matching glob patterns listed in file, one per
370 line. The file is assumed to be in the argv encoding."""
371 abs_filepath = argv_to_abspath(filepath)
373 exclude_file = file(abs_filepath)
375 raise BackupConfigurationError('Error opening exclude file %s.' % quote_output(abs_filepath))
377 for line in exclude_file:
378 self.opt_exclude(line)
382 def opt_exclude_vcs(self):
383 """Exclude files and directories used by following version control
384 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
386 for pattern in self.vcs_patterns:
387 self.opt_exclude(pattern)
389 def filter_listdir(self, listdir):
390 """Yields non-excluded childpaths in path."""
391 exclude = self['exclude']
392 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
393 for filename in listdir:
394 for regexp in exclude_regexps:
395 if regexp.match(filename):
401 Add a versioned backup of the local FROM directory to a timestamped
402 subdirectory of the TO/Archives directory on the grid, sharing as many
403 files and directories as possible with earlier backups. Create TO/Latest
404 as a reference to the latest backup. Behaves somewhat like 'rsync -a
405 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
406 TO/Archives/(new) TO/Latest'."""
408 class WebopenOptions(VDriveOptions):
410 ("info", "i", "Open the t=info page for the file"),
412 def parseArgs(self, where=''):
413 self.where = argv_to_unicode(where)
415 def getSynopsis(self):
416 return "Usage: %s webopen [options] [ALIAS:PATH]" % (self.command_name,)
418 longdesc = """Open a web browser to the contents of some file or
419 directory on the grid. When run without arguments, open the Welcome
422 class ManifestOptions(VDriveOptions):
424 ("storage-index", "s", "Only print storage index strings, not pathname+cap."),
425 ("verify-cap", None, "Only print verifycap, not pathname+cap."),
426 ("repair-cap", None, "Only print repaircap, not pathname+cap."),
427 ("raw", "r", "Display raw JSON data instead of parsed."),
429 def parseArgs(self, where=''):
430 self.where = argv_to_unicode(where)
432 def getSynopsis(self):
433 return "Usage: %s manifest [options] [ALIAS:PATH]" % (self.command_name,)
435 longdesc = """Print a list of all files and directories reachable from
436 the given starting point."""
438 class StatsOptions(VDriveOptions):
440 ("raw", "r", "Display raw JSON data instead of parsed"),
442 def parseArgs(self, where=''):
443 self.where = argv_to_unicode(where)
445 def getSynopsis(self):
446 return "Usage: %s stats [options] [ALIAS:PATH]" % (self.command_name,)
448 longdesc = """Print statistics about of all files and directories
449 reachable from the given starting point."""
451 class CheckOptions(VDriveOptions):
453 ("raw", None, "Display raw JSON data instead of parsed."),
454 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
455 ("repair", None, "Automatically repair any problems found."),
456 ("add-lease", None, "Add/renew lease on all shares."),
458 def parseArgs(self, where=''):
459 self.where = argv_to_unicode(where)
461 def getSynopsis(self):
462 return "Usage: %s check [options] [ALIAS:PATH]" % (self.command_name,)
465 Check a single file or directory: count how many shares are available and
466 verify their hashes. Optionally repair the file if any problems were
469 class DeepCheckOptions(VDriveOptions):
471 ("raw", None, "Display raw JSON data instead of parsed."),
472 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
473 ("repair", None, "Automatically repair any problems found."),
474 ("add-lease", None, "Add/renew lease on all shares."),
475 ("verbose", "v", "Be noisy about what is happening."),
477 def parseArgs(self, where=''):
478 self.where = argv_to_unicode(where)
480 def getSynopsis(self):
481 return "Usage: %s deep-check [options] [ALIAS:PATH]" % (self.command_name,)
484 Check all files and directories reachable from the given starting point
485 (which must be a directory), like 'tahoe check' but for multiple files.
486 Optionally repair any problems found."""
489 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
490 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
491 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
492 ["list-aliases", None, ListAliasesOptions, "List all alias caps."],
493 ["ls", None, ListOptions, "List a directory."],
494 ["get", None, GetOptions, "Retrieve a file from the grid."],
495 ["put", None, PutOptions, "Upload a file into the grid."],
496 ["cp", None, CpOptions, "Copy one or more files or directories."],
497 ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."],
498 ["rm", None, RmOptions, "Unlink a file or directory on the grid (same as unlink)."],
499 ["mv", None, MvOptions, "Move a file within the grid."],
500 ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
501 ["backup", None, BackupOptions, "Make target dir look like local dir."],
502 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
503 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
504 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
505 ["check", None, CheckOptions, "Check a single file or directory."],
506 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
510 from allmydata.scripts import tahoe_mkdir
511 rc = tahoe_mkdir.mkdir(options)
514 def add_alias(options):
515 from allmydata.scripts import tahoe_add_alias
516 rc = tahoe_add_alias.add_alias(options)
519 def create_alias(options):
520 from allmydata.scripts import tahoe_add_alias
521 rc = tahoe_add_alias.create_alias(options)
524 def list_aliases(options):
525 from allmydata.scripts import tahoe_add_alias
526 rc = tahoe_add_alias.list_aliases(options)
530 from allmydata.scripts import tahoe_ls
531 rc = tahoe_ls.list(options)
535 from allmydata.scripts import tahoe_get
536 rc = tahoe_get.get(options)
538 if options.to_file is None:
539 # be quiet, since the file being written to stdout should be
540 # proof enough that it worked, unless the user is unlucky
541 # enough to have picked an empty file
544 print >>options.stderr, "%s retrieved and written to %s" % \
545 (options.from_file, options.to_file)
549 from allmydata.scripts import tahoe_put
550 rc = tahoe_put.put(options)
554 from allmydata.scripts import tahoe_cp
555 rc = tahoe_cp.copy(options)
558 def unlink(options, command="unlink"):
559 from allmydata.scripts import tahoe_unlink
560 rc = tahoe_unlink.unlink(options, command=command)
564 return unlink(options, command="rm")
567 from allmydata.scripts import tahoe_mv
568 rc = tahoe_mv.mv(options, mode="move")
572 from allmydata.scripts import tahoe_mv
573 rc = tahoe_mv.mv(options, mode="link")
577 from allmydata.scripts import tahoe_backup
578 rc = tahoe_backup.backup(options)
581 def webopen(options, opener=None):
582 from allmydata.scripts import tahoe_webopen
583 rc = tahoe_webopen.webopen(options, opener=opener)
586 def manifest(options):
587 from allmydata.scripts import tahoe_manifest
588 rc = tahoe_manifest.manifest(options)
592 from allmydata.scripts import tahoe_manifest
593 rc = tahoe_manifest.stats(options)
597 from allmydata.scripts import tahoe_check
598 rc = tahoe_check.check(options)
601 def deepcheck(options):
602 from allmydata.scripts import tahoe_check
603 rc = tahoe_check.deepcheck(options)
608 "add-alias": add_alias,
609 "create-alias": create_alias,
610 "list-aliases": list_aliases,
621 "manifest": manifest,
624 "deep-check": deepcheck,