1 import os.path, re, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import get_aliases, get_default_nodedir, \
4 DEFAULT_ALIAS, BaseOptions
5 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_local_unicode_path
7 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
9 _default_nodedir = get_default_nodedir()
11 class FilesystemOptions(BaseOptions):
13 ["node-url", "u", None,
14 "Specify the URL of the Tahoe gateway node, such as "
15 "'http://127.0.0.1:3456'. "
16 "This overrides the URL found in the --node-directory ."],
17 ["dir-cap", None, None,
18 "Specify which dirnode URI should be used as the 'tahoe' alias."]
21 def postOptions(self):
22 self["quiet"] = self.parent["quiet"]
23 if self.parent['node-directory']:
24 self['node-directory'] = argv_to_abspath(self.parent['node-directory'])
26 self['node-directory'] = _default_nodedir
28 # compute a node-url from the existing options, put in self['node-url']
30 if (not isinstance(self['node-url'], basestring)
31 or not NODEURL_RE.match(self['node-url'])):
32 msg = ("--node-url is required to be a string and look like "
33 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
35 raise usage.UsageError(msg)
37 node_url_file = os.path.join(self['node-directory'], "node.url")
38 self['node-url'] = open(node_url_file, "r").read().strip()
39 if self['node-url'][-1] != "/":
40 self['node-url'] += "/"
42 aliases = get_aliases(self['node-directory'])
44 aliases[DEFAULT_ALIAS] = self['dir-cap']
45 self.aliases = aliases # maps alias name to dircap
48 class MakeDirectoryOptions(FilesystemOptions):
50 ("format", None, None, "Create a directory with the given format: SDMF or MDMF (case-insensitive)"),
53 def parseArgs(self, where=""):
54 self.where = argv_to_unicode(where)
57 if self['format'].upper() not in ("SDMF", "MDMF"):
58 raise usage.UsageError("%s is an invalid format" % self['format'])
60 synopsis = "[options] [REMOTE_DIR]"
61 description = """Create a new directory, either unlinked or as a subdirectory."""
63 class AddAliasOptions(FilesystemOptions):
64 def parseArgs(self, alias, cap):
65 self.alias = argv_to_unicode(alias)
66 if self.alias.endswith(u':'):
67 self.alias = self.alias[:-1]
70 synopsis = "[options] ALIAS[:] DIRCAP"
71 description = """Add a new alias for an existing directory."""
73 class CreateAliasOptions(FilesystemOptions):
74 def parseArgs(self, alias):
75 self.alias = argv_to_unicode(alias)
76 if self.alias.endswith(u':'):
77 self.alias = self.alias[:-1]
79 synopsis = "[options] ALIAS[:]"
80 description = """Create a new directory and add an alias for it."""
82 class ListAliasesOptions(FilesystemOptions):
83 synopsis = "[options]"
84 description = """Display a table of all configured aliases."""
86 class ListOptions(FilesystemOptions):
88 ("long", "l", "Use long format: show file sizes, and timestamps."),
89 ("uri", None, "Show file/directory URIs."),
90 ("readonly-uri", None, "Show read-only file/directory URIs."),
91 ("classify", "F", "Append '/' to directory names, and '*' to mutable."),
92 ("json", None, "Show the raw JSON output."),
94 def parseArgs(self, where=""):
95 self.where = argv_to_unicode(where)
97 synopsis = "[options] [PATH]"
100 List the contents of some portion of the grid.
102 If PATH is omitted, "tahoe:" is assumed.
104 When the -l or --long option is used, each line is shown in the
107 drwx <size> <date/time> <name in this directory>
109 where each of the letters on the left may be replaced by '-'.
110 If 'd' is present, it indicates that the object is a directory.
111 If the 'd' is replaced by a '?', the object type is unknown.
112 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
113 then the object is writeable through its link in this directory
114 (note that the link might be replaceable even if the object is
115 not writeable through the current link).
116 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
117 only to indicate that the contents of a directory can be listed.
119 Directories have no size, so their size field is shown as '-'.
120 Otherwise the size of the file, when known, is given in bytes.
121 The size of mutable files or unknown objects is shown as '?'.
123 The date/time shows when this link in the Tahoe filesystem was
127 class GetOptions(FilesystemOptions):
128 def parseArgs(self, arg1, arg2=None):
129 # tahoe get FOO |less # write to stdout
130 # tahoe get tahoe:FOO |less # same
131 # tahoe get FOO bar # write to local file
132 # tahoe get tahoe:FOO bar # same
137 self.from_file = argv_to_unicode(arg1)
138 self.to_file = None if arg2 is None else argv_to_abspath(arg2)
140 synopsis = "[options] REMOTE_FILE LOCAL_FILE"
143 Retrieve a file from the grid and write it to the local filesystem. If
144 LOCAL_FILE is omitted or '-', the contents of the file will be written to
147 description_unwrapped = """
149 % tahoe get FOO |less # write to stdout
150 % tahoe get tahoe:FOO |less # same
151 % tahoe get FOO bar # write to local file
152 % tahoe get tahoe:FOO bar # same
155 class PutOptions(FilesystemOptions):
157 ("mutable", "m", "Create a mutable file instead of an immutable one (like --format=SDMF)"),
160 ("format", None, None, "Create a file with the given format: SDMF and MDMF for mutable, CHK (default) for immutable. (case-insensitive)"),
163 def parseArgs(self, arg1=None, arg2=None):
169 self.from_file = None if arg1 is None else argv_to_abspath(arg1)
170 self.to_file = None if arg2 is None else argv_to_unicode(arg2)
173 if self['format'].upper() not in ("SDMF", "MDMF", "CHK"):
174 raise usage.UsageError("%s is an invalid format" % self['format'])
176 synopsis = "[options] LOCAL_FILE REMOTE_FILE"
179 Put a file into the grid, copying its contents from the local filesystem.
180 If REMOTE_FILE is missing, upload the file but do not link it into a
181 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
182 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
183 with tahoe: unless otherwise specified.
185 If the destination file already exists and is mutable, it will be
186 modified in-place, whether or not --mutable is specified. (--mutable only
187 affects creation of new files.)
190 description_unwrapped = """
192 % cat FILE | tahoe put # create unlinked file from stdin
193 % cat FILE | tahoe put - # same
194 % tahoe put bar # create unlinked file from local 'bar'
195 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
196 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
197 % tahoe put bar tahoe:FOO # same
198 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
201 class CpOptions(FilesystemOptions):
203 ("recursive", "r", "Copy source directory recursively."),
204 ("verbose", "v", "Be noisy about what is happening."),
206 "When copying to local files, write out filecaps instead of actual "
207 "data (only useful for debugging and tree-comparison purposes)."),
210 def parseArgs(self, *args):
212 raise usage.UsageError("cp requires at least two arguments")
213 self.sources = map(argv_to_unicode, args[:-1])
214 self.destination = argv_to_unicode(args[-1])
216 synopsis = "[options] FROM.. TO"
219 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
220 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
221 files or non-file arguments. Directories will be copied recursively.
222 New Tahoe-side directories will be created when necessary. Assuming that
223 you have previously set up an alias 'home' with 'tahoe create-alias home',
224 here are some examples:
226 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
228 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
230 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
232 You can also use a dircap as either FROM or TO target:
234 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
236 This command still has some limitations: symlinks and special files
237 (device nodes, named pipes) are not handled very well. Arguments should
238 not have trailing slashes (they are ignored for directory arguments, but
239 trigger errors for file arguments). When copying directories, it can be
240 unclear whether you mean to copy the contents of a source directory, or
241 the source directory itself (i.e. whether the output goes under the
242 target directory, or one directory lower). Tahoe's rule is that source
243 directories with names are referring to the directory as a whole, and
244 source directories without names (e.g. a raw dircap) are referring to the
248 class UnlinkOptions(FilesystemOptions):
249 def parseArgs(self, where):
250 self.where = argv_to_unicode(where)
252 synopsis = "[options] REMOTE_FILE"
253 description = "Remove a named file from its parent directory."
255 class RmOptions(UnlinkOptions):
256 synopsis = "[options] REMOTE_FILE"
257 description = "Remove a named file from its parent directory."
259 class MvOptions(FilesystemOptions):
260 def parseArgs(self, frompath, topath):
261 self.from_file = argv_to_unicode(frompath)
262 self.to_file = argv_to_unicode(topath)
264 synopsis = "[options] FROM TO"
267 Use 'tahoe mv' to move files that are already on the grid elsewhere on
268 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
270 If moving a remote file into a remote directory, you'll need to append a
271 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
272 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
274 Note that it is not possible to use this command to move local files to
275 the grid -- use 'tahoe cp' for that.
278 class LnOptions(FilesystemOptions):
279 def parseArgs(self, frompath, topath):
280 self.from_file = argv_to_unicode(frompath)
281 self.to_file = argv_to_unicode(topath)
283 synopsis = "[options] FROM_LINK TO_LINK"
286 Use 'tahoe ln' to duplicate a link (directory entry) already on the grid
287 to elsewhere on the grid. For example 'tahoe ln alias:some_file
288 alias:new_file'. causes 'alias:new_file' to point to the same object that
289 'alias:some_file' points to.
291 (The argument order is the same as Unix ln. To remember the order, you
292 can think of this command as copying a link, rather than copying a file
293 as 'tahoe cp' does. Then the argument order is consistent with that of
296 When linking a remote file into a remote directory, you'll need to append
297 a '/' to the name of the remote directory, e.g. 'tahoe ln tahoe:file1
298 tahoe:dir/' (which is shorthand for 'tahoe ln tahoe:file1
299 tahoe:dir/file1'). If you forget the '/', e.g. 'tahoe ln tahoe:file1
300 tahoe:dir', the 'ln' command will refuse to overwrite the 'tahoe:dir'
301 directory, and will exit with an error.
303 Note that it is not possible to use this command to create links between
304 local and remote files.
307 class BackupConfigurationError(Exception):
310 class BackupOptions(FilesystemOptions):
312 ("verbose", "v", "Be noisy about what is happening."),
313 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
316 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
317 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
318 '=meta-update', '=update', '.bzr', '.bzrignore',
319 '.bzrtags', '.hg', '.hgignore', '_darcs')
322 super(BackupOptions, self).__init__()
323 self['exclude'] = set()
325 def parseArgs(self, localdir, topath):
326 self.from_dir = argv_to_abspath(localdir)
327 self.to_dir = argv_to_unicode(topath)
329 synopsis = "[options] FROM ALIAS:TO"
331 def opt_exclude(self, pattern):
332 """Ignore files matching a glob pattern. You may give multiple
333 '--exclude' options."""
334 g = argv_to_unicode(pattern).strip()
336 exclude = self['exclude']
339 def opt_exclude_from(self, filepath):
340 """Ignore file matching glob patterns listed in file, one per
341 line. The file is assumed to be in the argv encoding."""
342 abs_filepath = argv_to_abspath(filepath)
344 exclude_file = file(abs_filepath)
346 raise BackupConfigurationError('Error opening exclude file %s.' % quote_local_unicode_path(abs_filepath))
348 for line in exclude_file:
349 self.opt_exclude(line)
353 def opt_exclude_vcs(self):
354 """Exclude files and directories used by following version control
355 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
357 for pattern in self.vcs_patterns:
358 self.opt_exclude(pattern)
360 def filter_listdir(self, listdir):
361 """Yields non-excluded childpaths in path."""
362 exclude = self['exclude']
363 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
364 for filename in listdir:
365 for regexp in exclude_regexps:
366 if regexp.match(filename):
372 Add a versioned backup of the local FROM directory to a timestamped
373 subdirectory of the TO/Archives directory on the grid, sharing as many
374 files and directories as possible with earlier backups. Create TO/Latest
375 as a reference to the latest backup. Behaves somewhat like 'rsync -a
376 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
377 TO/Archives/(new) TO/Latest'."""
379 class WebopenOptions(FilesystemOptions):
381 ("info", "i", "Open the t=info page for the file"),
383 def parseArgs(self, where=''):
384 self.where = argv_to_unicode(where)
386 synopsis = "[options] [ALIAS:PATH]"
389 Open a web browser to the contents of some file or
390 directory on the grid. When run without arguments, open the Welcome
393 class ManifestOptions(FilesystemOptions):
395 ("storage-index", "s", "Only print storage index strings, not pathname+cap."),
396 ("verify-cap", None, "Only print verifycap, not pathname+cap."),
397 ("repair-cap", None, "Only print repaircap, not pathname+cap."),
398 ("raw", "r", "Display raw JSON data instead of parsed."),
400 def parseArgs(self, where=''):
401 self.where = argv_to_unicode(where)
403 synopsis = "[options] [ALIAS:PATH]"
405 Print a list of all files and directories reachable from the given
408 class StatsOptions(FilesystemOptions):
410 ("raw", "r", "Display raw JSON data instead of parsed"),
412 def parseArgs(self, where=''):
413 self.where = argv_to_unicode(where)
415 synopsis = "[options] [ALIAS:PATH]"
417 Print statistics about of all files and directories reachable from the
418 given starting point."""
420 class CheckOptions(FilesystemOptions):
422 ("raw", None, "Display raw JSON data instead of parsed."),
423 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
424 ("repair", None, "Automatically repair any problems found."),
425 ("add-lease", None, "Add/renew lease on all shares."),
427 def parseArgs(self, *locations):
428 self.locations = map(argv_to_unicode, locations)
430 synopsis = "[options] [ALIAS:PATH]"
432 Check a single file or directory: count how many shares are available and
433 verify their hashes. Optionally repair the file if any problems were
436 class DeepCheckOptions(FilesystemOptions):
438 ("raw", None, "Display raw JSON data instead of parsed."),
439 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
440 ("repair", None, "Automatically repair any problems found."),
441 ("add-lease", None, "Add/renew lease on all shares."),
442 ("verbose", "v", "Be noisy about what is happening."),
444 def parseArgs(self, *locations):
445 self.locations = map(argv_to_unicode, locations)
447 synopsis = "[options] [ALIAS:PATH]"
449 Check all files and directories reachable from the given starting point
450 (which must be a directory), like 'tahoe check' but for multiple files.
451 Optionally repair any problems found."""
454 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
455 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
456 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
457 ["list-aliases", None, ListAliasesOptions, "List all alias caps."],
458 ["ls", None, ListOptions, "List a directory."],
459 ["get", None, GetOptions, "Retrieve a file from the grid."],
460 ["put", None, PutOptions, "Upload a file into the grid."],
461 ["cp", None, CpOptions, "Copy one or more files or directories."],
462 ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."],
463 ["rm", None, RmOptions, "Unlink a file or directory on the grid (same as unlink)."],
464 ["mv", None, MvOptions, "Move a file within the grid."],
465 ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
466 ["backup", None, BackupOptions, "Make target dir look like local dir."],
467 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
468 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
469 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
470 ["check", None, CheckOptions, "Check a single file or directory."],
471 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
475 from allmydata.scripts import tahoe_mkdir
476 rc = tahoe_mkdir.mkdir(options)
479 def add_alias(options):
480 from allmydata.scripts import tahoe_add_alias
481 rc = tahoe_add_alias.add_alias(options)
484 def create_alias(options):
485 from allmydata.scripts import tahoe_add_alias
486 rc = tahoe_add_alias.create_alias(options)
489 def list_aliases(options):
490 from allmydata.scripts import tahoe_add_alias
491 rc = tahoe_add_alias.list_aliases(options)
495 from allmydata.scripts import tahoe_ls
496 rc = tahoe_ls.list(options)
500 from allmydata.scripts import tahoe_get
501 rc = tahoe_get.get(options)
503 if options.to_file is None:
504 # be quiet, since the file being written to stdout should be
505 # proof enough that it worked, unless the user is unlucky
506 # enough to have picked an empty file
509 print >>options.stderr, "%s retrieved and written to %s" % \
510 (options.from_file, options.to_file)
514 from allmydata.scripts import tahoe_put
515 rc = tahoe_put.put(options)
519 from allmydata.scripts import tahoe_cp
520 rc = tahoe_cp.copy(options)
523 def unlink(options, command="unlink"):
524 from allmydata.scripts import tahoe_unlink
525 rc = tahoe_unlink.unlink(options, command=command)
529 return unlink(options, command="rm")
532 from allmydata.scripts import tahoe_mv
533 rc = tahoe_mv.mv(options, mode="move")
537 from allmydata.scripts import tahoe_mv
538 rc = tahoe_mv.mv(options, mode="link")
542 from allmydata.scripts import tahoe_backup
543 rc = tahoe_backup.backup(options)
546 def webopen(options, opener=None):
547 from allmydata.scripts import tahoe_webopen
548 rc = tahoe_webopen.webopen(options, opener=opener)
551 def manifest(options):
552 from allmydata.scripts import tahoe_manifest
553 rc = tahoe_manifest.manifest(options)
557 from allmydata.scripts import tahoe_manifest
558 rc = tahoe_manifest.stats(options)
562 from allmydata.scripts import tahoe_check
563 rc = tahoe_check.check(options)
566 def deepcheck(options):
567 from allmydata.scripts import tahoe_check
568 rc = tahoe_check.deepcheck(options)
573 "add-alias": add_alias,
574 "create-alias": create_alias,
575 "list-aliases": list_aliases,
586 "manifest": manifest,
589 "deep-check": deepcheck,