1 import os.path, re, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import get_aliases, get_default_nodedir, \
4 DEFAULT_ALIAS, BaseOptions
5 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_local_unicode_path
7 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
9 _default_nodedir = get_default_nodedir()
11 class FilesystemOptions(BaseOptions):
13 ["node-url", "u", None,
14 "Specify the URL of the Tahoe gateway node, such as "
15 "'http://127.0.0.1:3456'. "
16 "This overrides the URL found in the --node-directory ."],
17 ["dir-cap", None, None,
18 "Specify which dirnode URI should be used as the 'tahoe' alias."]
21 def postOptions(self):
22 self["quiet"] = self.parent["quiet"]
23 if self.parent['node-directory']:
24 self['node-directory'] = argv_to_abspath(self.parent['node-directory'])
26 self['node-directory'] = _default_nodedir
28 # compute a node-url from the existing options, put in self['node-url']
30 if (not isinstance(self['node-url'], basestring)
31 or not NODEURL_RE.match(self['node-url'])):
32 msg = ("--node-url is required to be a string and look like "
33 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
35 raise usage.UsageError(msg)
37 node_url_file = os.path.join(self['node-directory'], "node.url")
38 self['node-url'] = open(node_url_file, "r").read().strip()
39 if self['node-url'][-1] != "/":
40 self['node-url'] += "/"
42 aliases = get_aliases(self['node-directory'])
44 aliases[DEFAULT_ALIAS] = self['dir-cap']
45 self.aliases = aliases # maps alias name to dircap
48 class MakeDirectoryOptions(FilesystemOptions):
50 ("format", None, None, "Create a directory with the given format: SDMF or MDMF (case-insensitive)"),
53 def parseArgs(self, where=""):
54 self.where = argv_to_unicode(where)
57 if self['format'].upper() not in ("SDMF", "MDMF"):
58 raise usage.UsageError("%s is an invalid format" % self['format'])
60 def getSynopsis(self):
61 return "Usage: %s [global-opts] mkdir [options] [REMOTE_DIR]" % (self.command_name,)
63 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
65 class AddAliasOptions(FilesystemOptions):
66 def parseArgs(self, alias, cap):
67 self.alias = argv_to_unicode(alias)
68 if self.alias.endswith(u':'):
69 self.alias = self.alias[:-1]
72 def getSynopsis(self):
73 return "Usage: %s [global-opts] add-alias [options] ALIAS[:] DIRCAP" % (self.command_name,)
75 longdesc = """Add a new alias for an existing directory."""
77 class CreateAliasOptions(FilesystemOptions):
78 def parseArgs(self, alias):
79 self.alias = argv_to_unicode(alias)
80 if self.alias.endswith(u':'):
81 self.alias = self.alias[:-1]
83 def getSynopsis(self):
84 return "Usage: %s [global-opts] create-alias [options] ALIAS[:]" % (self.command_name,)
86 longdesc = """Create a new directory and add an alias for it."""
88 class ListAliasesOptions(FilesystemOptions):
89 def getSynopsis(self):
90 return "Usage: %s [global-opts] list-aliases [options]" % (self.command_name,)
92 longdesc = """Display a table of all configured aliases."""
94 class ListOptions(FilesystemOptions):
96 ("long", "l", "Use long format: show file sizes, and timestamps."),
97 ("uri", "u", "Show file/directory URIs."),
98 ("readonly-uri", None, "Show read-only file/directory URIs."),
99 ("classify", "F", "Append '/' to directory names, and '*' to mutable."),
100 ("json", None, "Show the raw JSON output."),
102 def parseArgs(self, where=""):
103 self.where = argv_to_unicode(where)
105 def getSynopsis(self):
106 return "Usage: %s [global-opts] ls [options] [PATH]" % (self.command_name,)
109 List the contents of some portion of the grid.
111 If PATH is omitted, "tahoe:" is assumed.
113 When the -l or --long option is used, each line is shown in the
116 drwx <size> <date/time> <name in this directory>
118 where each of the letters on the left may be replaced by '-'.
119 If 'd' is present, it indicates that the object is a directory.
120 If the 'd' is replaced by a '?', the object type is unknown.
121 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
122 then the object is writeable through its link in this directory
123 (note that the link might be replaceable even if the object is
124 not writeable through the current link).
125 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
126 only to indicate that the contents of a directory can be listed.
128 Directories have no size, so their size field is shown as '-'.
129 Otherwise the size of the file, when known, is given in bytes.
130 The size of mutable files or unknown objects is shown as '?'.
132 The date/time shows when this link in the Tahoe filesystem was
136 class GetOptions(FilesystemOptions):
137 def parseArgs(self, arg1, arg2=None):
138 # tahoe get FOO |less # write to stdout
139 # tahoe get tahoe:FOO |less # same
140 # tahoe get FOO bar # write to local file
141 # tahoe get tahoe:FOO bar # same
146 self.from_file = argv_to_unicode(arg1)
147 self.to_file = None if arg2 is None else argv_to_abspath(arg2)
149 def getSynopsis(self):
150 return "Usage: %s [global-opts] get [options] REMOTE_FILE LOCAL_FILE" % (self.command_name,)
153 Retrieve a file from the grid and write it to the local filesystem. If
154 LOCAL_FILE is omitted or '-', the contents of the file will be written to
157 def getUsage(self, width=None):
158 t = FilesystemOptions.getUsage(self, width)
161 % tahoe get FOO |less # write to stdout
162 % tahoe get tahoe:FOO |less # same
163 % tahoe get FOO bar # write to local file
164 % tahoe get tahoe:FOO bar # same
168 class PutOptions(FilesystemOptions):
170 ("mutable", "m", "Create a mutable file instead of an immutable one (like --format=SDMF)"),
173 ("format", None, None, "Create a file with the given format: SDMF and MDMF for mutable, CHK (default) for immutable. (case-insensitive)"),
176 def parseArgs(self, arg1=None, arg2=None):
182 self.from_file = None if arg1 is None else argv_to_abspath(arg1)
183 self.to_file = None if arg2 is None else argv_to_unicode(arg2)
186 if self['format'].upper() not in ("SDMF", "MDMF", "CHK"):
187 raise usage.UsageError("%s is an invalid format" % self['format'])
189 def getSynopsis(self):
190 return "Usage: %s [global-opts] put [options] LOCAL_FILE REMOTE_FILE" % (self.command_name,)
193 Put a file into the grid, copying its contents from the local filesystem.
194 If REMOTE_FILE is missing, upload the file but do not link it into a
195 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
196 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
197 with tahoe: unless otherwise specified.
199 If the destination file already exists and is mutable, it will be modified
200 in-place, whether or not --mutable is specified. (--mutable only affects
201 creation of new files.)"""
203 def getUsage(self, width=None):
204 t = FilesystemOptions.getUsage(self, width)
207 % cat FILE | tahoe put # create unlinked file from stdin
208 % cat FILE | tahoe put - # same
209 % tahoe put bar # create unlinked file from local 'bar'
210 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
211 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
212 % tahoe put bar tahoe:FOO # same
213 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
217 class CpOptions(FilesystemOptions):
219 ("recursive", "r", "Copy source directory recursively."),
220 ("verbose", "v", "Be noisy about what is happening."),
222 "When copying to local files, write out filecaps instead of actual "
223 "data (only useful for debugging and tree-comparison purposes)."),
226 def parseArgs(self, *args):
228 raise usage.UsageError("cp requires at least two arguments")
229 self.sources = map(argv_to_unicode, args[:-1])
230 self.destination = argv_to_unicode(args[-1])
232 def getSynopsis(self):
233 return "Usage: %s [global-opts] cp [options] FROM.. TO" % (self.command_name,)
236 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
237 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
238 files or non-file arguments. Directories will be copied recursively.
239 New Tahoe-side directories will be created when necessary. Assuming that
240 you have previously set up an alias 'home' with 'tahoe create-alias home',
241 here are some examples:
243 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
245 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
247 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
249 You can also use a dircap as either FROM or TO target:
251 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
253 This command still has some limitations: symlinks and special files
254 (device nodes, named pipes) are not handled very well. Arguments should
255 probably not have trailing slashes. 'tahoe cp' does not behave as much
256 like /bin/cp as you would wish, especially with respect to trailing
260 class UnlinkOptions(FilesystemOptions):
261 def parseArgs(self, where):
262 self.where = argv_to_unicode(where)
264 def getSynopsis(self):
265 return "Usage: %s [global-opts] unlink [options] REMOTE_FILE" % (self.command_name,)
267 class RmOptions(UnlinkOptions):
268 def getSynopsis(self):
269 return "Usage: %s [global-opts] rm [options] REMOTE_FILE" % (self.command_name,)
271 class MvOptions(FilesystemOptions):
272 def parseArgs(self, frompath, topath):
273 self.from_file = argv_to_unicode(frompath)
274 self.to_file = argv_to_unicode(topath)
276 def getSynopsis(self):
277 return "Usage: %s [global-opts] mv [options] FROM TO" % (self.command_name,)
280 Use 'tahoe mv' to move files that are already on the grid elsewhere on
281 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
283 If moving a remote file into a remote directory, you'll need to append a
284 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
285 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
287 Note that it is not possible to use this command to move local files to
288 the grid -- use 'tahoe cp' for that.
291 class LnOptions(FilesystemOptions):
292 def parseArgs(self, frompath, topath):
293 self.from_file = argv_to_unicode(frompath)
294 self.to_file = argv_to_unicode(topath)
296 def getSynopsis(self):
297 return "Usage: %s [global-opts] ln [options] FROM_LINK TO_LINK" % (self.command_name,)
300 Use 'tahoe ln' to duplicate a link (directory entry) already on the grid
301 to elsewhere on the grid. For example 'tahoe ln alias:some_file
302 alias:new_file'. causes 'alias:new_file' to point to the same object that
303 'alias:some_file' points to.
305 (The argument order is the same as Unix ln. To remember the order, you
306 can think of this command as copying a link, rather than copying a file
307 as 'tahoe cp' does. Then the argument order is consistent with that of
310 When linking a remote file into a remote directory, you'll need to append
311 a '/' to the name of the remote directory, e.g. 'tahoe ln tahoe:file1
312 tahoe:dir/' (which is shorthand for 'tahoe ln tahoe:file1
313 tahoe:dir/file1'). If you forget the '/', e.g. 'tahoe ln tahoe:file1
314 tahoe:dir', the 'ln' command will refuse to overwrite the 'tahoe:dir'
315 directory, and will exit with an error.
317 Note that it is not possible to use this command to create links between
318 local and remote files.
321 class BackupConfigurationError(Exception):
324 class BackupOptions(FilesystemOptions):
326 ("verbose", "v", "Be noisy about what is happening."),
327 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
330 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
331 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
332 '=meta-update', '=update', '.bzr', '.bzrignore',
333 '.bzrtags', '.hg', '.hgignore', '_darcs')
336 super(BackupOptions, self).__init__()
337 self['exclude'] = set()
339 def parseArgs(self, localdir, topath):
340 self.from_dir = argv_to_abspath(localdir)
341 self.to_dir = argv_to_unicode(topath)
343 def getSynopsis(self):
344 return "Usage: %s [global-opts] backup [options] FROM ALIAS:TO" % (self.command_name,)
346 def opt_exclude(self, pattern):
347 """Ignore files matching a glob pattern. You may give multiple
348 '--exclude' options."""
349 g = argv_to_unicode(pattern).strip()
351 exclude = self['exclude']
354 def opt_exclude_from(self, filepath):
355 """Ignore file matching glob patterns listed in file, one per
356 line. The file is assumed to be in the argv encoding."""
357 abs_filepath = argv_to_abspath(filepath)
359 exclude_file = file(abs_filepath)
361 raise BackupConfigurationError('Error opening exclude file %s.' % quote_local_unicode_path(abs_filepath))
363 for line in exclude_file:
364 self.opt_exclude(line)
368 def opt_exclude_vcs(self):
369 """Exclude files and directories used by following version control
370 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
372 for pattern in self.vcs_patterns:
373 self.opt_exclude(pattern)
375 def filter_listdir(self, listdir):
376 """Yields non-excluded childpaths in path."""
377 exclude = self['exclude']
378 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
379 for filename in listdir:
380 for regexp in exclude_regexps:
381 if regexp.match(filename):
387 Add a versioned backup of the local FROM directory to a timestamped
388 subdirectory of the TO/Archives directory on the grid, sharing as many
389 files and directories as possible with earlier backups. Create TO/Latest
390 as a reference to the latest backup. Behaves somewhat like 'rsync -a
391 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
392 TO/Archives/(new) TO/Latest'."""
394 class WebopenOptions(FilesystemOptions):
396 ("info", "i", "Open the t=info page for the file"),
398 def parseArgs(self, where=''):
399 self.where = argv_to_unicode(where)
401 def getSynopsis(self):
402 return "Usage: %s [global-opts] webopen [options] [ALIAS:PATH]" % (self.command_name,)
404 longdesc = """Open a web browser to the contents of some file or
405 directory on the grid. When run without arguments, open the Welcome
408 class ManifestOptions(FilesystemOptions):
410 ("storage-index", "s", "Only print storage index strings, not pathname+cap."),
411 ("verify-cap", None, "Only print verifycap, not pathname+cap."),
412 ("repair-cap", None, "Only print repaircap, not pathname+cap."),
413 ("raw", "r", "Display raw JSON data instead of parsed."),
415 def parseArgs(self, where=''):
416 self.where = argv_to_unicode(where)
418 def getSynopsis(self):
419 return "Usage: %s [global-opts] manifest [options] [ALIAS:PATH]" % (self.command_name,)
421 longdesc = """Print a list of all files and directories reachable from
422 the given starting point."""
424 class StatsOptions(FilesystemOptions):
426 ("raw", "r", "Display raw JSON data instead of parsed"),
428 def parseArgs(self, where=''):
429 self.where = argv_to_unicode(where)
431 def getSynopsis(self):
432 return "Usage: %s [global-opts] stats [options] [ALIAS:PATH]" % (self.command_name,)
434 longdesc = """Print statistics about of all files and directories
435 reachable from the given starting point."""
437 class CheckOptions(FilesystemOptions):
439 ("raw", None, "Display raw JSON data instead of parsed."),
440 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
441 ("repair", None, "Automatically repair any problems found."),
442 ("add-lease", None, "Add/renew lease on all shares."),
444 def parseArgs(self, *locations):
445 self.locations = map(argv_to_unicode, locations)
447 def getSynopsis(self):
448 return "Usage: %s [global-opts] check [options] [ALIAS:PATH]" % (self.command_name,)
451 Check a single file or directory: count how many shares are available and
452 verify their hashes. Optionally repair the file if any problems were
455 class DeepCheckOptions(FilesystemOptions):
457 ("raw", None, "Display raw JSON data instead of parsed."),
458 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
459 ("repair", None, "Automatically repair any problems found."),
460 ("add-lease", None, "Add/renew lease on all shares."),
461 ("verbose", "v", "Be noisy about what is happening."),
463 def parseArgs(self, *locations):
464 self.locations = map(argv_to_unicode, locations)
466 def getSynopsis(self):
467 return "Usage: %s [global-opts] deep-check [options] [ALIAS:PATH]" % (self.command_name,)
470 Check all files and directories reachable from the given starting point
471 (which must be a directory), like 'tahoe check' but for multiple files.
472 Optionally repair any problems found."""
475 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
476 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
477 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
478 ["list-aliases", None, ListAliasesOptions, "List all alias caps."],
479 ["ls", None, ListOptions, "List a directory."],
480 ["get", None, GetOptions, "Retrieve a file from the grid."],
481 ["put", None, PutOptions, "Upload a file into the grid."],
482 ["cp", None, CpOptions, "Copy one or more files or directories."],
483 ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."],
484 ["rm", None, RmOptions, "Unlink a file or directory on the grid (same as unlink)."],
485 ["mv", None, MvOptions, "Move a file within the grid."],
486 ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
487 ["backup", None, BackupOptions, "Make target dir look like local dir."],
488 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
489 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
490 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
491 ["check", None, CheckOptions, "Check a single file or directory."],
492 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
496 from allmydata.scripts import tahoe_mkdir
497 rc = tahoe_mkdir.mkdir(options)
500 def add_alias(options):
501 from allmydata.scripts import tahoe_add_alias
502 rc = tahoe_add_alias.add_alias(options)
505 def create_alias(options):
506 from allmydata.scripts import tahoe_add_alias
507 rc = tahoe_add_alias.create_alias(options)
510 def list_aliases(options):
511 from allmydata.scripts import tahoe_add_alias
512 rc = tahoe_add_alias.list_aliases(options)
516 from allmydata.scripts import tahoe_ls
517 rc = tahoe_ls.list(options)
521 from allmydata.scripts import tahoe_get
522 rc = tahoe_get.get(options)
524 if options.to_file is None:
525 # be quiet, since the file being written to stdout should be
526 # proof enough that it worked, unless the user is unlucky
527 # enough to have picked an empty file
530 print >>options.stderr, "%s retrieved and written to %s" % \
531 (options.from_file, options.to_file)
535 from allmydata.scripts import tahoe_put
536 rc = tahoe_put.put(options)
540 from allmydata.scripts import tahoe_cp
541 rc = tahoe_cp.copy(options)
544 def unlink(options, command="unlink"):
545 from allmydata.scripts import tahoe_unlink
546 rc = tahoe_unlink.unlink(options, command=command)
550 return unlink(options, command="rm")
553 from allmydata.scripts import tahoe_mv
554 rc = tahoe_mv.mv(options, mode="move")
558 from allmydata.scripts import tahoe_mv
559 rc = tahoe_mv.mv(options, mode="link")
563 from allmydata.scripts import tahoe_backup
564 rc = tahoe_backup.backup(options)
567 def webopen(options, opener=None):
568 from allmydata.scripts import tahoe_webopen
569 rc = tahoe_webopen.webopen(options, opener=opener)
572 def manifest(options):
573 from allmydata.scripts import tahoe_manifest
574 rc = tahoe_manifest.manifest(options)
578 from allmydata.scripts import tahoe_manifest
579 rc = tahoe_manifest.stats(options)
583 from allmydata.scripts import tahoe_check
584 rc = tahoe_check.check(options)
587 def deepcheck(options):
588 from allmydata.scripts import tahoe_check
589 rc = tahoe_check.deepcheck(options)
594 "add-alias": add_alias,
595 "create-alias": create_alias,
596 "list-aliases": list_aliases,
607 "manifest": manifest,
610 "deep-check": deepcheck,