1 import os.path, re, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import BaseOptions, get_aliases, get_default_nodedir, DEFAULT_ALIAS
4 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_output
6 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
8 _default_nodedir = get_default_nodedir()
10 class VDriveOptions(BaseOptions):
12 ["node-directory", "d", None,
13 "Specify which Tahoe node directory should be used. The directory "
14 "should either contain a full Tahoe node, or a file named node.url "
15 "that points to some other Tahoe node. It should also contain a file "
16 "named '" + os.path.join('private', 'aliases') + "' which contains the "
17 "mapping from alias name to root dirnode URI." + (
18 _default_nodedir and (" [default: " + quote_output(_default_nodedir) + "]") or "")],
19 ["node-url", "u", None,
20 "Specify the URL of the Tahoe gateway node, such as 'http://127.0.0.1:3456'. "
21 "This overrides the URL found in the --node-directory ."],
22 ["dir-cap", None, None,
23 "Specify which dirnode URI should be used as the 'tahoe' alias."]
26 def postOptions(self):
27 if self['node-directory']:
28 self['node-directory'] = argv_to_abspath(self['node-directory'])
30 self['node-directory'] = _default_nodedir
32 # compute a node-url from the existing options, put in self['node-url']
34 if (not isinstance(self['node-url'], basestring)
35 or not NODEURL_RE.match(self['node-url'])):
36 msg = ("--node-url is required to be a string and look like "
37 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
39 raise usage.UsageError(msg)
41 node_url_file = os.path.join(self['node-directory'], "node.url")
42 self['node-url'] = open(node_url_file, "r").read().strip()
43 if self['node-url'][-1] != "/":
44 self['node-url'] += "/"
46 aliases = get_aliases(self['node-directory'])
48 aliases[DEFAULT_ALIAS] = self['dir-cap']
49 self.aliases = aliases # maps alias name to dircap
52 class MakeDirectoryOptions(VDriveOptions):
54 ("format", None, None, "Create directory with the given format: SDMF and MDMF for mutable. (case-insensitive)"),
57 def parseArgs(self, where=""):
58 self.where = argv_to_unicode(where)
61 if self['format'].upper() not in ("SDMF", "MDMF", "CHK"):
62 raise usage.UsageError("%s is an invalid format" % self['format'])
64 def getSynopsis(self):
65 return "Usage: %s mkdir [options] [REMOTE_DIR]" % (self.command_name,)
67 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
69 class AddAliasOptions(VDriveOptions):
70 def parseArgs(self, alias, cap):
71 self.alias = argv_to_unicode(alias)
72 if self.alias.endswith(u':'):
73 self.alias = self.alias[:-1]
76 def getSynopsis(self):
77 return "Usage: %s add-alias [options] ALIAS[:] DIRCAP" % (self.command_name,)
79 longdesc = """Add a new alias for an existing directory."""
81 class CreateAliasOptions(VDriveOptions):
82 def parseArgs(self, alias):
83 self.alias = argv_to_unicode(alias)
84 if self.alias.endswith(u':'):
85 self.alias = self.alias[:-1]
87 def getSynopsis(self):
88 return "Usage: %s create-alias [options] ALIAS[:]" % (self.command_name,)
90 longdesc = """Create a new directory and add an alias for it."""
92 class ListAliasesOptions(VDriveOptions):
93 def getSynopsis(self):
94 return "Usage: %s list-aliases [options]" % (self.command_name,)
96 longdesc = """Display a table of all configured aliases."""
98 class ListOptions(VDriveOptions):
100 ("long", "l", "Use long format: show file sizes, and timestamps."),
101 ("uri", "u", "Show file/directory URIs."),
102 ("readonly-uri", None, "Show read-only file/directory URIs."),
103 ("classify", "F", "Append '/' to directory names, and '*' to mutable."),
104 ("json", None, "Show the raw JSON output."),
106 def parseArgs(self, where=""):
107 self.where = argv_to_unicode(where)
110 List the contents of some portion of the grid.
112 When the -l or --long option is used, each line is shown in the
115 drwx <size> <date/time> <name in this directory>
117 where each of the letters on the left may be replaced by '-'.
118 If 'd' is present, it indicates that the object is a directory.
119 If the 'd' is replaced by a '?', the object type is unknown.
120 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
121 then the object is writeable through its link in this directory
122 (note that the link might be replaceable even if the object is
123 not writeable through the current link).
124 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
125 only to indicate that the contents of a directory can be listed.
127 Directories have no size, so their size field is shown as '-'.
128 Otherwise the size of the file, when known, is given in bytes.
129 The size of mutable files or unknown objects is shown as '?'.
131 The date/time shows when this link in the Tahoe filesystem was
135 class GetOptions(VDriveOptions):
136 def parseArgs(self, arg1, arg2=None):
137 # tahoe get FOO |less # write to stdout
138 # tahoe get tahoe:FOO |less # same
139 # tahoe get FOO bar # write to local file
140 # tahoe get tahoe:FOO bar # same
142 self.from_file = argv_to_unicode(arg1)
145 self.to_file = argv_to_unicode(arg2)
149 if self.to_file == "-":
152 def getSynopsis(self):
153 return "Usage: %s get [options] REMOTE_FILE LOCAL_FILE" % (self.command_name,)
156 Retrieve a file from the grid and write it to the local filesystem. If
157 LOCAL_FILE is omitted or '-', the contents of the file will be written to
160 def getUsage(self, width=None):
161 t = VDriveOptions.getUsage(self, width)
164 % tahoe get FOO |less # write to stdout
165 % tahoe get tahoe:FOO |less # same
166 % tahoe get FOO bar # write to local file
167 % tahoe get tahoe:FOO bar # same
171 class PutOptions(VDriveOptions):
173 ("mutable", "m", "Create a mutable file instead of an immutable one. (DEPRECATED, use --format=SDMF)"),
176 ("format", None, None, "Create file with the given format: SDMF and MDMF for mutable, CHK (default) for immutable. (case-insensitive)"),
179 def parseArgs(self, arg1=None, arg2=None):
182 if arg1 is not None and arg2 is not None:
183 self.from_file = argv_to_unicode(arg1)
184 self.to_file = argv_to_unicode(arg2)
185 elif arg1 is not None and arg2 is None:
186 self.from_file = argv_to_unicode(arg1) # might be "-"
189 self.from_file = None
191 if self.from_file == u"-":
192 self.from_file = None
195 if self['format'].upper() not in ("SDMF", "MDMF", "CHK"):
196 raise usage.UsageError("%s is an invalid format" % self['format'])
198 def getSynopsis(self):
199 return "Usage: %s put [options] LOCAL_FILE REMOTE_FILE" % (self.command_name,)
202 Put a file into the grid, copying its contents from the local filesystem.
203 If REMOTE_FILE is missing, upload the file but do not link it into a
204 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
205 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
206 with tahoe: unless otherwise specified."""
208 def getUsage(self, width=None):
209 t = VDriveOptions.getUsage(self, width)
212 % cat FILE | tahoe put # create unlinked file from stdin
213 % cat FILE | tahoe put - # same
214 % tahoe put bar # create unlinked file from local 'bar'
215 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
216 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
217 % tahoe put bar tahoe:FOO # same
218 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
222 class CpOptions(VDriveOptions):
224 ("recursive", "r", "Copy source directory recursively."),
225 ("verbose", "v", "Be noisy about what is happening."),
227 "When copying to local files, write out filecaps instead of actual "
228 "data (only useful for debugging and tree-comparison purposes)."),
231 def parseArgs(self, *args):
233 raise usage.UsageError("cp requires at least two arguments")
234 self.sources = map(argv_to_unicode, args[:-1])
235 self.destination = argv_to_unicode(args[-1])
237 def getSynopsis(self):
238 return "Usage: %s cp [options] FROM.. TO" % (self.command_name,)
241 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
242 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
243 files or non-file arguments. Directories will be copied recursively.
244 New Tahoe-side directories will be created when necessary. Assuming that
245 you have previously set up an alias 'home' with 'tahoe create-alias home',
246 here are some examples:
248 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
250 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
252 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
254 You can also use a dircap as either FROM or TO target:
256 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
258 This command still has some limitations: symlinks and special files
259 (device nodes, named pipes) are not handled very well. Arguments should
260 probably not have trailing slashes. 'tahoe cp' does not behave as much
261 like /bin/cp as you would wish, especially with respect to trailing
265 class UnlinkOptions(VDriveOptions):
266 def parseArgs(self, where):
267 self.where = argv_to_unicode(where)
269 def getSynopsis(self):
270 return "Usage: %s unlink [options] REMOTE_FILE" % (self.command_name,)
272 class RmOptions(UnlinkOptions):
273 def getSynopsis(self):
274 return "Usage: %s rm [options] REMOTE_FILE" % (self.command_name,)
276 class MvOptions(VDriveOptions):
277 def parseArgs(self, frompath, topath):
278 self.from_file = argv_to_unicode(frompath)
279 self.to_file = argv_to_unicode(topath)
281 def getSynopsis(self):
282 return "Usage: %s mv [options] FROM TO" % (self.command_name,)
285 Use 'tahoe mv' to move files that are already on the grid elsewhere on
286 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
288 If moving a remote file into a remote directory, you'll need to append a
289 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
290 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
292 Note that it is not possible to use this command to move local files to
293 the grid -- use 'tahoe cp' for that.
296 class LnOptions(VDriveOptions):
297 def parseArgs(self, frompath, topath):
298 self.from_file = argv_to_unicode(frompath)
299 self.to_file = argv_to_unicode(topath)
301 def getSynopsis(self):
302 return "Usage: %s ln [options] FROM_LINK TO_LINK" % (self.command_name,)
305 Use 'tahoe ln' to duplicate a link (directory entry) already on the grid
306 to elsewhere on the grid. For example 'tahoe ln alias:some_file
307 alias:new_file'. causes 'alias:new_file' to point to the same object that
308 'alias:some_file' points to.
310 (The argument order is the same as Unix ln. To remember the order, you
311 can think of this command as copying a link, rather than copying a file
312 as 'tahoe cp' does. Then the argument order is consistent with that of
315 When linking a remote file into a remote directory, you'll need to append
316 a '/' to the name of the remote directory, e.g. 'tahoe ln tahoe:file1
317 tahoe:dir/' (which is shorthand for 'tahoe ln tahoe:file1
318 tahoe:dir/file1'). If you forget the '/', e.g. 'tahoe ln tahoe:file1
319 tahoe:dir', the 'ln' command will refuse to overwrite the 'tahoe:dir'
320 directory, and will exit with an error.
322 Note that it is not possible to use this command to create links between
323 local and remote files.
326 class BackupConfigurationError(Exception):
329 class BackupOptions(VDriveOptions):
331 ("verbose", "v", "Be noisy about what is happening."),
332 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
335 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
336 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
337 '=meta-update', '=update', '.bzr', '.bzrignore',
338 '.bzrtags', '.hg', '.hgignore', '_darcs')
341 super(BackupOptions, self).__init__()
342 self['exclude'] = set()
344 def parseArgs(self, localdir, topath):
345 self.from_dir = argv_to_unicode(localdir)
346 self.to_dir = argv_to_unicode(topath)
348 def getSynopsis(self):
349 return "Usage: %s backup [options] FROM ALIAS:TO" % (self.command_name,)
351 def opt_exclude(self, pattern):
352 """Ignore files matching a glob pattern. You may give multiple
353 '--exclude' options."""
354 g = argv_to_unicode(pattern).strip()
356 exclude = self['exclude']
359 def opt_exclude_from(self, filepath):
360 """Ignore file matching glob patterns listed in file, one per
361 line. The file is assumed to be in the argv encoding."""
362 abs_filepath = argv_to_abspath(filepath)
364 exclude_file = file(abs_filepath)
366 raise BackupConfigurationError('Error opening exclude file %s.' % quote_output(abs_filepath))
368 for line in exclude_file:
369 self.opt_exclude(line)
373 def opt_exclude_vcs(self):
374 """Exclude files and directories used by following version control
375 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
377 for pattern in self.vcs_patterns:
378 self.opt_exclude(pattern)
380 def filter_listdir(self, listdir):
381 """Yields non-excluded childpaths in path."""
382 exclude = self['exclude']
383 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
384 for filename in listdir:
385 for regexp in exclude_regexps:
386 if regexp.match(filename):
392 Add a versioned backup of the local FROM directory to a timestamped
393 subdirectory of the TO/Archives directory on the grid, sharing as many
394 files and directories as possible with earlier backups. Create TO/Latest
395 as a reference to the latest backup. Behaves somewhat like 'rsync -a
396 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
397 TO/Archives/(new) TO/Latest'."""
399 class WebopenOptions(VDriveOptions):
401 ("info", "i", "Open the t=info page for the file"),
403 def parseArgs(self, where=''):
404 self.where = argv_to_unicode(where)
406 def getSynopsis(self):
407 return "Usage: %s webopen [options] [ALIAS:PATH]" % (self.command_name,)
409 longdesc = """Open a web browser to the contents of some file or
410 directory on the grid. When run without arguments, open the Welcome
413 class ManifestOptions(VDriveOptions):
415 ("storage-index", "s", "Only print storage index strings, not pathname+cap."),
416 ("verify-cap", None, "Only print verifycap, not pathname+cap."),
417 ("repair-cap", None, "Only print repaircap, not pathname+cap."),
418 ("raw", "r", "Display raw JSON data instead of parsed."),
420 def parseArgs(self, where=''):
421 self.where = argv_to_unicode(where)
423 def getSynopsis(self):
424 return "Usage: %s manifest [options] [ALIAS:PATH]" % (self.command_name,)
426 longdesc = """Print a list of all files and directories reachable from
427 the given starting point."""
429 class StatsOptions(VDriveOptions):
431 ("raw", "r", "Display raw JSON data instead of parsed"),
433 def parseArgs(self, where=''):
434 self.where = argv_to_unicode(where)
436 def getSynopsis(self):
437 return "Usage: %s stats [options] [ALIAS:PATH]" % (self.command_name,)
439 longdesc = """Print statistics about of all files and directories
440 reachable from the given starting point."""
442 class CheckOptions(VDriveOptions):
444 ("raw", None, "Display raw JSON data instead of parsed."),
445 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
446 ("repair", None, "Automatically repair any problems found."),
447 ("add-lease", None, "Add/renew lease on all shares."),
449 def parseArgs(self, where=''):
450 self.where = argv_to_unicode(where)
452 def getSynopsis(self):
453 return "Usage: %s check [options] [ALIAS:PATH]" % (self.command_name,)
456 Check a single file or directory: count how many shares are available and
457 verify their hashes. Optionally repair the file if any problems were
460 class DeepCheckOptions(VDriveOptions):
462 ("raw", None, "Display raw JSON data instead of parsed."),
463 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
464 ("repair", None, "Automatically repair any problems found."),
465 ("add-lease", None, "Add/renew lease on all shares."),
466 ("verbose", "v", "Be noisy about what is happening."),
468 def parseArgs(self, where=''):
469 self.where = argv_to_unicode(where)
471 def getSynopsis(self):
472 return "Usage: %s deep-check [options] [ALIAS:PATH]" % (self.command_name,)
475 Check all files and directories reachable from the given starting point
476 (which must be a directory), like 'tahoe check' but for multiple files.
477 Optionally repair any problems found."""
480 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
481 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
482 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
483 ["list-aliases", None, ListAliasesOptions, "List all alias caps."],
484 ["ls", None, ListOptions, "List a directory."],
485 ["get", None, GetOptions, "Retrieve a file from the grid."],
486 ["put", None, PutOptions, "Upload a file into the grid."],
487 ["cp", None, CpOptions, "Copy one or more files or directories."],
488 ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."],
489 ["rm", None, RmOptions, "Unlink a file or directory on the grid (same as unlink)."],
490 ["mv", None, MvOptions, "Move a file within the grid."],
491 ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
492 ["backup", None, BackupOptions, "Make target dir look like local dir."],
493 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
494 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
495 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
496 ["check", None, CheckOptions, "Check a single file or directory."],
497 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
501 from allmydata.scripts import tahoe_mkdir
502 rc = tahoe_mkdir.mkdir(options)
505 def add_alias(options):
506 from allmydata.scripts import tahoe_add_alias
507 rc = tahoe_add_alias.add_alias(options)
510 def create_alias(options):
511 from allmydata.scripts import tahoe_add_alias
512 rc = tahoe_add_alias.create_alias(options)
515 def list_aliases(options):
516 from allmydata.scripts import tahoe_add_alias
517 rc = tahoe_add_alias.list_aliases(options)
521 from allmydata.scripts import tahoe_ls
522 rc = tahoe_ls.list(options)
526 from allmydata.scripts import tahoe_get
527 rc = tahoe_get.get(options)
529 if options.to_file is None:
530 # be quiet, since the file being written to stdout should be
531 # proof enough that it worked, unless the user is unlucky
532 # enough to have picked an empty file
535 print >>options.stderr, "%s retrieved and written to %s" % \
536 (options.from_file, options.to_file)
540 from allmydata.scripts import tahoe_put
541 rc = tahoe_put.put(options)
545 from allmydata.scripts import tahoe_cp
546 rc = tahoe_cp.copy(options)
549 def unlink(options, command="unlink"):
550 from allmydata.scripts import tahoe_unlink
551 rc = tahoe_unlink.unlink(options, command=command)
555 return unlink(options, command="rm")
558 from allmydata.scripts import tahoe_mv
559 rc = tahoe_mv.mv(options, mode="move")
563 from allmydata.scripts import tahoe_mv
564 rc = tahoe_mv.mv(options, mode="link")
568 from allmydata.scripts import tahoe_backup
569 rc = tahoe_backup.backup(options)
572 def webopen(options, opener=None):
573 from allmydata.scripts import tahoe_webopen
574 rc = tahoe_webopen.webopen(options, opener=opener)
577 def manifest(options):
578 from allmydata.scripts import tahoe_manifest
579 rc = tahoe_manifest.manifest(options)
583 from allmydata.scripts import tahoe_manifest
584 rc = tahoe_manifest.stats(options)
588 from allmydata.scripts import tahoe_check
589 rc = tahoe_check.check(options)
592 def deepcheck(options):
593 from allmydata.scripts import tahoe_check
594 rc = tahoe_check.deepcheck(options)
599 "add-alias": add_alias,
600 "create-alias": create_alias,
601 "list-aliases": list_aliases,
612 "manifest": manifest,
615 "deep-check": deepcheck,