1 import os.path, re, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import BaseOptions, get_aliases, get_default_nodedir, DEFAULT_ALIAS
4 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_output
6 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
8 _default_nodedir = get_default_nodedir()
10 class VDriveOptions(BaseOptions):
12 ["node-directory", "d", None,
13 "Specify which Tahoe node directory should be used. The directory "
14 "should either contain a full Tahoe node, or a file named node.url "
15 "that points to some other Tahoe node. It should also contain a file "
16 "named '" + os.path.join('private', 'aliases') + "' which contains the "
17 "mapping from alias name to root dirnode URI." + (
18 _default_nodedir and (" [default: " + quote_output(_default_nodedir) + "]") or "")],
19 ["node-url", "u", None,
20 "Specify the URL of the Tahoe gateway node, such as 'http://127.0.0.1:3456'. "
21 "This overrides the URL found in the --node-directory ."],
22 ["dir-cap", None, None,
23 "Specify which dirnode URI should be used as the 'tahoe' alias."]
26 def postOptions(self):
27 if self['node-directory']:
28 self['node-directory'] = argv_to_abspath(self['node-directory'])
30 self['node-directory'] = _default_nodedir
32 # compute a node-url from the existing options, put in self['node-url']
34 if (not isinstance(self['node-url'], basestring)
35 or not NODEURL_RE.match(self['node-url'])):
36 msg = ("--node-url is required to be a string and look like "
37 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
39 raise usage.UsageError(msg)
41 node_url_file = os.path.join(self['node-directory'], "node.url")
42 self['node-url'] = open(node_url_file, "r").read().strip()
43 if self['node-url'][-1] != "/":
44 self['node-url'] += "/"
46 aliases = get_aliases(self['node-directory'])
48 aliases[DEFAULT_ALIAS] = self['dir-cap']
49 self.aliases = aliases # maps alias name to dircap
52 class MakeDirectoryOptions(VDriveOptions):
54 ("mutable-type", None, False, "Create a mutable file in the given format. Valid formats are 'sdmf' for SDMF and 'mdmf' for MDMF"),
57 def parseArgs(self, where=""):
58 self.where = argv_to_unicode(where)
60 if self['mutable-type'] and self['mutable-type'] not in ("sdmf", "mdmf"):
61 raise usage.UsageError("%s is an invalid format" % self['mutable-type'])
63 def getSynopsis(self):
64 return "Usage: %s mkdir [options] [REMOTE_DIR]" % (self.command_name,)
66 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
68 class AddAliasOptions(VDriveOptions):
69 def parseArgs(self, alias, cap):
70 self.alias = argv_to_unicode(alias)
71 if self.alias.endswith(u':'):
72 self.alias = self.alias[:-1]
75 def getSynopsis(self):
76 return "Usage: %s add-alias [options] ALIAS[:] DIRCAP" % (self.command_name,)
78 longdesc = """Add a new alias for an existing directory."""
80 class CreateAliasOptions(VDriveOptions):
81 def parseArgs(self, alias):
82 self.alias = argv_to_unicode(alias)
83 if self.alias.endswith(u':'):
84 self.alias = self.alias[:-1]
86 def getSynopsis(self):
87 return "Usage: %s create-alias [options] ALIAS[:]" % (self.command_name,)
89 longdesc = """Create a new directory and add an alias for it."""
91 class ListAliasesOptions(VDriveOptions):
92 def getSynopsis(self):
93 return "Usage: %s list-aliases [options]" % (self.command_name,)
95 longdesc = """Display a table of all configured aliases."""
97 class ListOptions(VDriveOptions):
99 ("long", "l", "Use long format: show file sizes, and timestamps."),
100 ("uri", "u", "Show file/directory URIs."),
101 ("readonly-uri", None, "Show read-only file/directory URIs."),
102 ("classify", "F", "Append '/' to directory names, and '*' to mutable."),
103 ("json", None, "Show the raw JSON output."),
105 def parseArgs(self, where=""):
106 self.where = argv_to_unicode(where)
109 List the contents of some portion of the grid.
111 When the -l or --long option is used, each line is shown in the
114 drwx <size> <date/time> <name in this directory>
116 where each of the letters on the left may be replaced by '-'.
117 If 'd' is present, it indicates that the object is a directory.
118 If the 'd' is replaced by a '?', the object type is unknown.
119 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
120 then the object is writeable through its link in this directory
121 (note that the link might be replaceable even if the object is
122 not writeable through the current link).
123 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
124 only to indicate that the contents of a directory can be listed.
126 Directories have no size, so their size field is shown as '-'.
127 Otherwise the size of the file, when known, is given in bytes.
128 The size of mutable files or unknown objects is shown as '?'.
130 The date/time shows when this link in the Tahoe filesystem was
134 class GetOptions(VDriveOptions):
135 def parseArgs(self, arg1, arg2=None):
136 # tahoe get FOO |less # write to stdout
137 # tahoe get tahoe:FOO |less # same
138 # tahoe get FOO bar # write to local file
139 # tahoe get tahoe:FOO bar # same
141 self.from_file = argv_to_unicode(arg1)
144 self.to_file = argv_to_unicode(arg2)
148 if self.to_file == "-":
151 def getSynopsis(self):
152 return "Usage: %s get [options] REMOTE_FILE LOCAL_FILE" % (self.command_name,)
155 Retrieve a file from the grid and write it to the local filesystem. If
156 LOCAL_FILE is omitted or '-', the contents of the file will be written to
159 def getUsage(self, width=None):
160 t = VDriveOptions.getUsage(self, width)
163 % tahoe get FOO |less # write to stdout
164 % tahoe get tahoe:FOO |less # same
165 % tahoe get FOO bar # write to local file
166 % tahoe get tahoe:FOO bar # same
170 class PutOptions(VDriveOptions):
172 ("mutable", "m", "Create a mutable file instead of an immutable one."),
175 ("mutable-type", None, False, "Create a mutable file in the given format. Valid formats are 'sdmf' for SDMF and 'mdmf' for MDMF"),
178 def parseArgs(self, arg1=None, arg2=None):
181 if arg1 is not None and arg2 is not None:
182 self.from_file = argv_to_unicode(arg1)
183 self.to_file = argv_to_unicode(arg2)
184 elif arg1 is not None and arg2 is None:
185 self.from_file = argv_to_unicode(arg1) # might be "-"
188 self.from_file = None
190 if self.from_file == u"-":
191 self.from_file = None
193 if self['mutable-type'] and self['mutable-type'] not in ("sdmf", "mdmf"):
194 raise usage.UsageError("%s is an invalid format" % self['mutable-type'])
196 if self['mutable-type']:
197 self['mutable'] = True
200 def getSynopsis(self):
201 return "Usage: %s put [options] LOCAL_FILE REMOTE_FILE" % (self.command_name,)
204 Put a file into the grid, copying its contents from the local filesystem.
205 If REMOTE_FILE is missing, upload the file but do not link it into a
206 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
207 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
208 with tahoe: unless otherwise specified."""
210 def getUsage(self, width=None):
211 t = VDriveOptions.getUsage(self, width)
214 % cat FILE | tahoe put # create unlinked file from stdin
215 % cat FILE | tahoe put - # same
216 % tahoe put bar # create unlinked file from local 'bar'
217 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
218 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
219 % tahoe put bar tahoe:FOO # same
220 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
224 class CpOptions(VDriveOptions):
226 ("recursive", "r", "Copy source directory recursively."),
227 ("verbose", "v", "Be noisy about what is happening."),
229 "When copying to local files, write out filecaps instead of actual "
230 "data (only useful for debugging and tree-comparison purposes)."),
233 def parseArgs(self, *args):
235 raise usage.UsageError("cp requires at least two arguments")
236 self.sources = map(argv_to_unicode, args[:-1])
237 self.destination = argv_to_unicode(args[-1])
239 def getSynopsis(self):
240 return "Usage: %s cp [options] FROM.. TO" % (self.command_name,)
243 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
244 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
245 files or non-file arguments. Directories will be copied recursively.
246 New Tahoe-side directories will be created when necessary. Assuming that
247 you have previously set up an alias 'home' with 'tahoe create-alias home',
248 here are some examples:
250 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
252 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
254 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
256 You can also use a dircap as either FROM or TO target:
258 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
260 This command still has some limitations: symlinks and special files
261 (device nodes, named pipes) are not handled very well. Arguments should
262 probably not have trailing slashes. 'tahoe cp' does not behave as much
263 like /bin/cp as you would wish, especially with respect to trailing
267 class UnlinkOptions(VDriveOptions):
268 def parseArgs(self, where):
269 self.where = argv_to_unicode(where)
271 def getSynopsis(self):
272 return "Usage: %s unlink [options] REMOTE_FILE" % (self.command_name,)
274 class RmOptions(UnlinkOptions):
275 def getSynopsis(self):
276 return "Usage: %s rm [options] REMOTE_FILE" % (self.command_name,)
278 class MvOptions(VDriveOptions):
279 def parseArgs(self, frompath, topath):
280 self.from_file = argv_to_unicode(frompath)
281 self.to_file = argv_to_unicode(topath)
283 def getSynopsis(self):
284 return "Usage: %s mv [options] FROM TO" % (self.command_name,)
287 Use 'tahoe mv' to move files that are already on the grid elsewhere on
288 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
290 If moving a remote file into a remote directory, you'll need to append a
291 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
292 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
294 Note that it is not possible to use this command to move local files to
295 the grid -- use 'tahoe cp' for that.
298 class LnOptions(VDriveOptions):
299 def parseArgs(self, frompath, topath):
300 self.from_file = argv_to_unicode(frompath)
301 self.to_file = argv_to_unicode(topath)
303 def getSynopsis(self):
304 return "Usage: %s ln [options] FROM_LINK TO_LINK" % (self.command_name,)
307 Use 'tahoe ln' to duplicate a link (directory entry) already on the grid
308 to elsewhere on the grid. For example 'tahoe ln alias:some_file
309 alias:new_file'. causes 'alias:new_file' to point to the same object that
310 'alias:some_file' points to.
312 (The argument order is the same as Unix ln. To remember the order, you
313 can think of this command as copying a link, rather than copying a file
314 as 'tahoe cp' does. Then the argument order is consistent with that of
317 When linking a remote file into a remote directory, you'll need to append
318 a '/' to the name of the remote directory, e.g. 'tahoe ln tahoe:file1
319 tahoe:dir/' (which is shorthand for 'tahoe ln tahoe:file1
320 tahoe:dir/file1'). If you forget the '/', e.g. 'tahoe ln tahoe:file1
321 tahoe:dir', the 'ln' command will refuse to overwrite the 'tahoe:dir'
322 directory, and will exit with an error.
324 Note that it is not possible to use this command to create links between
325 local and remote files.
328 class BackupConfigurationError(Exception):
331 class BackupOptions(VDriveOptions):
333 ("verbose", "v", "Be noisy about what is happening."),
334 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
337 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
338 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
339 '=meta-update', '=update', '.bzr', '.bzrignore',
340 '.bzrtags', '.hg', '.hgignore', '_darcs')
343 super(BackupOptions, self).__init__()
344 self['exclude'] = set()
346 def parseArgs(self, localdir, topath):
347 self.from_dir = argv_to_unicode(localdir)
348 self.to_dir = argv_to_unicode(topath)
350 def getSynopsis(self):
351 return "Usage: %s backup [options] FROM ALIAS:TO" % (self.command_name,)
353 def opt_exclude(self, pattern):
354 """Ignore files matching a glob pattern. You may give multiple
355 '--exclude' options."""
356 g = argv_to_unicode(pattern).strip()
358 exclude = self['exclude']
361 def opt_exclude_from(self, filepath):
362 """Ignore file matching glob patterns listed in file, one per
363 line. The file is assumed to be in the argv encoding."""
364 abs_filepath = argv_to_abspath(filepath)
366 exclude_file = file(abs_filepath)
368 raise BackupConfigurationError('Error opening exclude file %s.' % quote_output(abs_filepath))
370 for line in exclude_file:
371 self.opt_exclude(line)
375 def opt_exclude_vcs(self):
376 """Exclude files and directories used by following version control
377 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
379 for pattern in self.vcs_patterns:
380 self.opt_exclude(pattern)
382 def filter_listdir(self, listdir):
383 """Yields non-excluded childpaths in path."""
384 exclude = self['exclude']
385 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
386 for filename in listdir:
387 for regexp in exclude_regexps:
388 if regexp.match(filename):
394 Add a versioned backup of the local FROM directory to a timestamped
395 subdirectory of the TO/Archives directory on the grid, sharing as many
396 files and directories as possible with earlier backups. Create TO/Latest
397 as a reference to the latest backup. Behaves somewhat like 'rsync -a
398 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
399 TO/Archives/(new) TO/Latest'."""
401 class WebopenOptions(VDriveOptions):
403 ("info", "i", "Open the t=info page for the file"),
405 def parseArgs(self, where=''):
406 self.where = argv_to_unicode(where)
408 def getSynopsis(self):
409 return "Usage: %s webopen [options] [ALIAS:PATH]" % (self.command_name,)
411 longdesc = """Open a web browser to the contents of some file or
412 directory on the grid. When run without arguments, open the Welcome
415 class ManifestOptions(VDriveOptions):
417 ("storage-index", "s", "Only print storage index strings, not pathname+cap."),
418 ("verify-cap", None, "Only print verifycap, not pathname+cap."),
419 ("repair-cap", None, "Only print repaircap, not pathname+cap."),
420 ("raw", "r", "Display raw JSON data instead of parsed."),
422 def parseArgs(self, where=''):
423 self.where = argv_to_unicode(where)
425 def getSynopsis(self):
426 return "Usage: %s manifest [options] [ALIAS:PATH]" % (self.command_name,)
428 longdesc = """Print a list of all files and directories reachable from
429 the given starting point."""
431 class StatsOptions(VDriveOptions):
433 ("raw", "r", "Display raw JSON data instead of parsed"),
435 def parseArgs(self, where=''):
436 self.where = argv_to_unicode(where)
438 def getSynopsis(self):
439 return "Usage: %s stats [options] [ALIAS:PATH]" % (self.command_name,)
441 longdesc = """Print statistics about of all files and directories
442 reachable from the given starting point."""
444 class CheckOptions(VDriveOptions):
446 ("raw", None, "Display raw JSON data instead of parsed."),
447 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
448 ("repair", None, "Automatically repair any problems found."),
449 ("add-lease", None, "Add/renew lease on all shares."),
451 def parseArgs(self, where=''):
452 self.where = argv_to_unicode(where)
454 def getSynopsis(self):
455 return "Usage: %s check [options] [ALIAS:PATH]" % (self.command_name,)
458 Check a single file or directory: count how many shares are available and
459 verify their hashes. Optionally repair the file if any problems were
462 class DeepCheckOptions(VDriveOptions):
464 ("raw", None, "Display raw JSON data instead of parsed."),
465 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
466 ("repair", None, "Automatically repair any problems found."),
467 ("add-lease", None, "Add/renew lease on all shares."),
468 ("verbose", "v", "Be noisy about what is happening."),
470 def parseArgs(self, where=''):
471 self.where = argv_to_unicode(where)
473 def getSynopsis(self):
474 return "Usage: %s deep-check [options] [ALIAS:PATH]" % (self.command_name,)
477 Check all files and directories reachable from the given starting point
478 (which must be a directory), like 'tahoe check' but for multiple files.
479 Optionally repair any problems found."""
482 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
483 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
484 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
485 ["list-aliases", None, ListAliasesOptions, "List all alias caps."],
486 ["ls", None, ListOptions, "List a directory."],
487 ["get", None, GetOptions, "Retrieve a file from the grid."],
488 ["put", None, PutOptions, "Upload a file into the grid."],
489 ["cp", None, CpOptions, "Copy one or more files or directories."],
490 ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."],
491 ["rm", None, RmOptions, "Unlink a file or directory on the grid (same as unlink)."],
492 ["mv", None, MvOptions, "Move a file within the grid."],
493 ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
494 ["backup", None, BackupOptions, "Make target dir look like local dir."],
495 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
496 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
497 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
498 ["check", None, CheckOptions, "Check a single file or directory."],
499 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
503 from allmydata.scripts import tahoe_mkdir
504 rc = tahoe_mkdir.mkdir(options)
507 def add_alias(options):
508 from allmydata.scripts import tahoe_add_alias
509 rc = tahoe_add_alias.add_alias(options)
512 def create_alias(options):
513 from allmydata.scripts import tahoe_add_alias
514 rc = tahoe_add_alias.create_alias(options)
517 def list_aliases(options):
518 from allmydata.scripts import tahoe_add_alias
519 rc = tahoe_add_alias.list_aliases(options)
523 from allmydata.scripts import tahoe_ls
524 rc = tahoe_ls.list(options)
528 from allmydata.scripts import tahoe_get
529 rc = tahoe_get.get(options)
531 if options.to_file is None:
532 # be quiet, since the file being written to stdout should be
533 # proof enough that it worked, unless the user is unlucky
534 # enough to have picked an empty file
537 print >>options.stderr, "%s retrieved and written to %s" % \
538 (options.from_file, options.to_file)
542 from allmydata.scripts import tahoe_put
543 rc = tahoe_put.put(options)
547 from allmydata.scripts import tahoe_cp
548 rc = tahoe_cp.copy(options)
551 def unlink(options, command="unlink"):
552 from allmydata.scripts import tahoe_unlink
553 rc = tahoe_unlink.unlink(options, command=command)
557 return unlink(options, command="rm")
560 from allmydata.scripts import tahoe_mv
561 rc = tahoe_mv.mv(options, mode="move")
565 from allmydata.scripts import tahoe_mv
566 rc = tahoe_mv.mv(options, mode="link")
570 from allmydata.scripts import tahoe_backup
571 rc = tahoe_backup.backup(options)
574 def webopen(options, opener=None):
575 from allmydata.scripts import tahoe_webopen
576 rc = tahoe_webopen.webopen(options, opener=opener)
579 def manifest(options):
580 from allmydata.scripts import tahoe_manifest
581 rc = tahoe_manifest.manifest(options)
585 from allmydata.scripts import tahoe_manifest
586 rc = tahoe_manifest.stats(options)
590 from allmydata.scripts import tahoe_check
591 rc = tahoe_check.check(options)
594 def deepcheck(options):
595 from allmydata.scripts import tahoe_check
596 rc = tahoe_check.deepcheck(options)
601 "add-alias": add_alias,
602 "create-alias": create_alias,
603 "list-aliases": list_aliases,
614 "manifest": manifest,
617 "deep-check": deepcheck,