1 import os.path, re, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import BaseOptions, get_aliases, get_default_nodedir, DEFAULT_ALIAS
4 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_output
6 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
8 _default_nodedir = get_default_nodedir()
10 class VDriveOptions(BaseOptions):
12 ["node-directory", "d", None,
13 "Specify which Tahoe node directory should be used. The directory "
14 "should either contain a full Tahoe node, or a file named node.url "
15 "that points to some other Tahoe node. It should also contain a file "
16 "named '" + os.path.join('private', 'aliases') + "' which contains the "
17 "mapping from alias name to root dirnode URI." + (
18 _default_nodedir and (" [default: " + quote_output(_default_nodedir) + "]") or "")],
19 ["node-url", "u", None,
20 "Specify the URL of the Tahoe gateway node, such as 'http://127.0.0.1:3456'. "
21 "This overrides the URL found in the --node-directory ."],
22 ["dir-cap", None, None,
23 "Specify which dirnode URI should be used as the 'tahoe' alias."]
26 def postOptions(self):
27 if self['node-directory']:
28 self['node-directory'] = argv_to_abspath(self['node-directory'])
30 self['node-directory'] = _default_nodedir
32 # compute a node-url from the existing options, put in self['node-url']
34 if (not isinstance(self['node-url'], basestring)
35 or not NODEURL_RE.match(self['node-url'])):
36 msg = ("--node-url is required to be a string and look like "
37 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
39 raise usage.UsageError(msg)
41 node_url_file = os.path.join(self['node-directory'], "node.url")
42 self['node-url'] = open(node_url_file, "r").read().strip()
43 if self['node-url'][-1] != "/":
44 self['node-url'] += "/"
46 aliases = get_aliases(self['node-directory'])
48 aliases[DEFAULT_ALIAS] = self['dir-cap']
49 self.aliases = aliases # maps alias name to dircap
52 class MakeDirectoryOptions(VDriveOptions):
54 ("mutable-type", None, None, "Create a mutable directory in the given format. "
55 "Valid formats are SDMF and MDMF, case-insensitive."),
58 def parseArgs(self, where=""):
59 self.where = argv_to_unicode(where)
61 if self['mutable-type']:
62 if self['mutable-type'].lower() not in ("sdmf", "mdmf"):
63 raise usage.UsageError("%s is an invalid format" % self['mutable-type'])
64 self['mutable-type'] = self['mutable-type'].lower()
66 def getSynopsis(self):
67 return "Usage: %s mkdir [options] [REMOTE_DIR]" % (self.command_name,)
69 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
71 class AddAliasOptions(VDriveOptions):
72 def parseArgs(self, alias, cap):
73 self.alias = argv_to_unicode(alias)
74 if self.alias.endswith(u':'):
75 self.alias = self.alias[:-1]
78 def getSynopsis(self):
79 return "Usage: %s add-alias [options] ALIAS[:] DIRCAP" % (self.command_name,)
81 longdesc = """Add a new alias for an existing directory."""
83 class CreateAliasOptions(VDriveOptions):
84 def parseArgs(self, alias):
85 self.alias = argv_to_unicode(alias)
86 if self.alias.endswith(u':'):
87 self.alias = self.alias[:-1]
89 def getSynopsis(self):
90 return "Usage: %s create-alias [options] ALIAS[:]" % (self.command_name,)
92 longdesc = """Create a new directory and add an alias for it."""
94 class ListAliasesOptions(VDriveOptions):
95 def getSynopsis(self):
96 return "Usage: %s list-aliases [options]" % (self.command_name,)
98 longdesc = """Display a table of all configured aliases."""
100 class ListOptions(VDriveOptions):
102 ("long", "l", "Use long format: show file sizes, and timestamps."),
103 ("uri", "u", "Show file/directory URIs."),
104 ("readonly-uri", None, "Show read-only file/directory URIs."),
105 ("classify", "F", "Append '/' to directory names, and '*' to mutable."),
106 ("json", None, "Show the raw JSON output."),
108 def parseArgs(self, where=""):
109 self.where = argv_to_unicode(where)
112 List the contents of some portion of the grid.
114 When the -l or --long option is used, each line is shown in the
117 drwx <size> <date/time> <name in this directory>
119 where each of the letters on the left may be replaced by '-'.
120 If 'd' is present, it indicates that the object is a directory.
121 If the 'd' is replaced by a '?', the object type is unknown.
122 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
123 then the object is writeable through its link in this directory
124 (note that the link might be replaceable even if the object is
125 not writeable through the current link).
126 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
127 only to indicate that the contents of a directory can be listed.
129 Directories have no size, so their size field is shown as '-'.
130 Otherwise the size of the file, when known, is given in bytes.
131 The size of mutable files or unknown objects is shown as '?'.
133 The date/time shows when this link in the Tahoe filesystem was
137 class GetOptions(VDriveOptions):
138 def parseArgs(self, arg1, arg2=None):
139 # tahoe get FOO |less # write to stdout
140 # tahoe get tahoe:FOO |less # same
141 # tahoe get FOO bar # write to local file
142 # tahoe get tahoe:FOO bar # same
144 self.from_file = argv_to_unicode(arg1)
147 self.to_file = argv_to_unicode(arg2)
151 if self.to_file == "-":
154 def getSynopsis(self):
155 return "Usage: %s get [options] REMOTE_FILE LOCAL_FILE" % (self.command_name,)
158 Retrieve a file from the grid and write it to the local filesystem. If
159 LOCAL_FILE is omitted or '-', the contents of the file will be written to
162 def getUsage(self, width=None):
163 t = VDriveOptions.getUsage(self, width)
166 % tahoe get FOO |less # write to stdout
167 % tahoe get tahoe:FOO |less # same
168 % tahoe get FOO bar # write to local file
169 % tahoe get tahoe:FOO bar # same
173 class PutOptions(VDriveOptions):
175 ("mutable", "m", "Create a mutable file instead of an immutable one."),
178 ("mutable-type", None, None, "Create a mutable file in the given format (implies --mutable). "
179 "Valid formats are SDMF and MDMF, case-insensitive."),
182 def parseArgs(self, arg1=None, arg2=None):
185 if arg1 is not None and arg2 is not None:
186 self.from_file = argv_to_unicode(arg1)
187 self.to_file = argv_to_unicode(arg2)
188 elif arg1 is not None and arg2 is None:
189 self.from_file = argv_to_unicode(arg1) # might be "-"
192 self.from_file = None
194 if self.from_file == u"-":
195 self.from_file = None
197 if self['mutable-type']:
198 if self['mutable-type'].lower() not in ("sdmf", "mdmf"):
199 raise usage.UsageError("%s is an invalid format" % self['mutable-type'])
200 self['mutable-type'] = self['mutable-type'].lower()
202 if self['mutable-type']:
203 self['mutable'] = True
205 def getSynopsis(self):
206 return "Usage: %s put [options] LOCAL_FILE REMOTE_FILE" % (self.command_name,)
209 Put a file into the grid, copying its contents from the local filesystem.
210 If REMOTE_FILE is missing, upload the file but do not link it into a
211 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
212 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
213 with tahoe: unless otherwise specified."""
215 def getUsage(self, width=None):
216 t = VDriveOptions.getUsage(self, width)
219 % cat FILE | tahoe put # create unlinked file from stdin
220 % cat FILE | tahoe put - # same
221 % tahoe put bar # create unlinked file from local 'bar'
222 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
223 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
224 % tahoe put bar tahoe:FOO # same
225 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
229 class CpOptions(VDriveOptions):
231 ("recursive", "r", "Copy source directory recursively."),
232 ("verbose", "v", "Be noisy about what is happening."),
234 "When copying to local files, write out filecaps instead of actual "
235 "data (only useful for debugging and tree-comparison purposes)."),
238 def parseArgs(self, *args):
240 raise usage.UsageError("cp requires at least two arguments")
241 self.sources = map(argv_to_unicode, args[:-1])
242 self.destination = argv_to_unicode(args[-1])
244 def getSynopsis(self):
245 return "Usage: %s cp [options] FROM.. TO" % (self.command_name,)
248 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
249 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
250 files or non-file arguments. Directories will be copied recursively.
251 New Tahoe-side directories will be created when necessary. Assuming that
252 you have previously set up an alias 'home' with 'tahoe create-alias home',
253 here are some examples:
255 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
257 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
259 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
261 You can also use a dircap as either FROM or TO target:
263 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
265 This command still has some limitations: symlinks and special files
266 (device nodes, named pipes) are not handled very well. Arguments should
267 probably not have trailing slashes. 'tahoe cp' does not behave as much
268 like /bin/cp as you would wish, especially with respect to trailing
272 class UnlinkOptions(VDriveOptions):
273 def parseArgs(self, where):
274 self.where = argv_to_unicode(where)
276 def getSynopsis(self):
277 return "Usage: %s unlink [options] REMOTE_FILE" % (self.command_name,)
279 class RmOptions(UnlinkOptions):
280 def getSynopsis(self):
281 return "Usage: %s rm [options] REMOTE_FILE" % (self.command_name,)
283 class MvOptions(VDriveOptions):
284 def parseArgs(self, frompath, topath):
285 self.from_file = argv_to_unicode(frompath)
286 self.to_file = argv_to_unicode(topath)
288 def getSynopsis(self):
289 return "Usage: %s mv [options] FROM TO" % (self.command_name,)
292 Use 'tahoe mv' to move files that are already on the grid elsewhere on
293 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
295 If moving a remote file into a remote directory, you'll need to append a
296 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
297 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
299 Note that it is not possible to use this command to move local files to
300 the grid -- use 'tahoe cp' for that.
303 class LnOptions(VDriveOptions):
304 def parseArgs(self, frompath, topath):
305 self.from_file = argv_to_unicode(frompath)
306 self.to_file = argv_to_unicode(topath)
308 def getSynopsis(self):
309 return "Usage: %s ln [options] FROM_LINK TO_LINK" % (self.command_name,)
312 Use 'tahoe ln' to duplicate a link (directory entry) already on the grid
313 to elsewhere on the grid. For example 'tahoe ln alias:some_file
314 alias:new_file'. causes 'alias:new_file' to point to the same object that
315 'alias:some_file' points to.
317 (The argument order is the same as Unix ln. To remember the order, you
318 can think of this command as copying a link, rather than copying a file
319 as 'tahoe cp' does. Then the argument order is consistent with that of
322 When linking a remote file into a remote directory, you'll need to append
323 a '/' to the name of the remote directory, e.g. 'tahoe ln tahoe:file1
324 tahoe:dir/' (which is shorthand for 'tahoe ln tahoe:file1
325 tahoe:dir/file1'). If you forget the '/', e.g. 'tahoe ln tahoe:file1
326 tahoe:dir', the 'ln' command will refuse to overwrite the 'tahoe:dir'
327 directory, and will exit with an error.
329 Note that it is not possible to use this command to create links between
330 local and remote files.
333 class BackupConfigurationError(Exception):
336 class BackupOptions(VDriveOptions):
338 ("verbose", "v", "Be noisy about what is happening."),
339 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
342 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
343 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
344 '=meta-update', '=update', '.bzr', '.bzrignore',
345 '.bzrtags', '.hg', '.hgignore', '_darcs')
348 super(BackupOptions, self).__init__()
349 self['exclude'] = set()
351 def parseArgs(self, localdir, topath):
352 self.from_dir = argv_to_unicode(localdir)
353 self.to_dir = argv_to_unicode(topath)
355 def getSynopsis(self):
356 return "Usage: %s backup [options] FROM ALIAS:TO" % (self.command_name,)
358 def opt_exclude(self, pattern):
359 """Ignore files matching a glob pattern. You may give multiple
360 '--exclude' options."""
361 g = argv_to_unicode(pattern).strip()
363 exclude = self['exclude']
366 def opt_exclude_from(self, filepath):
367 """Ignore file matching glob patterns listed in file, one per
368 line. The file is assumed to be in the argv encoding."""
369 abs_filepath = argv_to_abspath(filepath)
371 exclude_file = file(abs_filepath)
373 raise BackupConfigurationError('Error opening exclude file %s.' % quote_output(abs_filepath))
375 for line in exclude_file:
376 self.opt_exclude(line)
380 def opt_exclude_vcs(self):
381 """Exclude files and directories used by following version control
382 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
384 for pattern in self.vcs_patterns:
385 self.opt_exclude(pattern)
387 def filter_listdir(self, listdir):
388 """Yields non-excluded childpaths in path."""
389 exclude = self['exclude']
390 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
391 for filename in listdir:
392 for regexp in exclude_regexps:
393 if regexp.match(filename):
399 Add a versioned backup of the local FROM directory to a timestamped
400 subdirectory of the TO/Archives directory on the grid, sharing as many
401 files and directories as possible with earlier backups. Create TO/Latest
402 as a reference to the latest backup. Behaves somewhat like 'rsync -a
403 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
404 TO/Archives/(new) TO/Latest'."""
406 class WebopenOptions(VDriveOptions):
408 ("info", "i", "Open the t=info page for the file"),
410 def parseArgs(self, where=''):
411 self.where = argv_to_unicode(where)
413 def getSynopsis(self):
414 return "Usage: %s webopen [options] [ALIAS:PATH]" % (self.command_name,)
416 longdesc = """Open a web browser to the contents of some file or
417 directory on the grid. When run without arguments, open the Welcome
420 class ManifestOptions(VDriveOptions):
422 ("storage-index", "s", "Only print storage index strings, not pathname+cap."),
423 ("verify-cap", None, "Only print verifycap, not pathname+cap."),
424 ("repair-cap", None, "Only print repaircap, not pathname+cap."),
425 ("raw", "r", "Display raw JSON data instead of parsed."),
427 def parseArgs(self, where=''):
428 self.where = argv_to_unicode(where)
430 def getSynopsis(self):
431 return "Usage: %s manifest [options] [ALIAS:PATH]" % (self.command_name,)
433 longdesc = """Print a list of all files and directories reachable from
434 the given starting point."""
436 class StatsOptions(VDriveOptions):
438 ("raw", "r", "Display raw JSON data instead of parsed"),
440 def parseArgs(self, where=''):
441 self.where = argv_to_unicode(where)
443 def getSynopsis(self):
444 return "Usage: %s stats [options] [ALIAS:PATH]" % (self.command_name,)
446 longdesc = """Print statistics about of all files and directories
447 reachable from the given starting point."""
449 class CheckOptions(VDriveOptions):
451 ("raw", None, "Display raw JSON data instead of parsed."),
452 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
453 ("repair", None, "Automatically repair any problems found."),
454 ("add-lease", None, "Add/renew lease on all shares."),
456 def parseArgs(self, where=''):
457 self.where = argv_to_unicode(where)
459 def getSynopsis(self):
460 return "Usage: %s check [options] [ALIAS:PATH]" % (self.command_name,)
463 Check a single file or directory: count how many shares are available and
464 verify their hashes. Optionally repair the file if any problems were
467 class DeepCheckOptions(VDriveOptions):
469 ("raw", None, "Display raw JSON data instead of parsed."),
470 ("verify", None, "Verify all hashes, instead of merely querying share presence."),
471 ("repair", None, "Automatically repair any problems found."),
472 ("add-lease", None, "Add/renew lease on all shares."),
473 ("verbose", "v", "Be noisy about what is happening."),
475 def parseArgs(self, where=''):
476 self.where = argv_to_unicode(where)
478 def getSynopsis(self):
479 return "Usage: %s deep-check [options] [ALIAS:PATH]" % (self.command_name,)
482 Check all files and directories reachable from the given starting point
483 (which must be a directory), like 'tahoe check' but for multiple files.
484 Optionally repair any problems found."""
487 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
488 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
489 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
490 ["list-aliases", None, ListAliasesOptions, "List all alias caps."],
491 ["ls", None, ListOptions, "List a directory."],
492 ["get", None, GetOptions, "Retrieve a file from the grid."],
493 ["put", None, PutOptions, "Upload a file into the grid."],
494 ["cp", None, CpOptions, "Copy one or more files or directories."],
495 ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."],
496 ["rm", None, RmOptions, "Unlink a file or directory on the grid (same as unlink)."],
497 ["mv", None, MvOptions, "Move a file within the grid."],
498 ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
499 ["backup", None, BackupOptions, "Make target dir look like local dir."],
500 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
501 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
502 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
503 ["check", None, CheckOptions, "Check a single file or directory."],
504 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
508 from allmydata.scripts import tahoe_mkdir
509 rc = tahoe_mkdir.mkdir(options)
512 def add_alias(options):
513 from allmydata.scripts import tahoe_add_alias
514 rc = tahoe_add_alias.add_alias(options)
517 def create_alias(options):
518 from allmydata.scripts import tahoe_add_alias
519 rc = tahoe_add_alias.create_alias(options)
522 def list_aliases(options):
523 from allmydata.scripts import tahoe_add_alias
524 rc = tahoe_add_alias.list_aliases(options)
528 from allmydata.scripts import tahoe_ls
529 rc = tahoe_ls.list(options)
533 from allmydata.scripts import tahoe_get
534 rc = tahoe_get.get(options)
536 if options.to_file is None:
537 # be quiet, since the file being written to stdout should be
538 # proof enough that it worked, unless the user is unlucky
539 # enough to have picked an empty file
542 print >>options.stderr, "%s retrieved and written to %s" % \
543 (options.from_file, options.to_file)
547 from allmydata.scripts import tahoe_put
548 rc = tahoe_put.put(options)
552 from allmydata.scripts import tahoe_cp
553 rc = tahoe_cp.copy(options)
556 def unlink(options, command="unlink"):
557 from allmydata.scripts import tahoe_unlink
558 rc = tahoe_unlink.unlink(options, command=command)
562 return unlink(options, command="rm")
565 from allmydata.scripts import tahoe_mv
566 rc = tahoe_mv.mv(options, mode="move")
570 from allmydata.scripts import tahoe_mv
571 rc = tahoe_mv.mv(options, mode="link")
575 from allmydata.scripts import tahoe_backup
576 rc = tahoe_backup.backup(options)
579 def webopen(options, opener=None):
580 from allmydata.scripts import tahoe_webopen
581 rc = tahoe_webopen.webopen(options, opener=opener)
584 def manifest(options):
585 from allmydata.scripts import tahoe_manifest
586 rc = tahoe_manifest.manifest(options)
590 from allmydata.scripts import tahoe_manifest
591 rc = tahoe_manifest.stats(options)
595 from allmydata.scripts import tahoe_check
596 rc = tahoe_check.check(options)
599 def deepcheck(options):
600 from allmydata.scripts import tahoe_check
601 rc = tahoe_check.deepcheck(options)
606 "add-alias": add_alias,
607 "create-alias": create_alias,
608 "list-aliases": list_aliases,
619 "manifest": manifest,
622 "deep-check": deepcheck,