1 import os.path, re, sys, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import BaseOptions, get_aliases
4 from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath
6 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
8 class VDriveOptions(BaseOptions, usage.Options):
10 ["node-directory", "d", "~/.tahoe",
11 "Look here to find out which Tahoe node should be used for all "
12 "operations. The directory should either contain a full Tahoe node, "
13 "or a file named node.url which points to some other Tahoe node. "
14 "It should also contain a file named private/aliases which contains "
15 "the mapping from alias name to root dirnode URI."
17 ["node-url", "u", None,
18 "URL of the tahoe node to use, a URL like \"http://127.0.0.1:3456\". "
19 "This overrides the URL found in the --node-directory ."],
20 ["dir-cap", None, None,
21 "Which dirnode URI should be used as the 'tahoe' alias."]
24 def postOptions(self):
25 # compute a node-url from the existing options, put in self['node-url']
26 if self['node-directory']:
27 if sys.platform == 'win32' and self['node-directory'] == '~/.tahoe':
28 from allmydata.windows import registry
29 self['node-directory'] = registry.get_base_dir_path()
31 self['node-directory'] = argv_to_abspath(self['node-directory'])
33 if (not isinstance(self['node-url'], basestring)
34 or not NODEURL_RE.match(self['node-url'])):
35 msg = ("--node-url is required to be a string and look like "
36 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
38 raise usage.UsageError(msg)
40 node_url_file = os.path.join(self['node-directory'], "node.url")
41 self['node-url'] = open(node_url_file, "r").read().strip()
42 if self['node-url'][-1] != "/":
43 self['node-url'] += "/"
45 aliases = get_aliases(self['node-directory'])
47 aliases["tahoe"] = self['dir-cap']
48 self.aliases = aliases # maps alias name to dircap
51 class MakeDirectoryOptions(VDriveOptions):
52 def parseArgs(self, where=""):
53 self.where = argv_to_unicode(where)
54 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
56 class AddAliasOptions(VDriveOptions):
57 def parseArgs(self, alias, cap):
58 self.alias = argv_to_unicode(alias)
61 def getSynopsis(self):
62 return "%s add-alias ALIAS DIRCAP" % (os.path.basename(sys.argv[0]),)
64 longdesc = """Add a new alias for an existing directory."""
66 class CreateAliasOptions(VDriveOptions):
67 def parseArgs(self, alias):
68 self.alias = argv_to_unicode(alias)
70 def getSynopsis(self):
71 return "%s create-alias ALIAS" % (os.path.basename(sys.argv[0]),)
73 longdesc = """Create a new directory and add an alias for it."""
75 class ListAliasOptions(VDriveOptions):
76 longdesc = """Display a table of all configured aliases."""
78 class ListOptions(VDriveOptions):
80 ("long", "l", "Use long format: show file sizes, and timestamps"),
81 ("uri", "u", "Show file/directory URIs"),
82 ("readonly-uri", None, "Show readonly file/directory URIs"),
83 ("classify", "F", "Append '/' to directory names, and '*' to mutable"),
84 ("json", None, "Show the raw JSON output"),
86 def parseArgs(self, where=""):
87 self.where = argv_to_unicode(where)
90 List the contents of some portion of the grid.
92 When the -l or --long option is used, each line is shown in the
95 drwx <size> <date/time> <name in this directory>
97 where each of the letters on the left may be replaced by '-'.
98 If 'd' is present, it indicates that the object is a directory.
99 If the 'd' is replaced by a '?', the object type is unknown.
100 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
101 then the object is writeable through its link in this directory
102 (note that the link might be replaceable even if the object is
103 not writeable through the current link).
104 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
105 only to indicate that the contents of a directory can be listed.
107 Directories have no size, so their size field is shown as '-'.
108 Otherwise the size of the file, when known, is given in bytes.
109 The size of mutable files or unknown objects is shown as '?'.
111 The date/time shows when this link in the Tahoe filesystem was
115 class GetOptions(VDriveOptions):
116 def parseArgs(self, arg1, arg2=None):
117 # tahoe get FOO |less # write to stdout
118 # tahoe get tahoe:FOO |less # same
119 # tahoe get FOO bar # write to local file
120 # tahoe get tahoe:FOO bar # same
122 self.from_file = argv_to_unicode(arg1)
125 self.to_file = argv_to_unicode(arg2)
129 if self.to_file == "-":
132 def getSynopsis(self):
133 return "%s get REMOTE_FILE LOCAL_FILE" % (os.path.basename(sys.argv[0]),)
136 Retrieve a file from the grid and write it to the local filesystem. If
137 LOCAL_FILE is omitted or '-', the contents of the file will be written to
140 def getUsage(self, width=None):
141 t = VDriveOptions.getUsage(self, width)
144 % tahoe get FOO |less # write to stdout
145 % tahoe get tahoe:FOO |less # same
146 % tahoe get FOO bar # write to local file
147 % tahoe get tahoe:FOO bar # same
151 class PutOptions(VDriveOptions):
153 ("mutable", "m", "Create a mutable file instead of an immutable one."),
156 def parseArgs(self, arg1=None, arg2=None):
159 if arg1 is not None and arg2 is not None:
160 self.from_file = argv_to_unicode(arg1)
161 self.to_file = argv_to_unicode(arg2)
162 elif arg1 is not None and arg2 is None:
163 self.from_file = argv_to_unicode(arg1) # might be "-"
166 self.from_file = None
168 if self.from_file == u"-":
169 self.from_file = None
171 def getSynopsis(self):
172 return "%s put LOCAL_FILE REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
175 Put a file into the grid, copying its contents from the local filesystem.
176 If REMOTE_FILE is missing, upload the file but do not link it into a
177 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
178 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
179 with tahoe: unless otherwise specified."""
181 def getUsage(self, width=None):
182 t = VDriveOptions.getUsage(self, width)
185 % cat FILE | tahoe put # create unlinked file from stdin
186 % cat FILE | tahoe - # same
187 % tahoe put bar # create unlinked file from local 'bar'
188 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
189 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
190 % tahoe put bar tahoe:FOO # same
191 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
195 class CpOptions(VDriveOptions):
197 ("recursive", "r", "Copy source directory recursively."),
198 ("verbose", "v", "Be noisy about what is happening."),
200 "When copying to local files, write out filecaps instead of actual "
201 "data (only useful for debugging and tree-comparison purposes)."),
203 def parseArgs(self, *args):
205 raise usage.UsageError("cp requires at least two arguments")
206 self.sources = map(argv_to_unicode, args[:-1])
207 self.destination = argv_to_unicode(args[-1])
208 def getSynopsis(self):
209 return "Usage: tahoe [options] cp FROM.. TO"
211 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
212 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
213 files or non-file arguments. Directories will be copied recursively.
214 New Tahoe-side directories will be created when necessary. Assuming that
215 you have previously set up an alias 'home' with 'tahoe create-alias home',
216 here are some examples:
218 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
220 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
222 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
224 You can also use a dircap as either FROM or TO target:
226 tahoe cp URI:DIR2-RO:ixqhc4kdbjxc7o65xjnveoewym:5x6lwoxghrd5rxhwunzavft2qygfkt27oj3fbxlq4c6p45z5uneq/blog.html ./ # copy Zooko's wiki page to a local file
228 This command still has some limitations: symlinks, special files (device
229 nodes, named pipes), and non-ASCII filenames are not handled very well.
230 Arguments should probably not have trailing slashes. 'tahoe cp' does not
231 behave as much like /bin/cp as you would wish, especially with respect to
235 class RmOptions(VDriveOptions):
236 def parseArgs(self, where):
237 self.where = argv_to_unicode(where)
239 def getSynopsis(self):
240 return "%s rm REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
242 class MvOptions(VDriveOptions):
243 def parseArgs(self, frompath, topath):
244 self.from_file = argv_to_unicode(frompath)
245 self.to_file = argv_to_unicode(topath)
247 def getSynopsis(self):
248 return "%s mv FROM TO" % (os.path.basename(sys.argv[0]),)
250 Use 'tahoe mv' to move files that are already on the grid elsewhere on
251 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
253 If moving a remote file into a remote directory, you'll need to append a
254 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
255 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
257 Note that it is not possible to use this command to move local files to
258 the grid -- use 'tahoe cp' for that.
261 class LnOptions(VDriveOptions):
262 def parseArgs(self, frompath, topath):
263 self.from_file = argv_to_unicode(frompath)
264 self.to_file = argv_to_unicode(topath)
266 def getSynopsis(self):
267 return "%s ln FROM TO" % (os.path.basename(sys.argv[0]),)
269 class BackupConfigurationError(Exception):
272 class BackupOptions(VDriveOptions):
274 ("verbose", "v", "Be noisy about what is happening."),
275 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
278 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
279 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
280 '=meta-update', '=update', '.bzr', '.bzrignore',
281 '.bzrtags', '.hg', '.hgignore', '_darcs')
284 super(BackupOptions, self).__init__()
285 self['exclude'] = set()
287 def parseArgs(self, localdir, topath):
288 self.from_dir = argv_to_unicode(localdir)
289 self.to_dir = argv_to_unicode(topath)
291 def getSynopsis(Self):
292 return "%s backup FROM ALIAS:TO" % os.path.basename(sys.argv[0])
294 def opt_exclude(self, pattern):
295 """Ignore files matching a glob pattern. You may give multiple
296 '--exclude' options."""
297 g = argv_to_unicode(pattern).strip()
299 exclude = self['exclude']
302 def opt_exclude_from(self, filepath):
303 """Ignore file matching glob patterns listed in file, one per
304 line. The file is assumed to be in the argv encoding."""
306 exclude_file = file(filepath)
308 raise BackupConfigurationError('Error opening exclude file %r.' % filepath)
310 for line in exclude_file:
311 self.opt_exclude(line)
315 def opt_exclude_vcs(self):
316 """Exclude files and directories used by following version control
317 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
319 for pattern in self.vcs_patterns:
320 self.opt_exclude(pattern)
322 def filter_listdir(self, listdir):
323 """Yields non-excluded childpaths in path."""
324 exclude = self['exclude']
325 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
326 for filename in listdir:
327 for regexp in exclude_regexps:
328 if regexp.match(filename):
334 Add a versioned backup of the local FROM directory to a timestamped
335 subdirectory of the TO/Archives directory on the grid, sharing as many
336 files and directories as possible with earlier backups. Create TO/Latest
337 as a reference to the latest backup. Behaves somewhat like 'rsync -a
338 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
339 TO/Archives/(new) TO/Latest'."""
341 class WebopenOptions(VDriveOptions):
343 ("info", "i", "Open the t=info page for the file"),
345 def parseArgs(self, where=''):
346 self.where = argv_to_unicode(where)
348 def getSynopsis(self):
349 return "%s webopen [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
351 longdesc = """Open a web browser to the contents of some file or
352 directory on the grid. When run without arguments, open the Welcome
355 class ManifestOptions(VDriveOptions):
357 ("storage-index", "s", "Only print storage index strings, not pathname+cap"),
358 ("verify-cap", None, "Only print verifycap, not pathname+cap"),
359 ("repair-cap", None, "Only print repaircap, not pathname+cap"),
360 ("raw", "r", "Display raw JSON data instead of parsed"),
362 def parseArgs(self, where=''):
363 self.where = argv_to_unicode(where)
365 def getSynopsis(self):
366 return "%s manifest [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
368 longdesc = """Print a list of all files and directories reachable from
369 the given starting point."""
371 class StatsOptions(VDriveOptions):
373 ("raw", "r", "Display raw JSON data instead of parsed"),
375 def parseArgs(self, where=''):
376 self.where = argv_to_unicode(where)
378 def getSynopsis(self):
379 return "%s stats [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
381 longdesc = """Print statistics about of all files and directories
382 reachable from the given starting point."""
384 class CheckOptions(VDriveOptions):
386 ("raw", None, "Display raw JSON data instead of parsed"),
387 ("verify", None, "Verify all hashes, instead of merely querying share presence"),
388 ("repair", None, "Automatically repair any problems found"),
389 ("add-lease", None, "Add/renew lease on all shares"),
391 def parseArgs(self, where=''):
392 self.where = argv_to_unicode(where)
394 def getSynopsis(self):
395 return "%s check [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
398 Check a single file or directory: count how many shares are available and
399 verify their hashes. Optionally repair the file if any problems were
402 class DeepCheckOptions(VDriveOptions):
404 ("raw", None, "Display raw JSON data instead of parsed"),
405 ("verify", None, "Verify all hashes, instead of merely querying share presence"),
406 ("repair", None, "Automatically repair any problems found"),
407 ("add-lease", None, "Add/renew lease on all shares"),
408 ("verbose", "v", "Be noisy about what is happening."),
410 def parseArgs(self, where=''):
411 self.where = argv_to_unicode(where)
413 def getSynopsis(self):
414 return "%s deep-check [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
417 Check all files and directories reachable from the given starting point
418 (which must be a directory), like 'tahoe check' but for multiple files.
419 Optionally repair any problems found."""
422 ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
423 ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
424 ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
425 ["list-aliases", None, ListAliasOptions, "List all alias caps."],
426 ["ls", None, ListOptions, "List a directory."],
427 ["get", None, GetOptions, "Retrieve a file from the grid."],
428 ["put", None, PutOptions, "Upload a file into the grid."],
429 ["cp", None, CpOptions, "Copy one or more files."],
430 ["rm", None, RmOptions, "Unlink a file or directory on the grid."],
431 ["unlink", None, RmOptions, "Unlink a file or directory on the grid (same as rm)."],
432 ["mv", None, MvOptions, "Move a file within the grid."],
433 ["ln", None, LnOptions, "Make an additional link to an existing file."],
434 ["backup", None, BackupOptions, "Make target dir look like local dir."],
435 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
436 ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
437 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
438 ["check", None, CheckOptions, "Check a single file or directory."],
439 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
443 from allmydata.scripts import tahoe_mkdir
444 rc = tahoe_mkdir.mkdir(options)
447 def add_alias(options):
448 from allmydata.scripts import tahoe_add_alias
449 rc = tahoe_add_alias.add_alias(options)
452 def create_alias(options):
453 from allmydata.scripts import tahoe_add_alias
454 rc = tahoe_add_alias.create_alias(options)
457 def list_aliases(options):
458 from allmydata.scripts import tahoe_add_alias
459 rc = tahoe_add_alias.list_aliases(options)
463 from allmydata.scripts import tahoe_ls
464 rc = tahoe_ls.list(options)
468 from allmydata.scripts import tahoe_get
469 rc = tahoe_get.get(options)
471 if options.to_file is None:
472 # be quiet, since the file being written to stdout should be
473 # proof enough that it worked, unless the user is unlucky
474 # enough to have picked an empty file
477 print >>options.stderr, "%s retrieved and written to %s" % \
478 (options.from_file, options.to_file)
482 from allmydata.scripts import tahoe_put
483 rc = tahoe_put.put(options)
487 from allmydata.scripts import tahoe_cp
488 rc = tahoe_cp.copy(options)
492 from allmydata.scripts import tahoe_rm
493 rc = tahoe_rm.rm(options)
497 from allmydata.scripts import tahoe_mv
498 rc = tahoe_mv.mv(options, mode="move")
502 from allmydata.scripts import tahoe_mv
503 rc = tahoe_mv.mv(options, mode="link")
507 from allmydata.scripts import tahoe_backup
508 rc = tahoe_backup.backup(options)
511 def webopen(options, opener=None):
512 from allmydata.scripts import tahoe_webopen
513 rc = tahoe_webopen.webopen(options, opener=opener)
516 def manifest(options):
517 from allmydata.scripts import tahoe_manifest
518 rc = tahoe_manifest.manifest(options)
522 from allmydata.scripts import tahoe_manifest
523 rc = tahoe_manifest.stats(options)
527 from allmydata.scripts import tahoe_check
528 rc = tahoe_check.check(options)
531 def deepcheck(options):
532 from allmydata.scripts import tahoe_check
533 rc = tahoe_check.deepcheck(options)
538 "add-alias": add_alias,
539 "create-alias": create_alias,
540 "list-aliases": list_aliases,
551 "manifest": manifest,
554 "deep-check": deepcheck,