1 import os.path, re, sys, fnmatch
2 from twisted.python import usage
3 from allmydata.scripts.common import BaseOptions, get_aliases
5 NODEURL_RE=re.compile("http://([^:]*)(:([1-9][0-9]*))?")
7 class VDriveOptions(BaseOptions, usage.Options):
9 ["node-directory", "d", "~/.tahoe",
10 "Look here to find out which Tahoe node should be used for all "
11 "operations. The directory should either contain a full Tahoe node, "
12 "or a file named node.url which points to some other Tahoe node. "
13 "It should also contain a file named private/aliases which contains "
14 "the mapping from alias name to root dirnode URI."
16 ["node-url", "u", None,
17 "URL of the tahoe node to use, a URL like \"http://127.0.0.1:3456\". "
18 "This overrides the URL found in the --node-directory ."],
19 ["dir-cap", None, None,
20 "Which dirnode URI should be used as the 'tahoe' alias."]
23 def postOptions(self):
24 # compute a node-url from the existing options, put in self['node-url']
25 if self['node-directory']:
26 if sys.platform == 'win32' and self['node-directory'] == '~/.tahoe':
27 from allmydata.windows import registry
28 self['node-directory'] = registry.get_base_dir_path()
30 self['node-directory'] = os.path.expanduser(self['node-directory'])
32 if (not isinstance(self['node-url'], basestring)
33 or not NODEURL_RE.match(self['node-url'])):
34 msg = ("--node-url is required to be a string and look like "
35 "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
37 raise usage.UsageError(msg)
39 node_url_file = os.path.join(self['node-directory'], "node.url")
40 self['node-url'] = open(node_url_file, "r").read().strip()
41 if self['node-url'][-1] != "/":
42 self['node-url'] += "/"
44 aliases = get_aliases(self['node-directory'])
46 aliases["tahoe"] = self['dir-cap']
47 self.aliases = aliases # maps alias name to dircap
50 class MakeDirectoryOptions(VDriveOptions):
51 def parseArgs(self, where=""):
53 longdesc = """Create a new directory, either unlinked or as a subdirectory."""
55 class AddAliasOptions(VDriveOptions):
56 def parseArgs(self, alias, cap):
60 def getSynopsis(self):
61 return "%s add-alias ALIAS DIRCAP" % (os.path.basename(sys.argv[0]),)
63 longdesc = """Add a new alias for an existing directory."""
65 class CreateAliasOptions(VDriveOptions):
66 def parseArgs(self, alias):
69 def getSynopsis(self):
70 return "%s create-alias ALIAS" % (os.path.basename(sys.argv[0]),)
72 longdesc = """Create a new directory and add an alias for it."""
74 class ListAliasOptions(VDriveOptions):
75 longdesc = """Display a table of all configured aliases."""
77 class ListOptions(VDriveOptions):
79 ("long", "l", "Use long format: show file sizes, and timestamps"),
80 ("uri", "u", "Show file/directory URIs"),
81 ("readonly-uri", None, "Show readonly file/directory URIs"),
82 ("classify", "F", "Append '/' to directory names, and '*' to mutable"),
83 ("json", None, "Show the raw JSON output"),
85 def parseArgs(self, where=""):
89 List the contents of some portion of the grid.
91 When the -l or --long option is used, each line is shown in the
94 drwx <size> <date/time> <name in this directory>
96 where each of the letters on the left may be replaced by '-'.
97 If 'd' is present, it indicates that the object is a directory.
98 If the 'd' is replaced by a '?', the object type is unknown.
99 'rwx' is a Unix-like permissions mask: if the mask includes 'w',
100 then the object is writeable through its link in this directory
101 (note that the link might be replaceable even if the object is
102 not writeable through the current link).
103 The 'x' is a legacy of Unix filesystems. In Tahoe it is used
104 only to indicate that the contents of a directory can be listed.
106 Directories have no size, so their size field is shown as '-'.
107 Otherwise the size of the file, when known, is given in bytes.
108 The size of mutable files or unknown objects is shown as '?'.
110 The date/time shows when this link in the Tahoe filesystem was
114 class GetOptions(VDriveOptions):
115 def parseArgs(self, arg1, arg2=None):
116 # tahoe get FOO |less # write to stdout
117 # tahoe get tahoe:FOO |less # same
118 # tahoe get FOO bar # write to local file
119 # tahoe get tahoe:FOO bar # same
121 self.from_file = arg1
123 if self.to_file == "-":
126 def getSynopsis(self):
127 return "%s get REMOTE_FILE LOCAL_FILE" % (os.path.basename(sys.argv[0]),)
130 Retrieve a file from the grid and write it to the local filesystem. If
131 LOCAL_FILE is omitted or '-', the contents of the file will be written to
134 def getUsage(self, width=None):
135 t = VDriveOptions.getUsage(self, width)
138 % tahoe get FOO |less # write to stdout
139 % tahoe get tahoe:FOO |less # same
140 % tahoe get FOO bar # write to local file
141 % tahoe get tahoe:FOO bar # same
145 class PutOptions(VDriveOptions):
147 ("mutable", "m", "Create a mutable file instead of an immutable one."),
150 def parseArgs(self, arg1=None, arg2=None):
153 if arg1 is not None and arg2 is not None:
154 self.from_file = arg1
156 elif arg1 is not None and arg2 is None:
157 self.from_file = arg1 # might be "-"
160 self.from_file = None
162 if self.from_file == "-":
163 self.from_file = None
165 def getSynopsis(self):
166 return "%s put LOCAL_FILE REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
169 Put a file into the grid, copying its contents from the local filesystem.
170 If REMOTE_FILE is missing, upload the file but do not link it into a
171 directory; also print the new filecap to stdout. If LOCAL_FILE is missing
172 or '-', data will be copied from stdin. REMOTE_FILE is assumed to start
173 with tahoe: unless otherwise specified."""
175 def getUsage(self, width=None):
176 t = VDriveOptions.getUsage(self, width)
179 % cat FILE | tahoe put # create unlinked file from stdin
180 % cat FILE | tahoe - # same
181 % tahoe put bar # create unlinked file from local 'bar'
182 % cat FILE | tahoe put - FOO # create tahoe:FOO from stdin
183 % tahoe put bar FOO # copy local 'bar' to tahoe:FOO
184 % tahoe put bar tahoe:FOO # same
185 % tahoe put bar MUTABLE-FILE-WRITECAP # modify the mutable file in-place
189 class CpOptions(VDriveOptions):
191 ("recursive", "r", "Copy source directory recursively."),
192 ("verbose", "v", "Be noisy about what is happening."),
194 "When copying to local files, write out filecaps instead of actual "
195 "data (only useful for debugging and tree-comparison purposes)."),
197 def parseArgs(self, *args):
199 raise usage.UsageError("cp requires at least two arguments")
200 self.sources = args[:-1]
201 self.destination = args[-1]
202 def getSynopsis(self):
203 return "Usage: tahoe [options] cp FROM.. TO"
205 Use 'tahoe cp' to copy files between a local filesystem and a Tahoe grid.
206 Any FROM/TO arguments that begin with an alias indicate Tahoe-side
207 files or non-file arguments. Directories will be copied recursively.
208 New Tahoe-side directories will be created when necessary. Assuming that
209 you have previously set up an alias 'home' with 'tahoe create-alias home',
210 here are some examples:
212 tahoe cp ~/foo.txt home: # creates tahoe-side home:foo.txt
214 tahoe cp ~/foo.txt /tmp/bar.txt home: # copies two files to home:
216 tahoe cp ~/Pictures home:stuff/my-pictures # copies directory recursively
218 You can also use a dircap as either FROM or TO target:
220 tahoe cp URI:DIR2-RO:j74uhg25nwdpjpacl6rkat2yhm:kav7ijeft5h7r7rxdp5bgtlt3viv32yabqajkrdykozia5544jqa/wiki.html ./ # copy Zooko's wiki page to a local file
222 This command still has some limitations: symlinks, special files (device
223 nodes, named pipes), and non-ASCII filenames are not handled very well.
224 Arguments should probably not have trailing slashes. 'tahoe cp' does not
225 behave as much like /bin/cp as you would wish, especially with respect to
229 class RmOptions(VDriveOptions):
230 def parseArgs(self, where):
233 def getSynopsis(self):
234 return "%s rm REMOTE_FILE" % (os.path.basename(sys.argv[0]),)
236 class MvOptions(VDriveOptions):
237 def parseArgs(self, frompath, topath):
238 self.from_file = frompath
239 self.to_file = topath
241 def getSynopsis(self):
242 return "%s mv FROM TO" % (os.path.basename(sys.argv[0]),)
244 Use 'tahoe mv' to move files that are already on the grid elsewhere on
245 the grid, e.g., 'tahoe mv alias:some_file alias:new_file'.
247 If moving a remote file into a remote directory, you'll need to append a
248 '/' to the name of the remote directory, e.g., 'tahoe mv tahoe:file1
249 tahoe:dir/', not 'tahoe mv tahoe:file1 tahoe:dir'.
251 Note that it is not possible to use this command to move local files to
252 the grid -- use 'tahoe cp' for that.
255 class LnOptions(VDriveOptions):
256 def parseArgs(self, frompath, topath):
257 self.from_file = frompath
258 self.to_file = topath
260 def getSynopsis(self):
261 return "%s ln FROM TO" % (os.path.basename(sys.argv[0]),)
263 class BackupConfigurationError(Exception):
266 class BackupOptions(VDriveOptions):
268 ("verbose", "v", "Be noisy about what is happening."),
269 ("ignore-timestamps", None, "Do not use backupdb timestamps to decide whether a local file is unchanged."),
272 vcs_patterns = ('CVS', 'RCS', 'SCCS', '.git', '.gitignore', '.cvsignore',
273 '.svn', '.arch-ids','{arch}', '=RELEASE-ID',
274 '=meta-update', '=update', '.bzr', '.bzrignore',
275 '.bzrtags', '.hg', '.hgignore', '_darcs')
278 super(BackupOptions, self).__init__()
279 self['exclude'] = set()
281 def parseArgs(self, localdir, topath):
282 self.from_dir = localdir
285 def getSynopsis(Self):
286 return "%s backup FROM ALIAS:TO" % os.path.basename(sys.argv[0])
288 def opt_exclude(self, pattern):
289 """Ignore files matching a glob pattern. You may give multiple
290 '--exclude' options."""
293 exclude = self['exclude']
296 def opt_exclude_from(self, filepath):
297 """Ignore file matching glob patterns listed in file, one per
300 exclude_file = file(filepath)
302 raise BackupConfigurationError('Error opening exclude file %r.' % filepath)
304 for line in exclude_file:
305 self.opt_exclude(line)
309 def opt_exclude_vcs(self):
310 """Exclude files and directories used by following version control
311 systems: CVS, RCS, SCCS, Git, SVN, Arch, Bazaar(bzr), Mercurial,
313 for pattern in self.vcs_patterns:
314 self.opt_exclude(pattern)
316 def filter_listdir(self, listdir):
317 """Yields non-excluded childpaths in path."""
318 exclude = self['exclude']
319 exclude_regexps = [re.compile(fnmatch.translate(pat)) for pat in exclude]
320 for filename in listdir:
321 for regexp in exclude_regexps:
322 if regexp.match(filename):
328 Add a versioned backup of the local FROM directory to a timestamped
329 subdirectory of the TO/Archives directory on the grid, sharing as many
330 files and directories as possible with earlier backups. Create TO/Latest
331 as a reference to the latest backup. Behaves somewhat like 'rsync -a
332 --link-dest=TO/Archives/(previous) FROM TO/Archives/(new); ln -sf
333 TO/Archives/(new) TO/Latest'."""
335 class WebopenOptions(VDriveOptions):
336 def parseArgs(self, where=''):
339 def getSynopsis(self):
340 return "%s webopen [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
342 longdesc = """Open a web browser to the contents of some file or
343 directory on the grid."""
345 class ManifestOptions(VDriveOptions):
347 ("storage-index", "s", "Only print storage index strings, not pathname+cap"),
348 ("verify-cap", None, "Only print verifycap, not pathname+cap"),
349 ("repair-cap", None, "Only print repaircap, not pathname+cap"),
350 ("raw", "r", "Display raw JSON data instead of parsed"),
352 def parseArgs(self, where=''):
355 def getSynopsis(self):
356 return "%s manifest [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
358 longdesc = """Print a list of all files and directories reachable from
359 the given starting point."""
361 class StatsOptions(VDriveOptions):
363 ("raw", "r", "Display raw JSON data instead of parsed"),
365 def parseArgs(self, where=''):
368 def getSynopsis(self):
369 return "%s stats [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
371 longdesc = """Print statistics about of all files and directories
372 reachable from the given starting point."""
374 class CheckOptions(VDriveOptions):
376 ("raw", None, "Display raw JSON data instead of parsed"),
377 ("verify", None, "Verify all hashes, instead of merely querying share presence"),
378 ("repair", None, "Automatically repair any problems found"),
379 ("add-lease", None, "Add/renew lease on all shares"),
381 def parseArgs(self, where=''):
384 def getSynopsis(self):
385 return "%s check [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
388 Check a single file or directory: count how many shares are available and
389 verify their hashes. Optionally repair the file if any problems were
392 class DeepCheckOptions(VDriveOptions):
394 ("raw", None, "Display raw JSON data instead of parsed"),
395 ("verify", None, "Verify all hashes, instead of merely querying share presence"),
396 ("repair", None, "Automatically repair any problems found"),
397 ("add-lease", None, "Add/renew lease on all shares"),
398 ("verbose", "v", "Be noisy about what is happening."),
400 def parseArgs(self, where=''):
403 def getSynopsis(self):
404 return "%s deep-check [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
407 Check all files and directories reachable from the given starting point
408 (which must be a directory), like 'tahoe check' but for multiple files.
409 Optionally repair any problems found."""
412 ["mkdir", None, MakeDirectoryOptions, "Create a new directory"],
413 ["add-alias", None, AddAliasOptions, "Add a new alias cap"],
414 ["create-alias", None, CreateAliasOptions, "Create a new alias cap"],
415 ["list-aliases", None, ListAliasOptions, "List all alias caps"],
416 ["ls", None, ListOptions, "List a directory"],
417 ["get", None, GetOptions, "Retrieve a file from the grid."],
418 ["put", None, PutOptions, "Upload a file into the grid."],
419 ["cp", None, CpOptions, "Copy one or more files."],
420 ["rm", None, RmOptions, "Unlink a file or directory on the grid."],
421 ["mv", None, MvOptions, "Move a file within the grid."],
422 ["ln", None, LnOptions, "Make an additional link to an existing file."],
423 ["backup", None, BackupOptions, "Make target dir look like local dir."],
424 ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
425 ["manifest", None, ManifestOptions, "List all files/directories in a subtree"],
426 ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree"],
427 ["check", None, CheckOptions, "Check a single file or directory"],
428 ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point"],
432 from allmydata.scripts import tahoe_mkdir
433 rc = tahoe_mkdir.mkdir(options)
436 def add_alias(options):
437 from allmydata.scripts import tahoe_add_alias
438 rc = tahoe_add_alias.add_alias(options)
441 def create_alias(options):
442 from allmydata.scripts import tahoe_add_alias
443 rc = tahoe_add_alias.create_alias(options)
446 def list_aliases(options):
447 from allmydata.scripts import tahoe_add_alias
448 rc = tahoe_add_alias.list_aliases(options)
452 from allmydata.scripts import tahoe_ls
453 rc = tahoe_ls.list(options)
457 from allmydata.scripts import tahoe_get
458 rc = tahoe_get.get(options)
460 if options.to_file is None:
461 # be quiet, since the file being written to stdout should be
462 # proof enough that it worked, unless the user is unlucky
463 # enough to have picked an empty file
466 print >>options.stderr, "%s retrieved and written to %s" % \
467 (options.from_file, options.to_file)
471 from allmydata.scripts import tahoe_put
472 rc = tahoe_put.put(options)
476 from allmydata.scripts import tahoe_cp
477 rc = tahoe_cp.copy(options)
481 from allmydata.scripts import tahoe_rm
482 rc = tahoe_rm.rm(options)
486 from allmydata.scripts import tahoe_mv
487 rc = tahoe_mv.mv(options, mode="move")
491 from allmydata.scripts import tahoe_mv
492 rc = tahoe_mv.mv(options, mode="link")
496 from allmydata.scripts import tahoe_backup
497 rc = tahoe_backup.backup(options)
500 def webopen(options, opener=None):
501 from allmydata.scripts import tahoe_webopen
502 rc = tahoe_webopen.webopen(options, opener=opener)
505 def manifest(options):
506 from allmydata.scripts import tahoe_manifest
507 rc = tahoe_manifest.manifest(options)
511 from allmydata.scripts import tahoe_manifest
512 rc = tahoe_manifest.stats(options)
516 from allmydata.scripts import tahoe_check
517 rc = tahoe_check.check(options)
520 def deepcheck(options):
521 from allmydata.scripts import tahoe_check
522 rc = tahoe_check.deepcheck(options)
527 "add-alias": add_alias,
528 "create-alias": create_alias,
529 "list-aliases": list_aliases,
539 "manifest": manifest,
542 "deep-check": deepcheck,