3 # Allmydata Tahoe -- secure, distributed storage grid
5 # Copyright (C) 2008 Allmydata, Inc.
7 # This file is part of tahoe.
9 # See the docs/about.html file for licensing information.
11 import os, re, sys, subprocess
13 ##### sys.path management
15 basedir = os.path.dirname(os.path.abspath(__file__))
16 pyver = "python%d.%d" % (sys.version_info[:2])
17 if sys.platform == "win32":
18 supportlib = os.path.join("support", "Lib", "site-packages")
20 supportlib = os.path.join("support", "lib", pyver, "site-packages")
21 supportlib = os.path.join(basedir, supportlib)
23 def add_tahoe_paths():
24 """Modify sys.path and PYTHONPATH to include Tahoe and supporting libraries
26 The first step towards building Tahoe is to run::
28 python setup.py build_tahoe
30 which is the equivalent of::
32 mkdir -p $(BASEDIR)/support/lib/python2.5/site-packages
33 (or cygpath equivalent)
34 setup.py develop --prefix=$(BASEDIR)/support
36 This installs .eggs for any dependent libraries that aren't already
37 available on the system, into support/lib/pythonN.N/site-packages (or
38 support/Lib/site-packages on windows). It also adds an .egg-link for
39 Tahoe itself into the same directory.
41 We add this directory to os.environ['PYTHONPATH'], so that any child
42 processes we spawn will be able to use these packages.
44 When the setuptools site.py sees that supportlib in PYTHONPATH, it scans
45 through it for .egg and .egg-link entries, and adds them to sys.path .
46 Since python has already processed all the site.py files by the time we
47 get here, we perform this same sort of processing ourselves: this makes
48 tahoe (and dependency libraries) available to code within setup.py
49 itself. This is used by the 'setup.py trial' subcommand, which invokes
50 trial directly rather than spawning a subprocess (this is easier than
51 locating the 'trial' executable, especially when Twisted was installed as
54 We'll need to add these .eggs to sys.path before importing anything that
55 isn't a part of stdlib. All the directories that we add this way are put
56 at the start of sys.path, so they will override anything that was present
57 on the system (and perhaps found lacking by the setuptools requirements
58 expressed in _auto_deps.py).
61 extra_syspath_items = []
62 extra_pythonpath_items = []
64 extra_syspath_items.append(supportlib)
65 extra_pythonpath_items.append(supportlib)
67 # Since we use setuptools to populate that directory, there will be a
68 # number of .egg and .egg-link entries there. Add all of them to
69 # sys.path, since that what the setuptools site.py would do if it
70 # encountered them at process start time. Without this step, the rest of
71 # this process would be unable to use the packages installed there. We
72 # don't need to add them to PYTHONPATH, since the site.py present there
73 # will add them when the child process starts up.
75 if os.path.isdir(supportlib):
76 for fn in os.listdir(supportlib):
77 if fn.endswith(".egg"):
78 extra_syspath_items.append(os.path.join(supportlib, fn))
80 # We also add our src/ directory, since that's where all the Tahoe code
81 # lives. This matches what site.py does when it sees the .egg-link file
82 # that is written to the support dir by an invocation of our 'setup.py
84 extra_syspath_items.append(os.path.join(basedir, "src"))
86 # and we put an extra copy of everything from PYTHONPATH in front, so
87 # that it is possible to override the packages that setuptools downloads
88 # with alternate versions, by doing e.g. "PYTHONPATH=foo python setup.py
90 oldpp = os.environ.get("PYTHONPATH", "").split(os.pathsep)
92 # grr silly split() behavior
94 extra_syspath_items = oldpp + extra_syspath_items
96 sys.path = extra_syspath_items + sys.path
98 # We also provide it to any child processes we spawn, via
99 # os.environ["PYTHONPATH"]
100 os.environ["PYTHONPATH"] = os.pathsep.join(oldpp + extra_pythonpath_items)
102 # add_tahoe_paths() must be called before use_setuptools() is called. I don't
103 # know why. If it isn't, then a later pkg_resources.requires(pycryptopp) call
104 # fails because an old version (in /usr/lib) was already loaded.
108 from ez_setup import use_setuptools
112 # This invokes our own customized version of ez_setup.py to make sure that
113 # setuptools >= v0.6c8 (a.k.a. v0.6-final) is installed.
115 # setuptools < v0.6c8 doesn't handle eggs which get installed into the CWD
116 # as a result of being transitively depended on in a setup_requires, but
117 # then are needed for the installed code to run, i.e. in an
119 use_setuptools(download_delay=0, min_version="0.6c8")
121 from setuptools import Extension, find_packages, setup
122 from setuptools.command import sdist
123 from distutils.core import Command
125 # Make the dependency-version-requirement, which is used by the Makefile at
126 # build-time, also available to the app at runtime:
129 shutil.copyfile("_auto_deps.py", os.path.join("src", "allmydata", "_auto_deps.py"))
130 except EnvironmentError:
131 # Nevermind then -- perhaps it is already in place and in any case we can do
136 "Development Status :: 5 - Production/Stable",
137 "Environment :: Console",
138 "Environment :: Web Environment",
139 "License :: OSI Approved :: GNU General Public License (GPL)",
140 "License :: DFSG approved",
141 "License :: Other/Proprietary License",
142 "Intended Audience :: Developers",
143 "Intended Audience :: End Users/Desktop",
144 "Intended Audience :: System Administrators",
145 "Operating System :: Microsoft",
146 "Operating System :: Microsoft :: Windows",
147 "Operating System :: Unix",
148 "Operating System :: POSIX :: Linux",
149 "Operating System :: POSIX",
150 "Operating System :: MacOS :: MacOS X",
151 "Operating System :: Microsoft :: Windows :: Windows NT/2000",
152 "Operating System :: OS Independent",
153 "Natural Language :: English",
154 "Programming Language :: C",
155 "Programming Language :: Python",
156 "Topic :: Utilities",
157 "Topic :: System :: Systems Administration",
158 "Topic :: System :: Filesystems",
159 "Topic :: System :: Distributed Computing",
160 "Topic :: Software Development :: Libraries",
161 "Topic :: Communications :: Usenet News",
162 "Topic :: System :: Archiving :: Backup",
163 "Topic :: System :: Archiving :: Mirroring",
164 "Topic :: System :: Archiving",
168 VERSIONFILE = "src/allmydata/_version.py"
171 verstrline = open(VERSIONFILE, "rt").read()
172 except EnvironmentError:
173 pass # Okay, there is no version file.
175 VSRE = r"^verstr = ['\"]([^'\"]*)['\"]"
176 mo = re.search(VSRE, verstrline, re.M)
180 print "unable to find version in %s" % (VERSIONFILE,)
181 raise RuntimeError("if %s.py exists, it is required to be well-formed" % (VERSIONFILE,))
184 """Welcome to the Tahoe project, a secure, decentralized, fault-tolerant
185 filesystem. All of the source code is available under a Free Software, Open
188 This filesystem is encrypted and spread over multiple peers in such a way that
189 it remains available even when some of the peers are unavailable,
190 malfunctioning, or malicious."""
192 # For Desert Island builds, assume that the user has extracted the dependency
193 # tarball into a directory named 'misc/dependencies'.
194 dependency_links=[os.path.join(os.getcwd(), 'misc', 'dependencies')]
196 # By adding a web page to the dependency_links we are able to put new packages
197 # up there and have them be automatically discovered by existing copies of the
198 # tahoe source when that source was built.
199 dependency_links.append("http://allmydata.org/trac/tahoe/wiki/Dependencies")
201 # Default setup_requires are pyutil for the Windows installer builder(see
202 # misc/sub-ver.py) and Twisted for the tests.
203 #setup_requires = ['pyutil >= 1.3.16', 'Twisted >= 2.4.0']
205 # darcsver is needed only if you want "./setup.py darcsver" to write a new
206 # version stamp in src/allmydata/_version.py, with a version number derived from
208 # http://pypi.python.org/pypi/darcsver
209 if 'darcsver' in sys.argv[1:]:
210 setup_requires.append('darcsver >= 1.1.5')
212 # setuptools_darcs is required to produce complete distributions (such as with
213 # "sdist" or "bdist_egg"), unless there is a PKG-INFO file present which shows
214 # that this is itself a source distribution.
215 # http://pypi.python.org/pypi/setuptools_darcs
216 if not os.path.exists('PKG-INFO'):
217 setup_requires.append('setuptools_darcs >= 1.1.0')
219 class ShowSupportLib(Command):
221 def initialize_options(self):
223 def finalize_options(self):
226 # TODO: --quiet suppresses the 'running show_supportlib' message.
227 # Find a way to do this all the time.
228 print supportlib # TODO windowsy
230 class ShowPythonPath(Command):
232 def initialize_options(self):
234 def finalize_options(self):
237 # TODO: --quiet suppresses the 'running show_supportlib' message.
238 # Find a way to do this all the time.
239 print "PYTHONPATH=%s" % os.environ["PYTHONPATH"]
241 class BuildTahoe(Command):
243 def initialize_options(self):
245 def finalize_options(self):
248 # 'setup.py develop --prefix SUPPORT' will complain if SUPPORTLIB is
249 # not on PYTHONPATH, because it thinks you are installing to a place
250 # that will not be searched at runtime (which is true, except that we
251 # add SUPPORTLIB to PYTHONPATH to run tests, etc). So set up
252 # PYTHONPATH now, then spawn a 'setup.py develop' command. Also, we
253 # have to create the directory ourselves.
254 if not os.path.isdir(supportlib):
255 os.makedirs(supportlib)
257 command = [sys.executable, "setup.py", "develop", "--prefix", "support"]
258 if sys.platform == "linux2":
259 # workaround for tahoe #229 / setuptools #17, on debian
260 command.extend(["--site-dirs", "/var/lib/python-support/" + pyver])
261 print "Command:", " ".join(command)
262 rc = subprocess.call(command)
264 print >>sys.stderr, "'setup.py develop' terminated by signal", -rc
267 print >>sys.stderr, "'setup.py develop' exited with rc", rc
270 class Trial(Command):
271 # Unlike 'build' and 'bdist_egg', the 'trial' subcommand cannot be run in
272 # conjunction with other subcommands.
274 # The '-a' argument is split on whitespace and passed into trial. (the
275 # distutils parser does not give subcommands access to the rest of
276 # sys.argv, so unfortunately we cannot just do something like:
277 # setup.py trial --reporter=text allmydata.test.test_util
280 # setup.py trial # run all tests
281 # setup.py trial -a allmydata.test.test_util # run some tests
282 # setup.py trial -a '--reporter=text allmydata.test.test_util' #other args
284 description = "Run unit tests via trial"
286 user_options = [ ("args=", "a", "Argument string to pass to trial: setup.py trial -a allmydata.test.test_util"),
288 def initialize_options(self):
289 self.args = "allmydata"
290 def finalize_options(self):
294 # make sure Twisted is available (for trial itself), and both the
295 # Tahoe source code and our dependent libraries are available (so
296 # that trial has some test code to work with)
298 from twisted.scripts import trial
300 args = self.args.strip().split()
302 # one wrinkle: we want to set the reactor here, because of bug #402
303 # (twisted bug #3218). We just jam in a "--reactor poll" at the start
304 # of the arglist. This does not permit the reactor to be overridden,
306 if sys.platform in ("linux2", "cygwin"):
307 # poll on linux2 to avoid #402 problems with select
308 # poll on cygwin since selectreactor runs out of fds
309 args = ["--reactor", "poll"] + args
311 # zooko also had os.environ["PYTHONUNBUFFERED"]="1" and
312 # args.append("--rterrors")
314 sys.argv = ["trial"] + args
316 print "To run this test directly, use:"
317 print "PYTHONPATH=%s %s" % (os.environ["PYTHONPATH"],
320 print "(run with -vv for trial command-line details)"
321 trial.run() # this does sys.exit
324 class MySdist(sdist.sdist):
325 """ A hook in the sdist command so that we can determine whether this the
326 tarball should be 'SUMO' or not, i.e. whether or not to include the
327 external dependency tarballs.
330 # Add our own sumo option to the sdist command, which toggles the
331 # external dependencies being included in the sdist.
332 user_options = sdist.sdist.user_options + \
333 [('sumo', 's', "create a 'sumo' sdist which includes the external " \
335 boolean_options = ['sumo']
337 def initialize_options(self):
338 sdist.sdist.initialize_options(self)
342 self.run_command('egg_info')
343 ei_cmd = self.get_finalized_command('egg_info')
344 self.filelist = ei_cmd.filelist
345 self.filelist.append(os.path.join(ei_cmd.egg_info,'SOURCES.txt'))
347 # If '--sumo' wasn't specified in the arguments, do not include
348 # the external dependency tarballs in the sdist.
350 self.filelist.exclude_pattern(None, prefix='misc/dependencies')
352 print self.filelist.files
354 self.check_metadata()
355 self.make_distribution()
357 dist_files = getattr(self.distribution,'dist_files',[])
358 for file in self.archive_files:
359 data = ('sdist', '', file)
360 if data not in dist_files:
361 dist_files.append(data)
363 # get a list of the libraries that we depend upon, for use in the call to
364 # setup() at the end of this file
365 from _auto_deps import install_requires
367 setup(name='allmydata-tahoe',
369 description='secure, decentralized, fault-tolerant filesystem',
370 long_description=LONG_DESCRIPTION,
371 author='the allmydata.org Tahoe project',
372 author_email='tahoe-dev@allmydata.org',
373 url='http://allmydata.org/',
375 cmdclass={"show_supportlib": ShowSupportLib,
376 "show_pythonpath": ShowPythonPath,
377 "build_tahoe": BuildTahoe,
381 package_dir = {'':'src'},
382 packages=find_packages("src"),
383 classifiers=trove_classifiers,
384 test_suite="allmydata.test",
385 install_requires=install_requires,
386 include_package_data=True,
387 setup_requires=setup_requires,
388 dependency_links=dependency_links,
389 entry_points = { 'console_scripts': [ 'tahoe = allmydata.scripts.runner:run' ] },
390 zip_safe=False, # We prefer unzipped for easier access.