Make the dependencies more like a standard Python project and hook up the optional dependencies to setuptools (#4298)

This commit is contained in:
Amber Brown 2018-12-22 01:37:26 +11:00 committed by GitHub
parent c8d32caba3
commit c26f49a664
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 120 additions and 167 deletions

View file

@ -86,7 +86,7 @@ Synapse is the reference Python/Twisted Matrix homeserver implementation.
System requirements:
- POSIX-compliant system (tested on Linux & OS X)
- Python 3.5, 3.6, or 2.7
- Python 3.5, 3.6, 3.7, or 2.7
- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
Installing from source
@ -148,7 +148,7 @@ To install the Synapse homeserver run::
source ~/synapse/env/bin/activate
pip install --upgrade pip
pip install --upgrade setuptools
pip install matrix-synapse
pip install matrix-synapse[all]
This installs Synapse, along with the libraries it uses, into a virtual
environment under ``~/synapse/env``. Feel free to pick a different directory
@ -158,7 +158,7 @@ This Synapse installation can then be later upgraded by using pip again with the
update flag::
source ~/synapse/env/bin/activate
pip install -U matrix-synapse
pip install -U matrix-synapse[all]
In case of problems, please see the _`Troubleshooting` section below.
@ -826,8 +826,7 @@ to install using pip and a virtualenv::
virtualenv -p python2.7 env
source env/bin/activate
python -m synapse.python_dependencies | xargs pip install
pip install lxml mock
python -m pip install -e .[all]
This will run a process of downloading and installing all the needed
dependencies into a virtual env.
@ -835,7 +834,7 @@ dependencies into a virtual env.
Once this is done, you may wish to run Synapse's unit tests, to
check that everything is installed as it should be::
PYTHONPATH="." trial tests
python -m twisted.trial tests
This should end with a 'PASSED' result::

1
changelog.d/4298.feature Normal file
View file

@ -0,0 +1 @@
Synapse can now have its conditional/extra dependencies installed by pip. This functionality can be used by using `pip install matrix-synapse[feature]`, where feature is a comma separated list with the possible values "email.enable_notifs", "ldap3", "postgres", "saml2", "url_preview", and "test". If you want to install all optional dependencies, you can use "all" instead.

View file

@ -84,13 +84,25 @@ version = exec_file(("synapse", "__init__.py"))["__version__"]
dependencies = exec_file(("synapse", "python_dependencies.py"))
long_description = read_file(("README.rst",))
REQUIREMENTS = dependencies['REQUIREMENTS']
CONDITIONAL_REQUIREMENTS = dependencies['CONDITIONAL_REQUIREMENTS']
# Make `pip install matrix-synapse[all]` install all the optional dependencies.
ALL_OPTIONAL_REQUIREMENTS = set()
for optional_deps in CONDITIONAL_REQUIREMENTS.values():
ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS
CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS)
setup(
name="matrix-synapse",
version=version,
packages=find_packages(exclude=["tests", "tests.*"]),
description="Reference homeserver for the Matrix decentralised comms protocol",
install_requires=dependencies['requirements'](include_conditional=True).keys(),
dependency_links=dependencies["DEPENDENCY_LINKS"].values(),
install_requires=REQUIREMENTS,
extras_require=CONDITIONAL_REQUIREMENTS,
include_package_data=True,
zip_safe=False,
long_description=long_description,

View file

@ -22,11 +22,11 @@ sys.dont_write_bytecode = True
try:
python_dependencies.check_requirements()
except python_dependencies.MissingRequirementError as e:
except python_dependencies.DependencyException as e:
message = "\n".join([
"Missing Requirement: %s" % (str(e),),
"Missing Requirements: %s" % (", ".join(e.dependencies),),
"To install run:",
" pip install --upgrade --force \"%s\"" % (e.dependency,),
" pip install --upgrade --force %s" % (" ".join(e.dependencies),),
"",
])
sys.stderr.writelines(message)

View file

@ -322,9 +322,6 @@ def setup(config_options):
synapse.config.logger.setup_logging(config, use_worker_options=False)
# check any extra requirements we have now we have a config
check_requirements(config)
events.USE_FROZEN_DICTS = config.use_frozen_dicts
tls_server_context_factory = context_factory.ServerContextFactory(config)

View file

@ -15,175 +15,121 @@
# limitations under the License.
import logging
from distutils.version import LooseVersion
from pkg_resources import DistributionNotFound, VersionConflict, get_distribution
logger = logging.getLogger(__name__)
# this dict maps from python package name to a list of modules we expect it to
# provide.
# REQUIREMENTS is a simple list of requirement specifiers[1], and must be
# installed. It is passed to setup() as install_requires in setup.py.
#
# the key is a "requirement specifier", as used as a parameter to `pip
# install`[1], or an `install_requires` argument to `setuptools.setup` [2].
#
# the value is a sequence of strings; each entry should be the name of the
# python module, optionally followed by a version assertion which can be either
# ">=<ver>" or "==<ver>".
# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict
# of lists. The dict key is the optional dependency name and can be passed to
# pip when installing. The list is a series of requirement specifiers[1] to be
# installed when that optional dependency requirement is specified. It is passed
# to setup() as extras_require in setup.py
#
# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers.
# [2] https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-dependencies
REQUIREMENTS = {
"jsonschema>=2.5.1": ["jsonschema>=2.5.1"],
"frozendict>=1": ["frozendict"],
"unpaddedbase64>=1.1.0": ["unpaddedbase64>=1.1.0"],
"canonicaljson>=1.1.3": ["canonicaljson>=1.1.3"],
"signedjson>=1.0.0": ["signedjson>=1.0.0"],
"pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
"service_identity>=16.0.0": ["service_identity>=16.0.0"],
"Twisted>=17.1.0": ["twisted>=17.1.0"],
"treq>=15.1": ["treq>=15.1"],
REQUIREMENTS = [
"jsonschema>=2.5.1",
"frozendict>=1",
"unpaddedbase64>=1.1.0",
"canonicaljson>=1.1.3",
"signedjson>=1.0.0",
"pynacl>=1.2.1",
"service_identity>=16.0.0",
"Twisted>=17.1.0",
"treq>=15.1",
# Twisted has required pyopenssl 16.0 since about Twisted 16.6.
"pyopenssl>=16.0.0": ["OpenSSL>=16.0.0"],
"pyyaml>=3.11": ["yaml"],
"pyasn1>=0.1.9": ["pyasn1"],
"pyasn1-modules>=0.0.7": ["pyasn1_modules"],
"daemonize>=2.3.1": ["daemonize"],
"bcrypt>=3.1.0": ["bcrypt>=3.1.0"],
"pillow>=3.1.2": ["PIL"],
"sortedcontainers>=1.4.4": ["sortedcontainers"],
"psutil>=2.0.0": ["psutil>=2.0.0"],
"pymacaroons-pynacl>=0.9.3": ["pymacaroons"],
"msgpack-python>=0.4.2": ["msgpack"],
"phonenumbers>=8.2.0": ["phonenumbers"],
"six>=1.10": ["six"],
"pyopenssl>=16.0.0",
"pyyaml>=3.11",
"pyasn1>=0.1.9",
"pyasn1-modules>=0.0.7",
"daemonize>=2.3.1",
"bcrypt>=3.1.0",
"pillow>=3.1.2",
"sortedcontainers>=1.4.4",
"psutil>=2.0.0",
"pymacaroons-pynacl>=0.9.3",
"msgpack-python>=0.4.2",
"phonenumbers>=8.2.0",
"six>=1.10",
# prometheus_client 0.4.0 changed the format of counter metrics
# (cf https://github.com/matrix-org/synapse/issues/4001)
"prometheus_client>=0.0.18,<0.4.0": ["prometheus_client"],
"prometheus_client>=0.0.18,<0.4.0",
# we use attr.s(slots), which arrived in 16.0.0
"attrs>=16.0.0": ["attr>=16.0.0"],
"netaddr>=0.7.18": ["netaddr"],
}
"attrs>=16.0.0",
"netaddr>=0.7.18",
]
CONDITIONAL_REQUIREMENTS = {
"email.enable_notifs": {
"Jinja2>=2.8": ["Jinja2>=2.8"],
"bleach>=1.4.2": ["bleach>=1.4.2"],
},
"matrix-synapse-ldap3": {
"matrix-synapse-ldap3>=0.1": ["ldap_auth_provider"],
},
"postgres": {
"psycopg2>=2.6": ["psycopg2"]
},
"saml2": {
"pysaml2>=4.5.0": ["saml2"],
},
"email.enable_notifs": ["Jinja2>=2.8", "bleach>=1.4.2"],
"matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"],
"postgres": ["psycopg2>=2.6"],
"saml2": ["pysaml2>=4.5.0"],
"url_preview": ["lxml>=3.5.0"],
"test": ["mock>=2.0"],
}
def requirements(config=None, include_conditional=False):
reqs = REQUIREMENTS.copy()
if include_conditional:
for _, req in CONDITIONAL_REQUIREMENTS.items():
reqs.update(req)
return reqs
def github_link(project, version, egg):
return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg)
DEPENDENCY_LINKS = {
}
class MissingRequirementError(Exception):
def __init__(self, message, module_name, dependency):
super(MissingRequirementError, self).__init__(message)
self.module_name = module_name
self.dependency = dependency
def check_requirements(config=None):
"""Checks that all the modules needed by synapse have been correctly
installed and are at the correct version"""
for dependency, module_requirements in (
requirements(config, include_conditional=False).items()):
for module_requirement in module_requirements:
if ">=" in module_requirement:
module_name, required_version = module_requirement.split(">=")
version_test = ">="
elif "==" in module_requirement:
module_name, required_version = module_requirement.split("==")
version_test = "=="
else:
module_name = module_requirement
version_test = None
try:
module = __import__(module_name)
except ImportError:
logging.exception(
"Can't import %r which is part of %r",
module_name, dependency
)
raise MissingRequirementError(
"Can't import %r which is part of %r"
% (module_name, dependency), module_name, dependency
)
version = getattr(module, "__version__", None)
file_path = getattr(module, "__file__", None)
logger.info(
"Using %r version %r from %r to satisfy %r",
module_name, version, file_path, dependency
)
if version_test == ">=":
if version is None:
raise MissingRequirementError(
"Version of %r isn't set as __version__ of module %r"
% (dependency, module_name), module_name, dependency
)
if LooseVersion(version) < LooseVersion(required_version):
raise MissingRequirementError(
"Version of %r in %r is too old. %r < %r"
% (dependency, file_path, version, required_version),
module_name, dependency
)
elif version_test == "==":
if version is None:
raise MissingRequirementError(
"Version of %r isn't set as __version__ of module %r"
% (dependency, module_name), module_name, dependency
)
if LooseVersion(version) != LooseVersion(required_version):
raise MissingRequirementError(
"Unexpected version of %r in %r. %r != %r"
% (dependency, file_path, version, required_version),
module_name, dependency
)
def list_requirements():
result = []
linked = []
for link in DEPENDENCY_LINKS.values():
egg = link.split("#egg=")[1]
linked.append(egg.split('-')[0])
result.append(link)
for requirement in requirements(include_conditional=True):
is_linked = False
for link in linked:
if requirement.replace('-', '_').startswith(link):
is_linked = True
if not is_linked:
result.append(requirement)
return result
deps = set(REQUIREMENTS)
for opt in CONDITIONAL_REQUIREMENTS.values():
deps = set(opt) | deps
return list(deps)
class DependencyException(Exception):
@property
def dependencies(self):
for i in self.args[0]:
yield '"' + i + '"'
def check_requirements(_get_distribution=get_distribution):
deps_needed = []
errors = []
# Check the base dependencies exist -- they all must be installed.
for dependency in REQUIREMENTS:
try:
_get_distribution(dependency)
except VersionConflict as e:
deps_needed.append(dependency)
errors.append(
"Needed %s, got %s==%s"
% (dependency, e.dist.project_name, e.dist.version)
)
except DistributionNotFound:
deps_needed.append(dependency)
errors.append("Needed %s but it was not installed" % (dependency,))
# Check the optional dependencies are up to date. We allow them to not be
# installed.
OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), [])
for dependency in OPTS:
try:
_get_distribution(dependency)
except VersionConflict:
deps_needed.append(dependency)
errors.append("Needed %s but it was not installed" % (dependency,))
except DistributionNotFound:
# If it's not found, we don't care
pass
if deps_needed:
for e in errors:
logging.exception(e)
raise DependencyException(deps_needed)
if __name__ == "__main__":
import sys
sys.stdout.writelines(req + "\n" for req in list_requirements())

View file

@ -9,9 +9,6 @@ deps =
junitxml
coverage
# needed by some of the tests
lxml
# cyptography 2.2 requires setuptools >= 18.5
#
# older versions of virtualenv (?) give us a virtualenv with the same
@ -33,6 +30,7 @@ setenv =
[testenv]
deps =
{[base]deps}
extras = all
whitelist_externals =
sh