From b0c24a66ec7ede1c70e082fc1a652fb7b61bae9d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 6 Dec 2018 09:44:38 +0100 Subject: Rip out half-implemented m.login.saml2 support (#4265) * Rip out half-implemented m.login.saml2 support This was implemented in an odd way that left most of the work to the client, in a way that I really didn't understand. It's going to be a pain to maintain, so let's start by ripping it out. * drop undocumented dependency on dateutil It turns out we were relying on dateutil being pulled in transitively by pysaml2. There's no need for that bloat. --- synapse/python_dependencies.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/python_dependencies.py') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ca62ee7637..75ba9947cc 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -53,7 +53,6 @@ REQUIREMENTS = { "pillow>=3.1.2": ["PIL"], "sortedcontainers>=1.4.4": ["sortedcontainers"], "psutil>=2.0.0": ["psutil>=2.0.0"], - "pysaml2>=3.0.0": ["saml2"], "pymacaroons-pynacl>=0.9.3": ["pymacaroons"], "msgpack-python>=0.4.2": ["msgpack"], "phonenumbers>=8.2.0": ["phonenumbers"], -- cgit 1.5.1 From c7401a697f1ee3410b860afd8686f8bb012a8dce Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 7 Dec 2018 13:11:11 +0100 Subject: Implement SAML2 authentication (#4267) This implements both a SAML2 metadata endpoint (at `/_matrix/saml2/metadata.xml`), and a SAML2 response receiver (at `/_matrix/saml2/authn_response`). If the SAML2 response matches what's been configured, we complete the SSO login flow by redirecting to the client url (aka `RelayState` in SAML2 jargon) with a login token. What we don't yet have is anything to build a SAML2 request and redirect the user to the identity provider. That is left as an exercise for the reader. --- changelog.d/4267.feature | 1 + synapse/app/homeserver.py | 4 ++ synapse/config/homeserver.py | 3 +- synapse/config/saml2_config.py | 110 ++++++++++++++++++++++++++++++++ synapse/python_dependencies.py | 5 +- synapse/rest/saml2/__init__.py | 29 +++++++++ synapse/rest/saml2/metadata_resource.py | 36 +++++++++++ synapse/rest/saml2/response_resource.py | 71 +++++++++++++++++++++ tests/utils.py | 1 + 9 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4267.feature create mode 100644 synapse/config/saml2_config.py create mode 100644 synapse/rest/saml2/__init__.py create mode 100644 synapse/rest/saml2/metadata_resource.py create mode 100644 synapse/rest/saml2/response_resource.py (limited to 'synapse/python_dependencies.py') diff --git a/changelog.d/4267.feature b/changelog.d/4267.feature new file mode 100644 index 0000000000..da36986e2b --- /dev/null +++ b/changelog.d/4267.feature @@ -0,0 +1 @@ +Rework SAML2 authentication diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a03a3e4b8a..1e495a38b9 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -199,6 +199,10 @@ class SynapseHomeServer(HomeServer): "/.well-known/matrix/client": WellKnownResource(self), }) + if self.get_config().saml2_enabled: + from synapse.rest.saml2 import SAML2Resource + resources["/_matrix/saml2"] = SAML2Resource(self) + if name == "consent": from synapse.rest.consent.consent_resource import ConsentResource consent_resource = ConsentResource(self) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 36182267c2..9d740c7a71 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -32,6 +32,7 @@ from .ratelimiting import RatelimitConfig from .registration import RegistrationConfig from .repository import ContentRepositoryConfig from .room_directory import RoomDirectoryConfig +from .saml2_config import SAML2Config from .server import ServerConfig from .server_notices_config import ServerNoticesConfig from .spam_checker import SpamCheckerConfig @@ -44,7 +45,7 @@ from .workers import WorkerConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig, - AppServiceConfig, KeyConfig, CasConfig, + AppServiceConfig, KeyConfig, SAML2Config, CasConfig, JWTConfig, PasswordConfig, EmailConfig, WorkerConfig, PasswordAuthProviderConfig, PushConfig, SpamCheckerConfig, GroupsConfig, UserDirectoryConfig, diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2_config.py new file mode 100644 index 0000000000..86ffe334f5 --- /dev/null +++ b/synapse/config/saml2_config.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config, ConfigError + + +class SAML2Config(Config): + def read_config(self, config): + self.saml2_enabled = False + + saml2_config = config.get("saml2_config") + + if not saml2_config or not saml2_config.get("enabled", True): + return + + self.saml2_enabled = True + + import saml2.config + self.saml2_sp_config = saml2.config.SPConfig() + self.saml2_sp_config.load(self._default_saml_config_dict()) + self.saml2_sp_config.load(saml2_config.get("sp_config", {})) + + config_path = saml2_config.get("config_path", None) + if config_path is not None: + self.saml2_sp_config.load_file(config_path) + + def _default_saml_config_dict(self): + import saml2 + + public_baseurl = self.public_baseurl + if public_baseurl is None: + raise ConfigError( + "saml2_config requires a public_baseurl to be set" + ) + + metadata_url = public_baseurl + "_matrix/saml2/metadata.xml" + response_url = public_baseurl + "_matrix/saml2/authn_response" + return { + "entityid": metadata_url, + + "service": { + "sp": { + "endpoints": { + "assertion_consumer_service": [ + (response_url, saml2.BINDING_HTTP_POST), + ], + }, + "required_attributes": ["uid"], + "optional_attributes": ["mail", "surname", "givenname"], + }, + } + } + + def default_config(self, config_dir_path, server_name, **kwargs): + return """ + # Enable SAML2 for registration and login. Uses pysaml2. + # + # saml2_config: + # + # # The following is the configuration for the pysaml2 Service Provider. + # # See pysaml2 docs for format of config. + # # + # # Default values will be used for the 'entityid' and 'service' settings, + # # so it is not normally necessary to specify them unless you need to + # # override them. + # + # sp_config: + # # point this to the IdP's metadata. You can use either a local file or + # # (preferably) a URL. + # metadata: + # # local: ["saml2/idp.xml"] + # remote: + # - url: https://our_idp/metadata.xml + # + # # The following is just used to generate our metadata xml, and you + # # may well not need it, depending on your setup. Alternatively you + # # may need a whole lot more detail - see the pysaml2 docs! + # + # description: ["My awesome SP", "en"] + # name: ["Test SP", "en"] + # + # organization: + # name: Example com + # display_name: + # - ["Example co", "en"] + # url: "http://example.com" + # + # contact_person: + # - given_name: Bob + # sur_name: "the Sysadmin" + # email_address": ["admin@example.com"] + # contact_type": technical + # + # # Instead of putting the config inline as above, you can specify a + # # separate pysaml2 configuration file: + # # + # # config_path: "%(config_dir_path)s/sp_conf.py" + """ % {"config_dir_path": config_dir_path} diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 75ba9947cc..db631e0c6c 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -80,7 +80,10 @@ CONDITIONAL_REQUIREMENTS = { }, "postgres": { "psycopg2>=2.6": ["psycopg2"] - } + }, + "saml2": { + "pysaml2>=4.5.0": ["saml2"], + }, } diff --git a/synapse/rest/saml2/__init__.py b/synapse/rest/saml2/__init__.py new file mode 100644 index 0000000000..68da37ca6a --- /dev/null +++ b/synapse/rest/saml2/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from twisted.web.resource import Resource + +from synapse.rest.saml2.metadata_resource import SAML2MetadataResource +from synapse.rest.saml2.response_resource import SAML2ResponseResource + +logger = logging.getLogger(__name__) + + +class SAML2Resource(Resource): + def __init__(self, hs): + Resource.__init__(self) + self.putChild(b"metadata.xml", SAML2MetadataResource(hs)) + self.putChild(b"authn_response", SAML2ResponseResource(hs)) diff --git a/synapse/rest/saml2/metadata_resource.py b/synapse/rest/saml2/metadata_resource.py new file mode 100644 index 0000000000..e8c680aeb4 --- /dev/null +++ b/synapse/rest/saml2/metadata_resource.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import saml2.metadata + +from twisted.web.resource import Resource + + +class SAML2MetadataResource(Resource): + """A Twisted web resource which renders the SAML metadata""" + + isLeaf = 1 + + def __init__(self, hs): + Resource.__init__(self) + self.sp_config = hs.config.saml2_sp_config + + def render_GET(self, request): + metadata_xml = saml2.metadata.create_metadata_string( + configfile=None, config=self.sp_config, + ) + request.setHeader(b"Content-Type", b"text/xml; charset=utf-8") + return metadata_xml diff --git a/synapse/rest/saml2/response_resource.py b/synapse/rest/saml2/response_resource.py new file mode 100644 index 0000000000..ad2ed157b5 --- /dev/null +++ b/synapse/rest/saml2/response_resource.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +import saml2 +from saml2.client import Saml2Client + +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + +from synapse.api.errors import CodeMessageException +from synapse.http.server import wrap_html_request_handler +from synapse.http.servlet import parse_string +from synapse.rest.client.v1.login import SSOAuthHandler + +logger = logging.getLogger(__name__) + + +class SAML2ResponseResource(Resource): + """A Twisted web resource which handles the SAML response""" + + isLeaf = 1 + + def __init__(self, hs): + Resource.__init__(self) + + self._saml_client = Saml2Client(hs.config.saml2_sp_config) + self._sso_auth_handler = SSOAuthHandler(hs) + + def render_POST(self, request): + self._async_render_POST(request) + return NOT_DONE_YET + + @wrap_html_request_handler + def _async_render_POST(self, request): + resp_bytes = parse_string(request, 'SAMLResponse', required=True) + relay_state = parse_string(request, 'RelayState', required=True) + + try: + saml2_auth = self._saml_client.parse_authn_request_response( + resp_bytes, saml2.BINDING_HTTP_POST, + ) + except Exception as e: + logger.warning("Exception parsing SAML2 response", exc_info=1) + raise CodeMessageException( + 400, "Unable to parse SAML2 response: %s" % (e,), + ) + + if saml2_auth.not_signed: + raise CodeMessageException(400, "SAML2 response was not signed") + + if "uid" not in saml2_auth.ava: + raise CodeMessageException(400, "uid not in SAML2 response") + + username = saml2_auth.ava["uid"][0] + return self._sso_auth_handler.on_successful_auth( + username, request, relay_state, + ) diff --git a/tests/utils.py b/tests/utils.py index 52ab762010..04796a9b30 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -139,6 +139,7 @@ def default_config(name): config.admin_contact = None config.rc_messages_per_second = 10000 config.rc_message_burst_count = 10000 + config.saml2_enabled = False config.use_frozen_dicts = False -- cgit 1.5.1 From c26f49a6645a84730811ea7bc5158d826bc43484 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 22 Dec 2018 01:37:26 +1100 Subject: Make the dependencies more like a standard Python project and hook up the optional dependencies to setuptools (#4298) --- README.rst | 11 +- changelog.d/4298.feature | 1 + setup.py | 16 ++- synapse/app/__init__.py | 6 +- synapse/app/homeserver.py | 3 - synapse/python_dependencies.py | 228 ++++++++++++++++------------------------- tox.ini | 4 +- 7 files changed, 111 insertions(+), 158 deletions(-) create mode 100644 changelog.d/4298.feature (limited to 'synapse/python_dependencies.py') diff --git a/README.rst b/README.rst index 02b3db1bc6..c4df779ac7 100644 --- a/README.rst +++ b/README.rst @@ -86,7 +86,7 @@ Synapse is the reference Python/Twisted Matrix homeserver implementation. System requirements: - POSIX-compliant system (tested on Linux & OS X) -- Python 3.5, 3.6, or 2.7 +- Python 3.5, 3.6, 3.7, or 2.7 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org Installing from source @@ -148,7 +148,7 @@ To install the Synapse homeserver run:: source ~/synapse/env/bin/activate pip install --upgrade pip pip install --upgrade setuptools - pip install matrix-synapse + pip install matrix-synapse[all] This installs Synapse, along with the libraries it uses, into a virtual environment under ``~/synapse/env``. Feel free to pick a different directory @@ -158,7 +158,7 @@ This Synapse installation can then be later upgraded by using pip again with the update flag:: source ~/synapse/env/bin/activate - pip install -U matrix-synapse + pip install -U matrix-synapse[all] In case of problems, please see the _`Troubleshooting` section below. @@ -826,8 +826,7 @@ to install using pip and a virtualenv:: virtualenv -p python2.7 env source env/bin/activate - python -m synapse.python_dependencies | xargs pip install - pip install lxml mock + python -m pip install -e .[all] This will run a process of downloading and installing all the needed dependencies into a virtual env. @@ -835,7 +834,7 @@ dependencies into a virtual env. Once this is done, you may wish to run Synapse's unit tests, to check that everything is installed as it should be:: - PYTHONPATH="." trial tests + python -m twisted.trial tests This should end with a 'PASSED' result:: diff --git a/changelog.d/4298.feature b/changelog.d/4298.feature new file mode 100644 index 0000000000..05ad70fe72 --- /dev/null +++ b/changelog.d/4298.feature @@ -0,0 +1 @@ +Synapse can now have its conditional/extra dependencies installed by pip. This functionality can be used by using `pip install matrix-synapse[feature]`, where feature is a comma separated list with the possible values "email.enable_notifs", "ldap3", "postgres", "saml2", "url_preview", and "test". If you want to install all optional dependencies, you can use "all" instead. diff --git a/setup.py b/setup.py index 00b69c43f5..55b1b10a77 100755 --- a/setup.py +++ b/setup.py @@ -84,13 +84,25 @@ version = exec_file(("synapse", "__init__.py"))["__version__"] dependencies = exec_file(("synapse", "python_dependencies.py")) long_description = read_file(("README.rst",)) +REQUIREMENTS = dependencies['REQUIREMENTS'] +CONDITIONAL_REQUIREMENTS = dependencies['CONDITIONAL_REQUIREMENTS'] + +# Make `pip install matrix-synapse[all]` install all the optional dependencies. +ALL_OPTIONAL_REQUIREMENTS = set() + +for optional_deps in CONDITIONAL_REQUIREMENTS.values(): + ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS + +CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS) + + setup( name="matrix-synapse", version=version, packages=find_packages(exclude=["tests", "tests.*"]), description="Reference homeserver for the Matrix decentralised comms protocol", - install_requires=dependencies['requirements'](include_conditional=True).keys(), - dependency_links=dependencies["DEPENDENCY_LINKS"].values(), + install_requires=REQUIREMENTS, + extras_require=CONDITIONAL_REQUIREMENTS, include_package_data=True, zip_safe=False, long_description=long_description, diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index c3afcc573b..233bf43fc8 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -22,11 +22,11 @@ sys.dont_write_bytecode = True try: python_dependencies.check_requirements() -except python_dependencies.MissingRequirementError as e: +except python_dependencies.DependencyException as e: message = "\n".join([ - "Missing Requirement: %s" % (str(e),), + "Missing Requirements: %s" % (", ".join(e.dependencies),), "To install run:", - " pip install --upgrade --force \"%s\"" % (e.dependency,), + " pip install --upgrade --force %s" % (" ".join(e.dependencies),), "", ]) sys.stderr.writelines(message) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f2064f9d0c..f3ac3d19f0 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -322,9 +322,6 @@ def setup(config_options): synapse.config.logger.setup_logging(config, use_worker_options=False) - # check any extra requirements we have now we have a config - check_requirements(config) - events.USE_FROZEN_DICTS = config.use_frozen_dicts tls_server_context_factory = context_factory.ServerContextFactory(config) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 72a92cc462..2c65ef5856 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -15,175 +15,121 @@ # limitations under the License. import logging -from distutils.version import LooseVersion + +from pkg_resources import DistributionNotFound, VersionConflict, get_distribution logger = logging.getLogger(__name__) -# this dict maps from python package name to a list of modules we expect it to -# provide. -# -# the key is a "requirement specifier", as used as a parameter to `pip -# install`[1], or an `install_requires` argument to `setuptools.setup` [2]. + +# REQUIREMENTS is a simple list of requirement specifiers[1], and must be +# installed. It is passed to setup() as install_requires in setup.py. # -# the value is a sequence of strings; each entry should be the name of the -# python module, optionally followed by a version assertion which can be either -# ">=" or "==". +# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict +# of lists. The dict key is the optional dependency name and can be passed to +# pip when installing. The list is a series of requirement specifiers[1] to be +# installed when that optional dependency requirement is specified. It is passed +# to setup() as extras_require in setup.py # # [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers. -# [2] https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-dependencies -REQUIREMENTS = { - "jsonschema>=2.5.1": ["jsonschema>=2.5.1"], - "frozendict>=1": ["frozendict"], - "unpaddedbase64>=1.1.0": ["unpaddedbase64>=1.1.0"], - "canonicaljson>=1.1.3": ["canonicaljson>=1.1.3"], - "signedjson>=1.0.0": ["signedjson>=1.0.0"], - "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"], - "service_identity>=16.0.0": ["service_identity>=16.0.0"], - "Twisted>=17.1.0": ["twisted>=17.1.0"], - "treq>=15.1": ["treq>=15.1"], +REQUIREMENTS = [ + "jsonschema>=2.5.1", + "frozendict>=1", + "unpaddedbase64>=1.1.0", + "canonicaljson>=1.1.3", + "signedjson>=1.0.0", + "pynacl>=1.2.1", + "service_identity>=16.0.0", + "Twisted>=17.1.0", + "treq>=15.1", # Twisted has required pyopenssl 16.0 since about Twisted 16.6. - "pyopenssl>=16.0.0": ["OpenSSL>=16.0.0"], - - "pyyaml>=3.11": ["yaml"], - "pyasn1>=0.1.9": ["pyasn1"], - "pyasn1-modules>=0.0.7": ["pyasn1_modules"], - "daemonize>=2.3.1": ["daemonize"], - "bcrypt>=3.1.0": ["bcrypt>=3.1.0"], - "pillow>=3.1.2": ["PIL"], - "sortedcontainers>=1.4.4": ["sortedcontainers"], - "psutil>=2.0.0": ["psutil>=2.0.0"], - "pymacaroons-pynacl>=0.9.3": ["pymacaroons"], - "msgpack-python>=0.4.2": ["msgpack"], - "phonenumbers>=8.2.0": ["phonenumbers"], - "six>=1.10": ["six"], - + "pyopenssl>=16.0.0", + "pyyaml>=3.11", + "pyasn1>=0.1.9", + "pyasn1-modules>=0.0.7", + "daemonize>=2.3.1", + "bcrypt>=3.1.0", + "pillow>=3.1.2", + "sortedcontainers>=1.4.4", + "psutil>=2.0.0", + "pymacaroons-pynacl>=0.9.3", + "msgpack-python>=0.4.2", + "phonenumbers>=8.2.0", + "six>=1.10", # prometheus_client 0.4.0 changed the format of counter metrics # (cf https://github.com/matrix-org/synapse/issues/4001) - "prometheus_client>=0.0.18,<0.4.0": ["prometheus_client"], - + "prometheus_client>=0.0.18,<0.4.0", # we use attr.s(slots), which arrived in 16.0.0 - "attrs>=16.0.0": ["attr>=16.0.0"], - "netaddr>=0.7.18": ["netaddr"], -} + "attrs>=16.0.0", + "netaddr>=0.7.18", +] CONDITIONAL_REQUIREMENTS = { - "email.enable_notifs": { - "Jinja2>=2.8": ["Jinja2>=2.8"], - "bleach>=1.4.2": ["bleach>=1.4.2"], - }, - "matrix-synapse-ldap3": { - "matrix-synapse-ldap3>=0.1": ["ldap_auth_provider"], - }, - "postgres": { - "psycopg2>=2.6": ["psycopg2"] - }, - "saml2": { - "pysaml2>=4.5.0": ["saml2"], - }, + "email.enable_notifs": ["Jinja2>=2.8", "bleach>=1.4.2"], + "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], + "postgres": ["psycopg2>=2.6"], + "saml2": ["pysaml2>=4.5.0"], + "url_preview": ["lxml>=3.5.0"], + "test": ["mock>=2.0"], } -def requirements(config=None, include_conditional=False): - reqs = REQUIREMENTS.copy() - if include_conditional: - for _, req in CONDITIONAL_REQUIREMENTS.items(): - reqs.update(req) - return reqs +def list_requirements(): + deps = set(REQUIREMENTS) + for opt in CONDITIONAL_REQUIREMENTS.values(): + deps = set(opt) | deps + return list(deps) -def github_link(project, version, egg): - return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) +class DependencyException(Exception): + @property + def dependencies(self): + for i in self.args[0]: + yield '"' + i + '"' -DEPENDENCY_LINKS = { -} +def check_requirements(_get_distribution=get_distribution): + + deps_needed = [] + errors = [] -class MissingRequirementError(Exception): - def __init__(self, message, module_name, dependency): - super(MissingRequirementError, self).__init__(message) - self.module_name = module_name - self.dependency = dependency - - -def check_requirements(config=None): - """Checks that all the modules needed by synapse have been correctly - installed and are at the correct version""" - for dependency, module_requirements in ( - requirements(config, include_conditional=False).items()): - for module_requirement in module_requirements: - if ">=" in module_requirement: - module_name, required_version = module_requirement.split(">=") - version_test = ">=" - elif "==" in module_requirement: - module_name, required_version = module_requirement.split("==") - version_test = "==" - else: - module_name = module_requirement - version_test = None - - try: - module = __import__(module_name) - except ImportError: - logging.exception( - "Can't import %r which is part of %r", - module_name, dependency - ) - raise MissingRequirementError( - "Can't import %r which is part of %r" - % (module_name, dependency), module_name, dependency - ) - version = getattr(module, "__version__", None) - file_path = getattr(module, "__file__", None) - logger.info( - "Using %r version %r from %r to satisfy %r", - module_name, version, file_path, dependency + # Check the base dependencies exist -- they all must be installed. + for dependency in REQUIREMENTS: + try: + _get_distribution(dependency) + except VersionConflict as e: + deps_needed.append(dependency) + errors.append( + "Needed %s, got %s==%s" + % (dependency, e.dist.project_name, e.dist.version) ) + except DistributionNotFound: + deps_needed.append(dependency) + errors.append("Needed %s but it was not installed" % (dependency,)) - if version_test == ">=": - if version is None: - raise MissingRequirementError( - "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name), module_name, dependency - ) - if LooseVersion(version) < LooseVersion(required_version): - raise MissingRequirementError( - "Version of %r in %r is too old. %r < %r" - % (dependency, file_path, version, required_version), - module_name, dependency - ) - elif version_test == "==": - if version is None: - raise MissingRequirementError( - "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name), module_name, dependency - ) - if LooseVersion(version) != LooseVersion(required_version): - raise MissingRequirementError( - "Unexpected version of %r in %r. %r != %r" - % (dependency, file_path, version, required_version), - module_name, dependency - ) + # Check the optional dependencies are up to date. We allow them to not be + # installed. + OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), []) + for dependency in OPTS: + try: + _get_distribution(dependency) + except VersionConflict: + deps_needed.append(dependency) + errors.append("Needed %s but it was not installed" % (dependency,)) + except DistributionNotFound: + # If it's not found, we don't care + pass -def list_requirements(): - result = [] - linked = [] - for link in DEPENDENCY_LINKS.values(): - egg = link.split("#egg=")[1] - linked.append(egg.split('-')[0]) - result.append(link) - for requirement in requirements(include_conditional=True): - is_linked = False - for link in linked: - if requirement.replace('-', '_').startswith(link): - is_linked = True - if not is_linked: - result.append(requirement) - return result + if deps_needed: + for e in errors: + logging.exception(e) + + raise DependencyException(deps_needed) if __name__ == "__main__": import sys + sys.stdout.writelines(req + "\n" for req in list_requirements()) diff --git a/tox.ini b/tox.ini index f5f07cb690..a0f5486829 100644 --- a/tox.ini +++ b/tox.ini @@ -9,9 +9,6 @@ deps = junitxml coverage - # needed by some of the tests - lxml - # cyptography 2.2 requires setuptools >= 18.5 # # older versions of virtualenv (?) give us a virtualenv with the same @@ -33,6 +30,7 @@ setenv = [testenv] deps = {[base]deps} +extras = all whitelist_externals = sh -- cgit 1.5.1 From b7c021881293c60ad0874f6d15375a9b7a40c22c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 7 Jan 2019 10:14:31 +0000 Subject: Check jinja version for consent resource (#4327) * Raise a ConfigError if an invalid resource is specified * Require Jinja 2.9 for the consent resource * changelog --- changelog.d/4327.bugfix | 1 + synapse/app/__init__.py | 9 +------ synapse/config/server.py | 38 ++++++++++++++++++++++++++++- synapse/python_dependencies.py | 55 +++++++++++++++++++++++++++--------------- 4 files changed, 75 insertions(+), 28 deletions(-) create mode 100644 changelog.d/4327.bugfix (limited to 'synapse/python_dependencies.py') diff --git a/changelog.d/4327.bugfix b/changelog.d/4327.bugfix new file mode 100644 index 0000000000..03bf05e05c --- /dev/null +++ b/changelog.d/4327.bugfix @@ -0,0 +1 @@ +Check jinja version for consent resource diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index 233bf43fc8..b45adafdd3 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -19,15 +19,8 @@ from synapse import python_dependencies # noqa: E402 sys.dont_write_bytecode = True - try: python_dependencies.check_requirements() except python_dependencies.DependencyException as e: - message = "\n".join([ - "Missing Requirements: %s" % (", ".join(e.dependencies),), - "To install run:", - " pip install --upgrade --force %s" % (" ".join(e.dependencies),), - "", - ]) - sys.stderr.writelines(message) + sys.stderr.writelines(e.message) sys.exit(1) diff --git a/synapse/config/server.py b/synapse/config/server.py index 120c2b81fc..fb57791098 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd -# Copyright 2017 New Vector Ltd +# Copyright 2017-2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import logging import os.path from synapse.http.endpoint import parse_and_validate_server_name +from synapse.python_dependencies import DependencyException, check_requirements from ._base import Config, ConfigError @@ -204,6 +205,8 @@ class ServerConfig(Config): ] }) + _check_resource_config(self.listeners) + def default_config(self, server_name, data_dir_path, **kwargs): _, bind_port = parse_and_validate_server_name(server_name) if bind_port is not None: @@ -465,3 +468,36 @@ def _warn_if_webclient_configured(listeners): if name == 'webclient': logger.warning(NO_MORE_WEB_CLIENT_WARNING) return + + +KNOWN_RESOURCES = ( + 'client', + 'consent', + 'federation', + 'keys', + 'media', + 'metrics', + 'replication', + 'static', + 'webclient', +) + + +def _check_resource_config(listeners): + resource_names = set( + res_name + for listener in listeners + for res in listener.get("resources", []) + for res_name in res.get("names", []) + ) + + for resource in resource_names: + if resource not in KNOWN_RESOURCES: + raise ConfigError( + "Unknown listener resource '%s'" % (resource, ) + ) + if resource == "consent": + try: + check_requirements('resources.consent') + except DependencyException as e: + raise ConfigError(e.message) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 2c65ef5856..69c5f9fe2e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -65,9 +65,13 @@ REQUIREMENTS = [ ] CONDITIONAL_REQUIREMENTS = { - "email.enable_notifs": ["Jinja2>=2.8", "bleach>=1.4.2"], + "email.enable_notifs": ["Jinja2>=2.9", "bleach>=1.4.2"], "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], "postgres": ["psycopg2>=2.6"], + + # ConsentResource uses select_autoescape, which arrived in jinja 2.9 + "resources.consent": ["Jinja2>=2.9"], + "saml2": ["pysaml2>=4.5.0"], "url_preview": ["lxml>=3.5.0"], "test": ["mock>=2.0"], @@ -83,19 +87,31 @@ def list_requirements(): class DependencyException(Exception): + @property + def message(self): + return "\n".join([ + "Missing Requirements: %s" % (", ".join(self.dependencies),), + "To install run:", + " pip install --upgrade --force %s" % (" ".join(self.dependencies),), + "", + ]) + @property def dependencies(self): for i in self.args[0]: yield '"' + i + '"' -def check_requirements(_get_distribution=get_distribution): - +def check_requirements(for_feature=None, _get_distribution=get_distribution): deps_needed = [] errors = [] - # Check the base dependencies exist -- they all must be installed. - for dependency in REQUIREMENTS: + if for_feature: + reqs = CONDITIONAL_REQUIREMENTS[for_feature] + else: + reqs = REQUIREMENTS + + for dependency in reqs: try: _get_distribution(dependency) except VersionConflict as e: @@ -108,23 +124,24 @@ def check_requirements(_get_distribution=get_distribution): deps_needed.append(dependency) errors.append("Needed %s but it was not installed" % (dependency,)) - # Check the optional dependencies are up to date. We allow them to not be - # installed. - OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), []) - - for dependency in OPTS: - try: - _get_distribution(dependency) - except VersionConflict: - deps_needed.append(dependency) - errors.append("Needed %s but it was not installed" % (dependency,)) - except DistributionNotFound: - # If it's not found, we don't care - pass + if not for_feature: + # Check the optional dependencies are up to date. We allow them to not be + # installed. + OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), []) + + for dependency in OPTS: + try: + _get_distribution(dependency) + except VersionConflict: + deps_needed.append(dependency) + errors.append("Needed %s but it was not installed" % (dependency,)) + except DistributionNotFound: + # If it's not found, we don't care + pass if deps_needed: for e in errors: - logging.exception(e) + logging.error(e) raise DependencyException(deps_needed) -- cgit 1.5.1