From 4cd1c9f2ffa46bc8ed258da200ae3b8ba25fcbb5 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 29 Oct 2018 23:57:24 +1100 Subject: Delete the disused & unspecced identicon functionality (#4106) --- synapse/python_dependencies.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/python_dependencies.py') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 943876456b..ca62ee7637 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -51,7 +51,6 @@ REQUIREMENTS = { "daemonize>=2.3.1": ["daemonize"], "bcrypt>=3.1.0": ["bcrypt>=3.1.0"], "pillow>=3.1.2": ["PIL"], - "pydenticon>=0.2": ["pydenticon"], "sortedcontainers>=1.4.4": ["sortedcontainers"], "psutil>=2.0.0": ["psutil>=2.0.0"], "pysaml2>=3.0.0": ["saml2"], -- cgit 1.5.1 From b0c24a66ec7ede1c70e082fc1a652fb7b61bae9d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 6 Dec 2018 09:44:38 +0100 Subject: Rip out half-implemented m.login.saml2 support (#4265) * Rip out half-implemented m.login.saml2 support This was implemented in an odd way that left most of the work to the client, in a way that I really didn't understand. It's going to be a pain to maintain, so let's start by ripping it out. * drop undocumented dependency on dateutil It turns out we were relying on dateutil being pulled in transitively by pysaml2. There's no need for that bloat. --- changelog.d/4265.feature | 1 + synapse/config/homeserver.py | 3 +- synapse/config/saml2.py | 55 -------------------------------- synapse/handlers/register.py | 29 ----------------- synapse/python_dependencies.py | 1 - synapse/rest/client/v1/login.py | 69 ++--------------------------------------- tests/handlers/test_register.py | 15 --------- 7 files changed, 4 insertions(+), 169 deletions(-) create mode 100644 changelog.d/4265.feature delete mode 100644 synapse/config/saml2.py (limited to 'synapse/python_dependencies.py') diff --git a/changelog.d/4265.feature b/changelog.d/4265.feature new file mode 100644 index 0000000000..da36986e2b --- /dev/null +++ b/changelog.d/4265.feature @@ -0,0 +1 @@ +Rework SAML2 authentication diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 10dd40159f..36182267c2 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -32,7 +32,6 @@ from .ratelimiting import RatelimitConfig from .registration import RegistrationConfig from .repository import ContentRepositoryConfig from .room_directory import RoomDirectoryConfig -from .saml2 import SAML2Config from .server import ServerConfig from .server_notices_config import ServerNoticesConfig from .spam_checker import SpamCheckerConfig @@ -45,7 +44,7 @@ from .workers import WorkerConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig, - AppServiceConfig, KeyConfig, SAML2Config, CasConfig, + AppServiceConfig, KeyConfig, CasConfig, JWTConfig, PasswordConfig, EmailConfig, WorkerConfig, PasswordAuthProviderConfig, PushConfig, SpamCheckerConfig, GroupsConfig, UserDirectoryConfig, diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py deleted file mode 100644 index 8d7f443021..0000000000 --- a/synapse/config/saml2.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 Ericsson -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ._base import Config - - -class SAML2Config(Config): - """SAML2 Configuration - Synapse uses pysaml2 libraries for providing SAML2 support - - config_path: Path to the sp_conf.py configuration file - idp_redirect_url: Identity provider URL which will redirect - the user back to /login/saml2 with proper info. - - sp_conf.py file is something like: - https://github.com/rohe/pysaml2/blob/master/example/sp-repoze/sp_conf.py.example - - More information: https://pythonhosted.org/pysaml2/howto/config.html - """ - - def read_config(self, config): - saml2_config = config.get("saml2_config", None) - if saml2_config: - self.saml2_enabled = saml2_config.get("enabled", True) - self.saml2_config_path = saml2_config["config_path"] - self.saml2_idp_redirect_url = saml2_config["idp_redirect_url"] - else: - self.saml2_enabled = False - self.saml2_config_path = None - self.saml2_idp_redirect_url = None - - def default_config(self, config_dir_path, server_name, **kwargs): - return """ - # Enable SAML2 for registration and login. Uses pysaml2 - # config_path: Path to the sp_conf.py configuration file - # idp_redirect_url: Identity provider URL which will redirect - # the user back to /login/saml2 with proper info. - # See pysaml2 docs for format of config. - #saml2_config: - # enabled: true - # config_path: "%s/sp_conf.py" - # idp_redirect_url: "http://%s/idp" - """ % (config_dir_path, server_name) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 015909bb26..0f87c4610e 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -327,35 +327,6 @@ class RegistrationHandler(BaseHandler): else: logger.info("Valid captcha entered from %s", ip) - @defer.inlineCallbacks - def register_saml2(self, localpart): - """ - Registers email_id as SAML2 Based Auth. - """ - if types.contains_invalid_mxid_characters(localpart): - raise SynapseError( - 400, - "User ID can only contain characters a-z, 0-9, or '=_-./'", - ) - yield self.auth.check_auth_blocking() - user = UserID(localpart, self.hs.hostname) - user_id = user.to_string() - - yield self.check_user_id_not_appservice_exclusive(user_id) - token = self.macaroon_gen.generate_access_token(user_id) - try: - yield self.store.register( - user_id=user_id, - token=token, - password_hash=None, - create_profile_with_localpart=user.localpart, - ) - except Exception as e: - yield self.store.add_access_token_to_user(user_id, token) - # Ignore Registration errors - logger.exception(e) - defer.returnValue((user_id, token)) - @defer.inlineCallbacks def register_email(self, threepidCreds): """ diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ca62ee7637..75ba9947cc 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -53,7 +53,6 @@ REQUIREMENTS = { "pillow>=3.1.2": ["PIL"], "sortedcontainers>=1.4.4": ["sortedcontainers"], "psutil>=2.0.0": ["psutil>=2.0.0"], - "pysaml2>=3.0.0": ["saml2"], "pymacaroons-pynacl>=0.9.3": ["pymacaroons"], "msgpack-python>=0.4.2": ["msgpack"], "phonenumbers>=8.2.0": ["phonenumbers"], diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index f6b4a85e40..011e84e8b1 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -18,10 +18,6 @@ import xml.etree.ElementTree as ET from six.moves import urllib -from canonicaljson import json -from saml2 import BINDING_HTTP_POST, config -from saml2.client import Saml2Client - from twisted.internet import defer from twisted.web.client import PartialDownloadError @@ -81,7 +77,6 @@ def login_id_thirdparty_from_phone(identifier): class LoginRestServlet(ClientV1RestServlet): PATTERNS = client_path_patterns("/login$") - SAML2_TYPE = "m.login.saml2" CAS_TYPE = "m.login.cas" SSO_TYPE = "m.login.sso" TOKEN_TYPE = "m.login.token" @@ -89,8 +84,6 @@ class LoginRestServlet(ClientV1RestServlet): def __init__(self, hs): super(LoginRestServlet, self).__init__(hs) - self.idp_redirect_url = hs.config.saml2_idp_redirect_url - self.saml2_enabled = hs.config.saml2_enabled self.jwt_enabled = hs.config.jwt_enabled self.jwt_secret = hs.config.jwt_secret self.jwt_algorithm = hs.config.jwt_algorithm @@ -103,8 +96,6 @@ class LoginRestServlet(ClientV1RestServlet): flows = [] if self.jwt_enabled: flows.append({"type": LoginRestServlet.JWT_TYPE}) - if self.saml2_enabled: - flows.append({"type": LoginRestServlet.SAML2_TYPE}) if self.cas_enabled: flows.append({"type": LoginRestServlet.SSO_TYPE}) @@ -134,18 +125,8 @@ class LoginRestServlet(ClientV1RestServlet): def on_POST(self, request): login_submission = parse_json_object_from_request(request) try: - if self.saml2_enabled and (login_submission["type"] == - LoginRestServlet.SAML2_TYPE): - relay_state = "" - if "relay_state" in login_submission: - relay_state = "&RelayState=" + urllib.parse.quote( - login_submission["relay_state"]) - result = { - "uri": "%s%s" % (self.idp_redirect_url, relay_state) - } - defer.returnValue((200, result)) - elif self.jwt_enabled and (login_submission["type"] == - LoginRestServlet.JWT_TYPE): + if self.jwt_enabled and (login_submission["type"] == + LoginRestServlet.JWT_TYPE): result = yield self.do_jwt_login(login_submission) defer.returnValue(result) elif login_submission["type"] == LoginRestServlet.TOKEN_TYPE: @@ -345,50 +326,6 @@ class LoginRestServlet(ClientV1RestServlet): ) -class SAML2RestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/login/saml2", releases=()) - - def __init__(self, hs): - super(SAML2RestServlet, self).__init__(hs) - self.sp_config = hs.config.saml2_config_path - self.handlers = hs.get_handlers() - - @defer.inlineCallbacks - def on_POST(self, request): - saml2_auth = None - try: - conf = config.SPConfig() - conf.load_file(self.sp_config) - SP = Saml2Client(conf) - saml2_auth = SP.parse_authn_request_response( - request.args['SAMLResponse'][0], BINDING_HTTP_POST) - except Exception as e: # Not authenticated - logger.exception(e) - if saml2_auth and saml2_auth.status_ok() and not saml2_auth.not_signed: - username = saml2_auth.name_id.text - handler = self.handlers.registration_handler - (user_id, token) = yield handler.register_saml2(username) - # Forward to the RelayState callback along with ava - if 'RelayState' in request.args: - request.redirect(urllib.parse.unquote( - request.args['RelayState'][0]) + - '?status=authenticated&access_token=' + - token + '&user_id=' + user_id + '&ava=' + - urllib.quote(json.dumps(saml2_auth.ava))) - finish_request(request) - defer.returnValue(None) - defer.returnValue((200, {"status": "authenticated", - "user_id": user_id, "token": token, - "ava": saml2_auth.ava})) - elif 'RelayState' in request.args: - request.redirect(urllib.parse.unquote( - request.args['RelayState'][0]) + - '?status=not_authenticated') - finish_request(request) - defer.returnValue(None) - defer.returnValue((200, {"status": "not_authenticated"})) - - class CasRedirectServlet(RestServlet): PATTERNS = client_path_patterns("/login/(cas|sso)/redirect") @@ -517,8 +454,6 @@ class CasTicketServlet(ClientV1RestServlet): def register_servlets(hs, http_server): LoginRestServlet(hs).register(http_server) - if hs.config.saml2_enabled: - SAML2RestServlet(hs).register(http_server) if hs.config.cas_enabled: CasRedirectServlet(hs).register(http_server) CasTicketServlet(hs).register(http_server) diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 90a2a76475..23bce6ee7d 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -129,21 +129,6 @@ class RegistrationTestCase(unittest.TestCase): with self.assertRaises(ResourceLimitError): yield self.handler.register(localpart="local_part") - @defer.inlineCallbacks - def test_register_saml2_mau_blocked(self): - self.hs.config.limit_usage_by_mau = True - self.store.get_monthly_active_count = Mock( - return_value=defer.succeed(self.lots_of_users) - ) - with self.assertRaises(ResourceLimitError): - yield self.handler.register_saml2(localpart="local_part") - - self.store.get_monthly_active_count = Mock( - return_value=defer.succeed(self.hs.config.max_mau_value) - ) - with self.assertRaises(ResourceLimitError): - yield self.handler.register_saml2(localpart="local_part") - @defer.inlineCallbacks def test_auto_create_auto_join_rooms(self): room_alias_str = "#room:test" -- cgit 1.5.1 From c7401a697f1ee3410b860afd8686f8bb012a8dce Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 7 Dec 2018 13:11:11 +0100 Subject: Implement SAML2 authentication (#4267) This implements both a SAML2 metadata endpoint (at `/_matrix/saml2/metadata.xml`), and a SAML2 response receiver (at `/_matrix/saml2/authn_response`). If the SAML2 response matches what's been configured, we complete the SSO login flow by redirecting to the client url (aka `RelayState` in SAML2 jargon) with a login token. What we don't yet have is anything to build a SAML2 request and redirect the user to the identity provider. That is left as an exercise for the reader. --- changelog.d/4267.feature | 1 + synapse/app/homeserver.py | 4 ++ synapse/config/homeserver.py | 3 +- synapse/config/saml2_config.py | 110 ++++++++++++++++++++++++++++++++ synapse/python_dependencies.py | 5 +- synapse/rest/saml2/__init__.py | 29 +++++++++ synapse/rest/saml2/metadata_resource.py | 36 +++++++++++ synapse/rest/saml2/response_resource.py | 71 +++++++++++++++++++++ tests/utils.py | 1 + 9 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4267.feature create mode 100644 synapse/config/saml2_config.py create mode 100644 synapse/rest/saml2/__init__.py create mode 100644 synapse/rest/saml2/metadata_resource.py create mode 100644 synapse/rest/saml2/response_resource.py (limited to 'synapse/python_dependencies.py') diff --git a/changelog.d/4267.feature b/changelog.d/4267.feature new file mode 100644 index 0000000000..da36986e2b --- /dev/null +++ b/changelog.d/4267.feature @@ -0,0 +1 @@ +Rework SAML2 authentication diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a03a3e4b8a..1e495a38b9 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -199,6 +199,10 @@ class SynapseHomeServer(HomeServer): "/.well-known/matrix/client": WellKnownResource(self), }) + if self.get_config().saml2_enabled: + from synapse.rest.saml2 import SAML2Resource + resources["/_matrix/saml2"] = SAML2Resource(self) + if name == "consent": from synapse.rest.consent.consent_resource import ConsentResource consent_resource = ConsentResource(self) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 36182267c2..9d740c7a71 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -32,6 +32,7 @@ from .ratelimiting import RatelimitConfig from .registration import RegistrationConfig from .repository import ContentRepositoryConfig from .room_directory import RoomDirectoryConfig +from .saml2_config import SAML2Config from .server import ServerConfig from .server_notices_config import ServerNoticesConfig from .spam_checker import SpamCheckerConfig @@ -44,7 +45,7 @@ from .workers import WorkerConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig, - AppServiceConfig, KeyConfig, CasConfig, + AppServiceConfig, KeyConfig, SAML2Config, CasConfig, JWTConfig, PasswordConfig, EmailConfig, WorkerConfig, PasswordAuthProviderConfig, PushConfig, SpamCheckerConfig, GroupsConfig, UserDirectoryConfig, diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2_config.py new file mode 100644 index 0000000000..86ffe334f5 --- /dev/null +++ b/synapse/config/saml2_config.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config, ConfigError + + +class SAML2Config(Config): + def read_config(self, config): + self.saml2_enabled = False + + saml2_config = config.get("saml2_config") + + if not saml2_config or not saml2_config.get("enabled", True): + return + + self.saml2_enabled = True + + import saml2.config + self.saml2_sp_config = saml2.config.SPConfig() + self.saml2_sp_config.load(self._default_saml_config_dict()) + self.saml2_sp_config.load(saml2_config.get("sp_config", {})) + + config_path = saml2_config.get("config_path", None) + if config_path is not None: + self.saml2_sp_config.load_file(config_path) + + def _default_saml_config_dict(self): + import saml2 + + public_baseurl = self.public_baseurl + if public_baseurl is None: + raise ConfigError( + "saml2_config requires a public_baseurl to be set" + ) + + metadata_url = public_baseurl + "_matrix/saml2/metadata.xml" + response_url = public_baseurl + "_matrix/saml2/authn_response" + return { + "entityid": metadata_url, + + "service": { + "sp": { + "endpoints": { + "assertion_consumer_service": [ + (response_url, saml2.BINDING_HTTP_POST), + ], + }, + "required_attributes": ["uid"], + "optional_attributes": ["mail", "surname", "givenname"], + }, + } + } + + def default_config(self, config_dir_path, server_name, **kwargs): + return """ + # Enable SAML2 for registration and login. Uses pysaml2. + # + # saml2_config: + # + # # The following is the configuration for the pysaml2 Service Provider. + # # See pysaml2 docs for format of config. + # # + # # Default values will be used for the 'entityid' and 'service' settings, + # # so it is not normally necessary to specify them unless you need to + # # override them. + # + # sp_config: + # # point this to the IdP's metadata. You can use either a local file or + # # (preferably) a URL. + # metadata: + # # local: ["saml2/idp.xml"] + # remote: + # - url: https://our_idp/metadata.xml + # + # # The following is just used to generate our metadata xml, and you + # # may well not need it, depending on your setup. Alternatively you + # # may need a whole lot more detail - see the pysaml2 docs! + # + # description: ["My awesome SP", "en"] + # name: ["Test SP", "en"] + # + # organization: + # name: Example com + # display_name: + # - ["Example co", "en"] + # url: "http://example.com" + # + # contact_person: + # - given_name: Bob + # sur_name: "the Sysadmin" + # email_address": ["admin@example.com"] + # contact_type": technical + # + # # Instead of putting the config inline as above, you can specify a + # # separate pysaml2 configuration file: + # # + # # config_path: "%(config_dir_path)s/sp_conf.py" + """ % {"config_dir_path": config_dir_path} diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 75ba9947cc..db631e0c6c 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -80,7 +80,10 @@ CONDITIONAL_REQUIREMENTS = { }, "postgres": { "psycopg2>=2.6": ["psycopg2"] - } + }, + "saml2": { + "pysaml2>=4.5.0": ["saml2"], + }, } diff --git a/synapse/rest/saml2/__init__.py b/synapse/rest/saml2/__init__.py new file mode 100644 index 0000000000..68da37ca6a --- /dev/null +++ b/synapse/rest/saml2/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from twisted.web.resource import Resource + +from synapse.rest.saml2.metadata_resource import SAML2MetadataResource +from synapse.rest.saml2.response_resource import SAML2ResponseResource + +logger = logging.getLogger(__name__) + + +class SAML2Resource(Resource): + def __init__(self, hs): + Resource.__init__(self) + self.putChild(b"metadata.xml", SAML2MetadataResource(hs)) + self.putChild(b"authn_response", SAML2ResponseResource(hs)) diff --git a/synapse/rest/saml2/metadata_resource.py b/synapse/rest/saml2/metadata_resource.py new file mode 100644 index 0000000000..e8c680aeb4 --- /dev/null +++ b/synapse/rest/saml2/metadata_resource.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import saml2.metadata + +from twisted.web.resource import Resource + + +class SAML2MetadataResource(Resource): + """A Twisted web resource which renders the SAML metadata""" + + isLeaf = 1 + + def __init__(self, hs): + Resource.__init__(self) + self.sp_config = hs.config.saml2_sp_config + + def render_GET(self, request): + metadata_xml = saml2.metadata.create_metadata_string( + configfile=None, config=self.sp_config, + ) + request.setHeader(b"Content-Type", b"text/xml; charset=utf-8") + return metadata_xml diff --git a/synapse/rest/saml2/response_resource.py b/synapse/rest/saml2/response_resource.py new file mode 100644 index 0000000000..ad2ed157b5 --- /dev/null +++ b/synapse/rest/saml2/response_resource.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +import saml2 +from saml2.client import Saml2Client + +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + +from synapse.api.errors import CodeMessageException +from synapse.http.server import wrap_html_request_handler +from synapse.http.servlet import parse_string +from synapse.rest.client.v1.login import SSOAuthHandler + +logger = logging.getLogger(__name__) + + +class SAML2ResponseResource(Resource): + """A Twisted web resource which handles the SAML response""" + + isLeaf = 1 + + def __init__(self, hs): + Resource.__init__(self) + + self._saml_client = Saml2Client(hs.config.saml2_sp_config) + self._sso_auth_handler = SSOAuthHandler(hs) + + def render_POST(self, request): + self._async_render_POST(request) + return NOT_DONE_YET + + @wrap_html_request_handler + def _async_render_POST(self, request): + resp_bytes = parse_string(request, 'SAMLResponse', required=True) + relay_state = parse_string(request, 'RelayState', required=True) + + try: + saml2_auth = self._saml_client.parse_authn_request_response( + resp_bytes, saml2.BINDING_HTTP_POST, + ) + except Exception as e: + logger.warning("Exception parsing SAML2 response", exc_info=1) + raise CodeMessageException( + 400, "Unable to parse SAML2 response: %s" % (e,), + ) + + if saml2_auth.not_signed: + raise CodeMessageException(400, "SAML2 response was not signed") + + if "uid" not in saml2_auth.ava: + raise CodeMessageException(400, "uid not in SAML2 response") + + username = saml2_auth.ava["uid"][0] + return self._sso_auth_handler.on_successful_auth( + username, request, relay_state, + ) diff --git a/tests/utils.py b/tests/utils.py index 52ab762010..04796a9b30 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -139,6 +139,7 @@ def default_config(name): config.admin_contact = None config.rc_messages_per_second = 10000 config.rc_message_burst_count = 10000 + config.saml2_enabled = False config.use_frozen_dicts = False -- cgit 1.5.1 From df96177ca7f6f646baf96e779e0bf0ef38cc5168 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 11 Dec 2018 13:18:48 +0000 Subject: Stop installing Matrix Console by default This is based on the work done by @krombel in #2601. --- README.rst | 9 +-------- UPGRADE.rst | 4 ++++ changelog.d/4290.feature | 1 + synapse/app/homeserver.py | 43 +++++++++++------------------------------- synapse/config/server.py | 42 ++++++++++++++++++++++++++--------------- synapse/python_dependencies.py | 3 --- 6 files changed, 44 insertions(+), 58 deletions(-) create mode 100644 changelog.d/4290.feature (limited to 'synapse/python_dependencies.py') diff --git a/README.rst b/README.rst index 664a0ea03f..02b3db1bc6 100644 --- a/README.rst +++ b/README.rst @@ -289,10 +289,6 @@ go back in your web client and proceed further. If all goes well you should at least be able to log in, create a room, and start sending messages. -(The homeserver runs a web client by default at https://localhost:8448/, though -as of the time of writing it is somewhat outdated and not really recommended - -https://github.com/matrix-org/synapse/issues/1527). - .. _`client-user-reg`: Registering a new user from a client @@ -372,10 +368,7 @@ ArchLinux The quickest way to get up and running with ArchLinux is probably with the community package https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of -the necessary dependencies. If the default web client is to be served (enabled by default in -the generated config), -https://www.archlinux.org/packages/community/any/python2-matrix-angular-sdk/ will also need to -be installed. +the necessary dependencies. pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: diff --git a/UPGRADE.rst b/UPGRADE.rst index 9d68a64058..95f4e33300 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -115,6 +115,10 @@ Upgrading to v0.34.0 case a hypothetical future identity server was put there. If you don't remove it, users may be unable to deactivate their accounts. +3. This release no longer installs the (unmaintained) Matrix Console web client + as part of the default installation. It is possible to re-enable it by + installing it separately and setting the ``web_client_location`` config + option, but please consider switching to another client. Upgrading to v0.33.7 ==================== diff --git a/changelog.d/4290.feature b/changelog.d/4290.feature new file mode 100644 index 0000000000..df5114bae9 --- /dev/null +++ b/changelog.d/4290.feature @@ -0,0 +1 @@ +Remove Matrix console from the default distribution diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 3e4dea2f19..a6af553f79 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -54,7 +54,7 @@ from synapse.metrics import RegistryProxy from synapse.metrics.background_process_metrics import run_as_background_process from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.module_api import ModuleApi -from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, check_requirements +from synapse.python_dependencies import check_requirements from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.rest import ClientRestResource @@ -79,36 +79,6 @@ def gz_wrap(r): return EncodingResourceWrapper(r, [GzipEncoderFactory()]) -def build_resource_for_web_client(hs): - webclient_path = hs.get_config().web_client_location - if not webclient_path: - try: - import syweb - except ImportError: - quit_with_error( - "Could not find a webclient.\n\n" - "Please either install the matrix-angular-sdk or configure\n" - "the location of the source to serve via the configuration\n" - "option `web_client_location`\n\n" - "To install the `matrix-angular-sdk` via pip, run:\n\n" - " pip install '%(dep)s'\n" - "\n" - "You can also disable hosting of the webclient via the\n" - "configuration option `web_client`\n" - % {"dep": CONDITIONAL_REQUIREMENTS["web_client"].keys()[0]} - ) - syweb_path = os.path.dirname(syweb.__file__) - webclient_path = os.path.join(syweb_path, "webclient") - # GZip is disabled here due to - # https://twistedmatrix.com/trac/ticket/7678 - # (It can stay enabled for the API resources: they call - # write() with the whole body and then finish() straight - # after and so do not trigger the bug. - # GzipFile was removed in commit 184ba09 - # return GzipFile(webclient_path) # TODO configurable? - return File(webclient_path) # TODO configurable? - - class SynapseHomeServer(HomeServer): DATASTORE_CLASS = DataStore @@ -237,7 +207,16 @@ class SynapseHomeServer(HomeServer): resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) if name == "webclient": - resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self) + webclient_path = self.get_config().web_client_location + + if webclient_path is None: + logger.warning( + "Not enabling webclient resource, as web_client_location is unset." + ) + else: + # GZip is disabled here due to + # https://twistedmatrix.com/trac/ticket/7678 + resources[WEB_CLIENT_PREFIX] = File(webclient_path) if name == "metrics" and self.get_config().enable_metrics: resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) diff --git a/synapse/config/server.py b/synapse/config/server.py index 4a5b902f8e..a9154ad462 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -34,7 +34,6 @@ class ServerConfig(Config): raise ConfigError(str(e)) self.pid_file = self.abspath(config.get("pid_file")) - self.web_client = config["web_client"] self.web_client_location = config.get("web_client_location", None) self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") @@ -128,6 +127,9 @@ class ServerConfig(Config): elif not bind_addresses: bind_addresses.append('') + if not self.web_client_location: + _warn_if_webclient_configured(self.listeners) + self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None)) bind_port = config.get("bind_port") @@ -136,8 +138,6 @@ class ServerConfig(Config): bind_host = config.get("bind_host", "") gzip_responses = config.get("gzip_responses", True) - names = ["client", "webclient"] if self.web_client else ["client"] - self.listeners.append({ "port": bind_port, "bind_addresses": [bind_host], @@ -145,7 +145,7 @@ class ServerConfig(Config): "type": "http", "resources": [ { - "names": names, + "names": ["client"], "compress": gzip_responses, }, { @@ -164,7 +164,7 @@ class ServerConfig(Config): "type": "http", "resources": [ { - "names": names, + "names": ["client"], "compress": gzip_responses, }, { @@ -247,13 +247,9 @@ class ServerConfig(Config): # # cpu_affinity: 0xFFFFFFFF - # Whether to serve a web client from the HTTP/HTTPS root resource. - web_client: True - - # The root directory to server for the above web client. - # If left undefined, synapse will serve the matrix-angular-sdk web client. - # Make sure matrix-angular-sdk is installed with pip if web_client is True - # and web_client_location is undefined + # The path to the web client which will be served at /_matrix/client/ + # if 'webclient' is configured under the 'listeners' configuration. + # # web_client_location: "/path/to/web/root" # The public-facing base URL for the client API (not including _matrix/...) @@ -320,8 +316,8 @@ class ServerConfig(Config): - # List of resources to host on this listener. names: - - client # The client-server APIs, both v1 and v2 - - webclient # The bundled webclient. + - client # The client-server APIs, both v1 and v2 + # - webclient # A web client. Requires web_client_location to be set. # Should synapse compress HTTP responses to clients that support it? # This should be disabled if running synapse behind a load balancer @@ -348,7 +344,7 @@ class ServerConfig(Config): x_forwarded: false resources: - - names: [client, webclient] + - names: [client] compress: true - names: [federation] compress: false @@ -452,3 +448,19 @@ def read_gc_thresholds(thresholds): raise ConfigError( "Value of `gc_threshold` must be a list of three integers if set" ) + + +NO_MORE_WEB_CLIENT_WARNING = """ +Synapse no longer includes a web client. To enable a web client, configure +web_client_location. To remove this warning, remove 'webclient' from the 'listeners' +configuration. +""" + + +def _warn_if_webclient_configured(listeners): + for listener in listeners: + for res in listener.get("resources", []): + for name in res.get("names", []): + if name == 'webclient': + logger.warning(NO_MORE_WEB_CLIENT_WARNING) + return diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ca62ee7637..92422c6ffc 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -69,9 +69,6 @@ REQUIREMENTS = { } CONDITIONAL_REQUIREMENTS = { - "web_client": { - "matrix_angular_sdk>=0.6.8": ["syweb>=0.6.8"], - }, "email.enable_notifs": { "Jinja2>=2.8": ["Jinja2>=2.8"], "bleach>=1.4.2": ["bleach>=1.4.2"], -- cgit 1.5.1 From c26f49a6645a84730811ea7bc5158d826bc43484 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 22 Dec 2018 01:37:26 +1100 Subject: Make the dependencies more like a standard Python project and hook up the optional dependencies to setuptools (#4298) --- README.rst | 11 +- changelog.d/4298.feature | 1 + setup.py | 16 ++- synapse/app/__init__.py | 6 +- synapse/app/homeserver.py | 3 - synapse/python_dependencies.py | 228 ++++++++++++++++------------------------- tox.ini | 4 +- 7 files changed, 111 insertions(+), 158 deletions(-) create mode 100644 changelog.d/4298.feature (limited to 'synapse/python_dependencies.py') diff --git a/README.rst b/README.rst index 02b3db1bc6..c4df779ac7 100644 --- a/README.rst +++ b/README.rst @@ -86,7 +86,7 @@ Synapse is the reference Python/Twisted Matrix homeserver implementation. System requirements: - POSIX-compliant system (tested on Linux & OS X) -- Python 3.5, 3.6, or 2.7 +- Python 3.5, 3.6, 3.7, or 2.7 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org Installing from source @@ -148,7 +148,7 @@ To install the Synapse homeserver run:: source ~/synapse/env/bin/activate pip install --upgrade pip pip install --upgrade setuptools - pip install matrix-synapse + pip install matrix-synapse[all] This installs Synapse, along with the libraries it uses, into a virtual environment under ``~/synapse/env``. Feel free to pick a different directory @@ -158,7 +158,7 @@ This Synapse installation can then be later upgraded by using pip again with the update flag:: source ~/synapse/env/bin/activate - pip install -U matrix-synapse + pip install -U matrix-synapse[all] In case of problems, please see the _`Troubleshooting` section below. @@ -826,8 +826,7 @@ to install using pip and a virtualenv:: virtualenv -p python2.7 env source env/bin/activate - python -m synapse.python_dependencies | xargs pip install - pip install lxml mock + python -m pip install -e .[all] This will run a process of downloading and installing all the needed dependencies into a virtual env. @@ -835,7 +834,7 @@ dependencies into a virtual env. Once this is done, you may wish to run Synapse's unit tests, to check that everything is installed as it should be:: - PYTHONPATH="." trial tests + python -m twisted.trial tests This should end with a 'PASSED' result:: diff --git a/changelog.d/4298.feature b/changelog.d/4298.feature new file mode 100644 index 0000000000..05ad70fe72 --- /dev/null +++ b/changelog.d/4298.feature @@ -0,0 +1 @@ +Synapse can now have its conditional/extra dependencies installed by pip. This functionality can be used by using `pip install matrix-synapse[feature]`, where feature is a comma separated list with the possible values "email.enable_notifs", "ldap3", "postgres", "saml2", "url_preview", and "test". If you want to install all optional dependencies, you can use "all" instead. diff --git a/setup.py b/setup.py index 00b69c43f5..55b1b10a77 100755 --- a/setup.py +++ b/setup.py @@ -84,13 +84,25 @@ version = exec_file(("synapse", "__init__.py"))["__version__"] dependencies = exec_file(("synapse", "python_dependencies.py")) long_description = read_file(("README.rst",)) +REQUIREMENTS = dependencies['REQUIREMENTS'] +CONDITIONAL_REQUIREMENTS = dependencies['CONDITIONAL_REQUIREMENTS'] + +# Make `pip install matrix-synapse[all]` install all the optional dependencies. +ALL_OPTIONAL_REQUIREMENTS = set() + +for optional_deps in CONDITIONAL_REQUIREMENTS.values(): + ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS + +CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS) + + setup( name="matrix-synapse", version=version, packages=find_packages(exclude=["tests", "tests.*"]), description="Reference homeserver for the Matrix decentralised comms protocol", - install_requires=dependencies['requirements'](include_conditional=True).keys(), - dependency_links=dependencies["DEPENDENCY_LINKS"].values(), + install_requires=REQUIREMENTS, + extras_require=CONDITIONAL_REQUIREMENTS, include_package_data=True, zip_safe=False, long_description=long_description, diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index c3afcc573b..233bf43fc8 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -22,11 +22,11 @@ sys.dont_write_bytecode = True try: python_dependencies.check_requirements() -except python_dependencies.MissingRequirementError as e: +except python_dependencies.DependencyException as e: message = "\n".join([ - "Missing Requirement: %s" % (str(e),), + "Missing Requirements: %s" % (", ".join(e.dependencies),), "To install run:", - " pip install --upgrade --force \"%s\"" % (e.dependency,), + " pip install --upgrade --force %s" % (" ".join(e.dependencies),), "", ]) sys.stderr.writelines(message) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f2064f9d0c..f3ac3d19f0 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -322,9 +322,6 @@ def setup(config_options): synapse.config.logger.setup_logging(config, use_worker_options=False) - # check any extra requirements we have now we have a config - check_requirements(config) - events.USE_FROZEN_DICTS = config.use_frozen_dicts tls_server_context_factory = context_factory.ServerContextFactory(config) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 72a92cc462..2c65ef5856 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -15,175 +15,121 @@ # limitations under the License. import logging -from distutils.version import LooseVersion + +from pkg_resources import DistributionNotFound, VersionConflict, get_distribution logger = logging.getLogger(__name__) -# this dict maps from python package name to a list of modules we expect it to -# provide. -# -# the key is a "requirement specifier", as used as a parameter to `pip -# install`[1], or an `install_requires` argument to `setuptools.setup` [2]. + +# REQUIREMENTS is a simple list of requirement specifiers[1], and must be +# installed. It is passed to setup() as install_requires in setup.py. # -# the value is a sequence of strings; each entry should be the name of the -# python module, optionally followed by a version assertion which can be either -# ">=" or "==". +# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict +# of lists. The dict key is the optional dependency name and can be passed to +# pip when installing. The list is a series of requirement specifiers[1] to be +# installed when that optional dependency requirement is specified. It is passed +# to setup() as extras_require in setup.py # # [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers. -# [2] https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-dependencies -REQUIREMENTS = { - "jsonschema>=2.5.1": ["jsonschema>=2.5.1"], - "frozendict>=1": ["frozendict"], - "unpaddedbase64>=1.1.0": ["unpaddedbase64>=1.1.0"], - "canonicaljson>=1.1.3": ["canonicaljson>=1.1.3"], - "signedjson>=1.0.0": ["signedjson>=1.0.0"], - "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"], - "service_identity>=16.0.0": ["service_identity>=16.0.0"], - "Twisted>=17.1.0": ["twisted>=17.1.0"], - "treq>=15.1": ["treq>=15.1"], +REQUIREMENTS = [ + "jsonschema>=2.5.1", + "frozendict>=1", + "unpaddedbase64>=1.1.0", + "canonicaljson>=1.1.3", + "signedjson>=1.0.0", + "pynacl>=1.2.1", + "service_identity>=16.0.0", + "Twisted>=17.1.0", + "treq>=15.1", # Twisted has required pyopenssl 16.0 since about Twisted 16.6. - "pyopenssl>=16.0.0": ["OpenSSL>=16.0.0"], - - "pyyaml>=3.11": ["yaml"], - "pyasn1>=0.1.9": ["pyasn1"], - "pyasn1-modules>=0.0.7": ["pyasn1_modules"], - "daemonize>=2.3.1": ["daemonize"], - "bcrypt>=3.1.0": ["bcrypt>=3.1.0"], - "pillow>=3.1.2": ["PIL"], - "sortedcontainers>=1.4.4": ["sortedcontainers"], - "psutil>=2.0.0": ["psutil>=2.0.0"], - "pymacaroons-pynacl>=0.9.3": ["pymacaroons"], - "msgpack-python>=0.4.2": ["msgpack"], - "phonenumbers>=8.2.0": ["phonenumbers"], - "six>=1.10": ["six"], - + "pyopenssl>=16.0.0", + "pyyaml>=3.11", + "pyasn1>=0.1.9", + "pyasn1-modules>=0.0.7", + "daemonize>=2.3.1", + "bcrypt>=3.1.0", + "pillow>=3.1.2", + "sortedcontainers>=1.4.4", + "psutil>=2.0.0", + "pymacaroons-pynacl>=0.9.3", + "msgpack-python>=0.4.2", + "phonenumbers>=8.2.0", + "six>=1.10", # prometheus_client 0.4.0 changed the format of counter metrics # (cf https://github.com/matrix-org/synapse/issues/4001) - "prometheus_client>=0.0.18,<0.4.0": ["prometheus_client"], - + "prometheus_client>=0.0.18,<0.4.0", # we use attr.s(slots), which arrived in 16.0.0 - "attrs>=16.0.0": ["attr>=16.0.0"], - "netaddr>=0.7.18": ["netaddr"], -} + "attrs>=16.0.0", + "netaddr>=0.7.18", +] CONDITIONAL_REQUIREMENTS = { - "email.enable_notifs": { - "Jinja2>=2.8": ["Jinja2>=2.8"], - "bleach>=1.4.2": ["bleach>=1.4.2"], - }, - "matrix-synapse-ldap3": { - "matrix-synapse-ldap3>=0.1": ["ldap_auth_provider"], - }, - "postgres": { - "psycopg2>=2.6": ["psycopg2"] - }, - "saml2": { - "pysaml2>=4.5.0": ["saml2"], - }, + "email.enable_notifs": ["Jinja2>=2.8", "bleach>=1.4.2"], + "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], + "postgres": ["psycopg2>=2.6"], + "saml2": ["pysaml2>=4.5.0"], + "url_preview": ["lxml>=3.5.0"], + "test": ["mock>=2.0"], } -def requirements(config=None, include_conditional=False): - reqs = REQUIREMENTS.copy() - if include_conditional: - for _, req in CONDITIONAL_REQUIREMENTS.items(): - reqs.update(req) - return reqs +def list_requirements(): + deps = set(REQUIREMENTS) + for opt in CONDITIONAL_REQUIREMENTS.values(): + deps = set(opt) | deps + return list(deps) -def github_link(project, version, egg): - return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) +class DependencyException(Exception): + @property + def dependencies(self): + for i in self.args[0]: + yield '"' + i + '"' -DEPENDENCY_LINKS = { -} +def check_requirements(_get_distribution=get_distribution): + + deps_needed = [] + errors = [] -class MissingRequirementError(Exception): - def __init__(self, message, module_name, dependency): - super(MissingRequirementError, self).__init__(message) - self.module_name = module_name - self.dependency = dependency - - -def check_requirements(config=None): - """Checks that all the modules needed by synapse have been correctly - installed and are at the correct version""" - for dependency, module_requirements in ( - requirements(config, include_conditional=False).items()): - for module_requirement in module_requirements: - if ">=" in module_requirement: - module_name, required_version = module_requirement.split(">=") - version_test = ">=" - elif "==" in module_requirement: - module_name, required_version = module_requirement.split("==") - version_test = "==" - else: - module_name = module_requirement - version_test = None - - try: - module = __import__(module_name) - except ImportError: - logging.exception( - "Can't import %r which is part of %r", - module_name, dependency - ) - raise MissingRequirementError( - "Can't import %r which is part of %r" - % (module_name, dependency), module_name, dependency - ) - version = getattr(module, "__version__", None) - file_path = getattr(module, "__file__", None) - logger.info( - "Using %r version %r from %r to satisfy %r", - module_name, version, file_path, dependency + # Check the base dependencies exist -- they all must be installed. + for dependency in REQUIREMENTS: + try: + _get_distribution(dependency) + except VersionConflict as e: + deps_needed.append(dependency) + errors.append( + "Needed %s, got %s==%s" + % (dependency, e.dist.project_name, e.dist.version) ) + except DistributionNotFound: + deps_needed.append(dependency) + errors.append("Needed %s but it was not installed" % (dependency,)) - if version_test == ">=": - if version is None: - raise MissingRequirementError( - "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name), module_name, dependency - ) - if LooseVersion(version) < LooseVersion(required_version): - raise MissingRequirementError( - "Version of %r in %r is too old. %r < %r" - % (dependency, file_path, version, required_version), - module_name, dependency - ) - elif version_test == "==": - if version is None: - raise MissingRequirementError( - "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name), module_name, dependency - ) - if LooseVersion(version) != LooseVersion(required_version): - raise MissingRequirementError( - "Unexpected version of %r in %r. %r != %r" - % (dependency, file_path, version, required_version), - module_name, dependency - ) + # Check the optional dependencies are up to date. We allow them to not be + # installed. + OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), []) + for dependency in OPTS: + try: + _get_distribution(dependency) + except VersionConflict: + deps_needed.append(dependency) + errors.append("Needed %s but it was not installed" % (dependency,)) + except DistributionNotFound: + # If it's not found, we don't care + pass -def list_requirements(): - result = [] - linked = [] - for link in DEPENDENCY_LINKS.values(): - egg = link.split("#egg=")[1] - linked.append(egg.split('-')[0]) - result.append(link) - for requirement in requirements(include_conditional=True): - is_linked = False - for link in linked: - if requirement.replace('-', '_').startswith(link): - is_linked = True - if not is_linked: - result.append(requirement) - return result + if deps_needed: + for e in errors: + logging.exception(e) + + raise DependencyException(deps_needed) if __name__ == "__main__": import sys + sys.stdout.writelines(req + "\n" for req in list_requirements()) diff --git a/tox.ini b/tox.ini index f5f07cb690..a0f5486829 100644 --- a/tox.ini +++ b/tox.ini @@ -9,9 +9,6 @@ deps = junitxml coverage - # needed by some of the tests - lxml - # cyptography 2.2 requires setuptools >= 18.5 # # older versions of virtualenv (?) give us a virtualenv with the same @@ -33,6 +30,7 @@ setenv = [testenv] deps = {[base]deps} +extras = all whitelist_externals = sh -- cgit 1.5.1 From b7c021881293c60ad0874f6d15375a9b7a40c22c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 7 Jan 2019 10:14:31 +0000 Subject: Check jinja version for consent resource (#4327) * Raise a ConfigError if an invalid resource is specified * Require Jinja 2.9 for the consent resource * changelog --- changelog.d/4327.bugfix | 1 + synapse/app/__init__.py | 9 +------ synapse/config/server.py | 38 ++++++++++++++++++++++++++++- synapse/python_dependencies.py | 55 +++++++++++++++++++++++++++--------------- 4 files changed, 75 insertions(+), 28 deletions(-) create mode 100644 changelog.d/4327.bugfix (limited to 'synapse/python_dependencies.py') diff --git a/changelog.d/4327.bugfix b/changelog.d/4327.bugfix new file mode 100644 index 0000000000..03bf05e05c --- /dev/null +++ b/changelog.d/4327.bugfix @@ -0,0 +1 @@ +Check jinja version for consent resource diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index 233bf43fc8..b45adafdd3 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -19,15 +19,8 @@ from synapse import python_dependencies # noqa: E402 sys.dont_write_bytecode = True - try: python_dependencies.check_requirements() except python_dependencies.DependencyException as e: - message = "\n".join([ - "Missing Requirements: %s" % (", ".join(e.dependencies),), - "To install run:", - " pip install --upgrade --force %s" % (" ".join(e.dependencies),), - "", - ]) - sys.stderr.writelines(message) + sys.stderr.writelines(e.message) sys.exit(1) diff --git a/synapse/config/server.py b/synapse/config/server.py index 120c2b81fc..fb57791098 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd -# Copyright 2017 New Vector Ltd +# Copyright 2017-2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import logging import os.path from synapse.http.endpoint import parse_and_validate_server_name +from synapse.python_dependencies import DependencyException, check_requirements from ._base import Config, ConfigError @@ -204,6 +205,8 @@ class ServerConfig(Config): ] }) + _check_resource_config(self.listeners) + def default_config(self, server_name, data_dir_path, **kwargs): _, bind_port = parse_and_validate_server_name(server_name) if bind_port is not None: @@ -465,3 +468,36 @@ def _warn_if_webclient_configured(listeners): if name == 'webclient': logger.warning(NO_MORE_WEB_CLIENT_WARNING) return + + +KNOWN_RESOURCES = ( + 'client', + 'consent', + 'federation', + 'keys', + 'media', + 'metrics', + 'replication', + 'static', + 'webclient', +) + + +def _check_resource_config(listeners): + resource_names = set( + res_name + for listener in listeners + for res in listener.get("resources", []) + for res_name in res.get("names", []) + ) + + for resource in resource_names: + if resource not in KNOWN_RESOURCES: + raise ConfigError( + "Unknown listener resource '%s'" % (resource, ) + ) + if resource == "consent": + try: + check_requirements('resources.consent') + except DependencyException as e: + raise ConfigError(e.message) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 2c65ef5856..69c5f9fe2e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -65,9 +65,13 @@ REQUIREMENTS = [ ] CONDITIONAL_REQUIREMENTS = { - "email.enable_notifs": ["Jinja2>=2.8", "bleach>=1.4.2"], + "email.enable_notifs": ["Jinja2>=2.9", "bleach>=1.4.2"], "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], "postgres": ["psycopg2>=2.6"], + + # ConsentResource uses select_autoescape, which arrived in jinja 2.9 + "resources.consent": ["Jinja2>=2.9"], + "saml2": ["pysaml2>=4.5.0"], "url_preview": ["lxml>=3.5.0"], "test": ["mock>=2.0"], @@ -83,19 +87,31 @@ def list_requirements(): class DependencyException(Exception): + @property + def message(self): + return "\n".join([ + "Missing Requirements: %s" % (", ".join(self.dependencies),), + "To install run:", + " pip install --upgrade --force %s" % (" ".join(self.dependencies),), + "", + ]) + @property def dependencies(self): for i in self.args[0]: yield '"' + i + '"' -def check_requirements(_get_distribution=get_distribution): - +def check_requirements(for_feature=None, _get_distribution=get_distribution): deps_needed = [] errors = [] - # Check the base dependencies exist -- they all must be installed. - for dependency in REQUIREMENTS: + if for_feature: + reqs = CONDITIONAL_REQUIREMENTS[for_feature] + else: + reqs = REQUIREMENTS + + for dependency in reqs: try: _get_distribution(dependency) except VersionConflict as e: @@ -108,23 +124,24 @@ def check_requirements(_get_distribution=get_distribution): deps_needed.append(dependency) errors.append("Needed %s but it was not installed" % (dependency,)) - # Check the optional dependencies are up to date. We allow them to not be - # installed. - OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), []) - - for dependency in OPTS: - try: - _get_distribution(dependency) - except VersionConflict: - deps_needed.append(dependency) - errors.append("Needed %s but it was not installed" % (dependency,)) - except DistributionNotFound: - # If it's not found, we don't care - pass + if not for_feature: + # Check the optional dependencies are up to date. We allow them to not be + # installed. + OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), []) + + for dependency in OPTS: + try: + _get_distribution(dependency) + except VersionConflict: + deps_needed.append(dependency) + errors.append("Needed %s but it was not installed" % (dependency,)) + except DistributionNotFound: + # If it's not found, we don't care + pass if deps_needed: for e in errors: - logging.exception(e) + logging.error(e) raise DependencyException(deps_needed) -- cgit 1.5.1 From fab948120f88e86df73d6bdd46b22bc5bc8583f9 Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Wed, 16 Jan 2019 09:41:29 +0100 Subject: Use msgpack instead of msgpack-python The package msgpack-python has been deprecated. Signed-off-by: Andrej Shadura --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/python_dependencies.py') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 69c5f9fe2e..c7dd4917d8 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -53,7 +53,7 @@ REQUIREMENTS = [ "sortedcontainers>=1.4.4", "psutil>=2.0.0", "pymacaroons-pynacl>=0.9.3", - "msgpack-python>=0.4.2", + "msgpack>=0.5.0", "phonenumbers>=8.2.0", "six>=1.10", # prometheus_client 0.4.0 changed the format of counter metrics -- cgit 1.5.1 From 64cf6788d91dbd2a1a3b3613881e9c5609dc10af Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Wed, 16 Jan 2019 09:44:22 +0100 Subject: Depend on pymacaroons >= 0.13.0 instead on pymacaroons-pynacl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since 0.13.0, pymacaroons works correctly with pynacl, so there isn’t any more reason to depend on an outdated pynacl fork. Signed-off-by: Andrej Shadura --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/python_dependencies.py') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index c7dd4917d8..2fe790f95d 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -52,7 +52,7 @@ REQUIREMENTS = [ "pillow>=3.1.2", "sortedcontainers>=1.4.4", "psutil>=2.0.0", - "pymacaroons-pynacl>=0.9.3", + "pymacaroons>=0.13.0", "msgpack>=0.5.0", "phonenumbers>=8.2.0", "six>=1.10", -- cgit 1.5.1 From 676cf2ee267cedd05109678dece0227fd1801423 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 17 Jan 2019 14:00:23 +0000 Subject: Fix incorrect logcontexts after a Deferred was cancelled (#4407) --- changelog.d/4407.bugfix | 1 + synapse/python_dependencies.py | 11 ++++- synapse/util/async_helpers.py | 4 +- tests/util/test_async_utils.py | 104 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4407.bugfix create mode 100644 tests/util/test_async_utils.py (limited to 'synapse/python_dependencies.py') diff --git a/changelog.d/4407.bugfix b/changelog.d/4407.bugfix new file mode 100644 index 0000000000..54c5e76d1f --- /dev/null +++ b/changelog.d/4407.bugfix @@ -0,0 +1 @@ +Fix incorrect logcontexts after a Deferred was cancelled diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 2fe790f95d..882e844eb1 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -40,7 +40,11 @@ REQUIREMENTS = [ "signedjson>=1.0.0", "pynacl>=1.2.1", "service_identity>=16.0.0", - "Twisted>=17.1.0", + + # our logcontext handling relies on the ability to cancel inlineCallbacks + # (https://twistedmatrix.com/trac/ticket/4632) which landed in Twisted 18.7. + "Twisted>=18.7.0", + "treq>=15.1", # Twisted has required pyopenssl 16.0 since about Twisted 16.6. "pyopenssl>=16.0.0", @@ -59,8 +63,11 @@ REQUIREMENTS = [ # prometheus_client 0.4.0 changed the format of counter metrics # (cf https://github.com/matrix-org/synapse/issues/4001) "prometheus_client>=0.0.18,<0.4.0", + # we use attr.s(slots), which arrived in 16.0.0 - "attrs>=16.0.0", + # Twisted 18.7.0 requires attrs>=17.4.0 + "attrs>=17.4.0", + "netaddr>=0.7.18", ] diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index ec7b2c9672..430bb15f51 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -387,12 +387,14 @@ def timeout_deferred(deferred, timeout, reactor, on_timeout_cancel=None): deferred that wraps and times out the given deferred, correctly handling the case where the given deferred's canceller throws. + (See https://twistedmatrix.com/trac/ticket/9534) + NOTE: Unlike `Deferred.addTimeout`, this function returns a new deferred Args: deferred (Deferred) timeout (float): Timeout in seconds - reactor (twisted.internet.reactor): The twisted reactor to use + reactor (twisted.interfaces.IReactorTime): The twisted reactor to use on_timeout_cancel (callable): A callable which is called immediately after the deferred times out, and not if this deferred is otherwise cancelled before the timeout. diff --git a/tests/util/test_async_utils.py b/tests/util/test_async_utils.py new file mode 100644 index 0000000000..84dd71e47a --- /dev/null +++ b/tests/util/test_async_utils.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from twisted.internet import defer +from twisted.internet.defer import CancelledError, Deferred +from twisted.internet.task import Clock + +from synapse.util import logcontext +from synapse.util.async_helpers import timeout_deferred +from synapse.util.logcontext import LoggingContext + +from tests.unittest import TestCase + + +class TimeoutDeferredTest(TestCase): + def setUp(self): + self.clock = Clock() + + def test_times_out(self): + """Basic test case that checks that the original deferred is cancelled and that + the timing-out deferred is errbacked + """ + cancelled = [False] + + def canceller(_d): + cancelled[0] = True + + non_completing_d = Deferred(canceller) + timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock) + + self.assertNoResult(timing_out_d) + self.assertFalse(cancelled[0], "deferred was cancelled prematurely") + + self.clock.pump((1.0, )) + + self.assertTrue(cancelled[0], "deferred was not cancelled by timeout") + self.failureResultOf(timing_out_d, defer.TimeoutError, ) + + def test_times_out_when_canceller_throws(self): + """Test that we have successfully worked around + https://twistedmatrix.com/trac/ticket/9534""" + + def canceller(_d): + raise Exception("can't cancel this deferred") + + non_completing_d = Deferred(canceller) + timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock) + + self.assertNoResult(timing_out_d) + + self.clock.pump((1.0, )) + + self.failureResultOf(timing_out_d, defer.TimeoutError, ) + + def test_logcontext_is_preserved_on_cancellation(self): + blocking_was_cancelled = [False] + + @defer.inlineCallbacks + def blocking(): + non_completing_d = Deferred() + with logcontext.PreserveLoggingContext(): + try: + yield non_completing_d + except CancelledError: + blocking_was_cancelled[0] = True + raise + + with logcontext.LoggingContext("one") as context_one: + # the errbacks should be run in the test logcontext + def errback(res, deferred_name): + self.assertIs( + LoggingContext.current_context(), context_one, + "errback %s run in unexpected logcontext %s" % ( + deferred_name, LoggingContext.current_context(), + ) + ) + return res + + original_deferred = blocking() + original_deferred.addErrback(errback, "orig") + timing_out_d = timeout_deferred(original_deferred, 1.0, self.clock) + self.assertNoResult(timing_out_d) + self.assertIs(LoggingContext.current_context(), LoggingContext.sentinel) + timing_out_d.addErrback(errback, "timingout") + + self.clock.pump((1.0, )) + + self.assertTrue( + blocking_was_cancelled[0], + "non-completing deferred was not cancelled", + ) + self.failureResultOf(timing_out_d, defer.TimeoutError, ) + self.assertIs(LoggingContext.current_context(), context_one) -- cgit 1.5.1 From 899e60be80736e3b747eb567171296829e96f716 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:55:37 +0200 Subject: Add parameterized Python module to test dependencies Allows running parameterized tests. BSD license. Signed-off-by: Jason Robinson --- synapse/python_dependencies.py | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse/python_dependencies.py') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 882e844eb1..b5555fbba9 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -81,7 +81,7 @@ CONDITIONAL_REQUIREMENTS = { "saml2": ["pysaml2>=4.5.0"], "url_preview": ["lxml>=3.5.0"], - "test": ["mock>=2.0"], + "test": ["mock>=2.0", "parameterized"], } diff --git a/tox.ini b/tox.ini index a0f5486829..9b6cb2036c 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ deps = python-subunit junitxml coverage + parameterized # cyptography 2.2 requires setuptools >= 18.5 # -- cgit 1.5.1 From 6129e52f437c2e03b711453434924e170f3d11bf Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 23 Jan 2019 19:39:06 +1100 Subject: Support ACME for certificate provisioning (#4384) --- changelog.d/4384.feature | 1 + scripts-dev/build_debian_packages | 2 +- synapse/app/homeserver.py | 56 ++++++++++++--- synapse/config/_base.py | 4 +- synapse/config/tls.py | 115 ++++++++++++++++++++++------- synapse/handlers/acme.py | 147 ++++++++++++++++++++++++++++++++++++++ synapse/python_dependencies.py | 4 ++ synapse/server.py | 5 ++ 8 files changed, 298 insertions(+), 36 deletions(-) create mode 100644 changelog.d/4384.feature create mode 100644 synapse/handlers/acme.py (limited to 'synapse/python_dependencies.py') diff --git a/changelog.d/4384.feature b/changelog.d/4384.feature new file mode 100644 index 0000000000..daedcd58c4 --- /dev/null +++ b/changelog.d/4384.feature @@ -0,0 +1 @@ +Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt). diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages index 577d93e6f6..6b9be99060 100755 --- a/scripts-dev/build_debian_packages +++ b/scripts-dev/build_debian_packages @@ -10,12 +10,12 @@ # can be passed on the commandline for debugging. import argparse -from concurrent.futures import ThreadPoolExecutor import os import signal import subprocess import sys import threading +from concurrent.futures import ThreadPoolExecutor DISTS = ( "debian:stretch", diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f3ac3d19f0..ffc49d77cc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -13,10 +13,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import gc import logging import os import sys +import traceback from six import iteritems @@ -324,17 +326,12 @@ def setup(config_options): events.USE_FROZEN_DICTS = config.use_frozen_dicts - tls_server_context_factory = context_factory.ServerContextFactory(config) - tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config) - database_engine = create_engine(config.database_config) config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection hs = SynapseHomeServer( config.server_name, db_config=config.database_config, - tls_server_context_factory=tls_server_context_factory, - tls_client_options_factory=tls_client_options_factory, config=config, version_string="Synapse/" + get_version_string(synapse), database_engine=database_engine, @@ -361,12 +358,53 @@ def setup(config_options): logger.info("Database prepared in %s.", config.database_config['name']) hs.setup() - hs.start_listening() + @defer.inlineCallbacks def start(): - hs.get_pusherpool().start() - hs.get_datastore().start_profiling() - hs.get_datastore().start_doing_background_updates() + try: + # Check if the certificate is still valid. + cert_days_remaining = hs.config.is_disk_cert_valid() + + if hs.config.acme_enabled: + # If ACME is enabled, we might need to provision a certificate + # before starting. + acme = hs.get_acme_handler() + + # Start up the webservices which we will respond to ACME + # challenges with. + yield acme.start_listening() + + # We want to reprovision if cert_days_remaining is None (meaning no + # certificate exists), or the days remaining number it returns + # is less than our re-registration threshold. + if (cert_days_remaining is None) or ( + not cert_days_remaining > hs.config.acme_reprovision_threshold + ): + yield acme.provision_certificate() + + # Read the certificate from disk and build the context factories for + # TLS. + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + config + ) + + # It is now safe to start your Synapse. + hs.start_listening() + hs.get_pusherpool().start() + hs.get_datastore().start_profiling() + hs.get_datastore().start_doing_background_updates() + except Exception as e: + # If a DeferredList failed (like in listening on the ACME listener), + # we need to print the subfailure explicitly. + if isinstance(e, defer.FirstError): + e.subFailure.printTraceback(sys.stderr) + sys.exit(1) + + # Something else went wrong when starting. Print it and bail out. + traceback.print_exc(file=sys.stderr) + sys.exit(1) reactor.callWhenRunning(start) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index fd2d6d52ef..5858fb92b4 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -367,7 +367,7 @@ class Config(object): if not keys_directory: keys_directory = os.path.dirname(config_files[-1]) - config_dir_path = os.path.abspath(keys_directory) + self.config_dir_path = os.path.abspath(keys_directory) specified_config = {} for config_file in config_files: @@ -379,7 +379,7 @@ class Config(object): server_name = specified_config["server_name"] config_string = self.generate_config( - config_dir_path=config_dir_path, + config_dir_path=self.config_dir_path, data_dir_path=os.getcwd(), server_name=server_name, generate_secrets=False, diff --git a/synapse/config/tls.py b/synapse/config/tls.py index bb8952c672..a75e233aa0 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -13,60 +13,110 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os +from datetime import datetime from hashlib import sha256 from unpaddedbase64 import encode_base64 from OpenSSL import crypto -from ._base import Config +from synapse.config._base import Config + +logger = logging.getLogger() class TlsConfig(Config): def read_config(self, config): - self.tls_certificate = self.read_tls_certificate( - config.get("tls_certificate_path") - ) - self.tls_certificate_file = config.get("tls_certificate_path") + acme_config = config.get("acme", {}) + self.acme_enabled = acme_config.get("enabled", False) + self.acme_url = acme_config.get( + "url", "https://acme-v01.api.letsencrypt.org/directory" + ) + self.acme_port = acme_config.get("port", 8449) + self.acme_bind_addresses = acme_config.get("bind_addresses", ["127.0.0.1"]) + self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30) + + self.tls_certificate_file = os.path.abspath(config.get("tls_certificate_path")) + self.tls_private_key_file = os.path.abspath(config.get("tls_private_key_path")) + self._original_tls_fingerprints = config["tls_fingerprints"] + self.tls_fingerprints = list(self._original_tls_fingerprints) self.no_tls = config.get("no_tls", False) - if self.no_tls: - self.tls_private_key = None - else: - self.tls_private_key = self.read_tls_private_key( - config.get("tls_private_key_path") - ) + # This config option applies to non-federation HTTP clients + # (e.g. for talking to recaptcha, identity servers, and such) + # It should never be used in production, and is intended for + # use only when running tests. + self.use_insecure_ssl_client_just_for_testing_do_not_use = config.get( + "use_insecure_ssl_client_just_for_testing_do_not_use" + ) + + self.tls_certificate = None + self.tls_private_key = None + + def is_disk_cert_valid(self): + """ + Is the certificate we have on disk valid, and if so, for how long? + + Returns: + int: Days remaining of certificate validity. + None: No certificate exists. + """ + if not os.path.exists(self.tls_certificate_file): + return None + + try: + with open(self.tls_certificate_file, 'rb') as f: + cert_pem = f.read() + except Exception: + logger.exception("Failed to read existing certificate off disk!") + raise + + try: + tls_certificate = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + except Exception: + logger.exception("Failed to parse existing certificate off disk!") + raise + + # YYYYMMDDhhmmssZ -- in UTC + expires_on = datetime.strptime( + tls_certificate.get_notAfter().decode('ascii'), "%Y%m%d%H%M%SZ" + ) + now = datetime.utcnow() + days_remaining = (expires_on - now).days + return days_remaining - self.tls_fingerprints = config["tls_fingerprints"] + def read_certificate_from_disk(self): + """ + Read the certificates from disk. + """ + self.tls_certificate = self.read_tls_certificate(self.tls_certificate_file) + + if not self.no_tls: + self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file) + + self.tls_fingerprints = list(self._original_tls_fingerprints) # Check that our own certificate is included in the list of fingerprints # and include it if it is not. x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, - self.tls_certificate + crypto.FILETYPE_ASN1, self.tls_certificate ) sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints) if sha256_fingerprint not in sha256_fingerprints: self.tls_fingerprints.append({u"sha256": sha256_fingerprint}) - # This config option applies to non-federation HTTP clients - # (e.g. for talking to recaptcha, identity servers, and such) - # It should never be used in production, and is intended for - # use only when running tests. - self.use_insecure_ssl_client_just_for_testing_do_not_use = config.get( - "use_insecure_ssl_client_just_for_testing_do_not_use" - ) - def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) tls_certificate_path = base_key_name + ".tls.crt" tls_private_key_path = base_key_name + ".tls.key" - return """\ + return ( + """\ # PEM encoded X509 certificate for TLS. # You can replace the self-signed certificate that synapse # autogenerates on launch with your own SSL certificate + key pair @@ -107,7 +157,24 @@ class TlsConfig(Config): # tls_fingerprints: [] # tls_fingerprints: [{"sha256": ""}] - """ % locals() + + ## Support for ACME certificate auto-provisioning. + # acme: + # enabled: false + ## ACME path. + ## If you only want to test, use the staging url: + ## https://acme-staging.api.letsencrypt.org/directory + # url: 'https://acme-v01.api.letsencrypt.org/directory' + ## Port number (to listen for the HTTP-01 challenge). + ## Using port 80 requires utilising something like authbind, or proxying to it. + # port: 8449 + ## Hosts to bind to. + # bind_addresses: ['127.0.0.1'] + ## How many days remaining on a certificate before it is renewed. + # reprovision_threshold: 30 + """ + % locals() + ) def read_tls_certificate(self, cert_path): cert_pem = self.read_file(cert_path, "tls_certificate") diff --git a/synapse/handlers/acme.py b/synapse/handlers/acme.py new file mode 100644 index 0000000000..73ea7ed018 --- /dev/null +++ b/synapse/handlers/acme.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import attr +from zope.interface import implementer + +from twisted.internet import defer +from twisted.internet.endpoints import serverFromString +from twisted.python.filepath import FilePath +from twisted.python.url import URL +from twisted.web import server, static +from twisted.web.resource import Resource + +logger = logging.getLogger(__name__) + +try: + from txacme.interfaces import ICertificateStore + + @attr.s + @implementer(ICertificateStore) + class ErsatzStore(object): + """ + A store that only stores in memory. + """ + + certs = attr.ib(default=attr.Factory(dict)) + + def store(self, server_name, pem_objects): + self.certs[server_name] = [o.as_bytes() for o in pem_objects] + return defer.succeed(None) + + +except ImportError: + # txacme is missing + pass + + +class AcmeHandler(object): + def __init__(self, hs): + self.hs = hs + self.reactor = hs.get_reactor() + + @defer.inlineCallbacks + def start_listening(self): + + # Configure logging for txacme, if you need to debug + # from eliot import add_destinations + # from eliot.twisted import TwistedDestination + # + # add_destinations(TwistedDestination()) + + from txacme.challenges import HTTP01Responder + from txacme.service import AcmeIssuingService + from txacme.endpoint import load_or_create_client_key + from txacme.client import Client + from josepy.jwa import RS256 + + self._store = ErsatzStore() + responder = HTTP01Responder() + + self._issuer = AcmeIssuingService( + cert_store=self._store, + client_creator=( + lambda: Client.from_url( + reactor=self.reactor, + url=URL.from_text(self.hs.config.acme_url), + key=load_or_create_client_key( + FilePath(self.hs.config.config_dir_path) + ), + alg=RS256, + ) + ), + clock=self.reactor, + responders=[responder], + ) + + well_known = Resource() + well_known.putChild(b'acme-challenge', responder.resource) + responder_resource = Resource() + responder_resource.putChild(b'.well-known', well_known) + responder_resource.putChild(b'check', static.Data(b'OK', b'text/plain')) + + srv = server.Site(responder_resource) + + listeners = [] + + for host in self.hs.config.acme_bind_addresses: + logger.info( + "Listening for ACME requests on %s:%s", host, self.hs.config.acme_port + ) + endpoint = serverFromString( + self.reactor, "tcp:%s:interface=%s" % (self.hs.config.acme_port, host) + ) + listeners.append(endpoint.listen(srv)) + + # Make sure we are registered to the ACME server. There's no public API + # for this, it is usually triggered by startService, but since we don't + # want it to control where we save the certificates, we have to reach in + # and trigger the registration machinery ourselves. + self._issuer._registered = False + yield self._issuer._ensure_registered() + + # Return a Deferred that will fire when all the servers have started up. + yield defer.DeferredList(listeners, fireOnOneErrback=True, consumeErrors=True) + + @defer.inlineCallbacks + def provision_certificate(self): + + logger.warning("Reprovisioning %s", self.hs.hostname) + + try: + yield self._issuer.issue_cert(self.hs.hostname) + except Exception: + logger.exception("Fail!") + raise + logger.warning("Reprovisioned %s, saving.", self.hs.hostname) + cert_chain = self._store.certs[self.hs.hostname] + + try: + with open(self.hs.config.tls_private_key_file, "wb") as private_key_file: + for x in cert_chain: + if x.startswith(b"-----BEGIN RSA PRIVATE KEY-----"): + private_key_file.write(x) + + with open(self.hs.config.tls_certificate_file, "wb") as certificate_file: + for x in cert_chain: + if x.startswith(b"-----BEGIN CERTIFICATE-----"): + certificate_file.write(x) + except Exception: + logger.exception("Failed saving!") + raise + + defer.returnValue(True) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 882e844eb1..756721e304 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -79,6 +79,10 @@ CONDITIONAL_REQUIREMENTS = { # ConsentResource uses select_autoescape, which arrived in jinja 2.9 "resources.consent": ["Jinja2>=2.9"], + # ACME support is required to provision TLS certificates from authorities + # that use the protocol, such as Let's Encrypt. + "acme": ["txacme>=0.9.2"], + "saml2": ["pysaml2>=4.5.0"], "url_preview": ["lxml>=3.5.0"], "test": ["mock>=2.0"], diff --git a/synapse/server.py b/synapse/server.py index 9985687b95..c8914302cf 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -46,6 +46,7 @@ from synapse.federation.transport.client import TransportLayerClient from synapse.groups.attestations import GroupAttestationSigning, GroupAttestionRenewer from synapse.groups.groups_server import GroupsServerHandler from synapse.handlers import Handlers +from synapse.handlers.acme import AcmeHandler from synapse.handlers.appservice import ApplicationServicesHandler from synapse.handlers.auth import AuthHandler, MacaroonGenerator from synapse.handlers.deactivate_account import DeactivateAccountHandler @@ -129,6 +130,7 @@ class HomeServer(object): 'sync_handler', 'typing_handler', 'room_list_handler', + 'acme_handler', 'auth_handler', 'device_handler', 'e2e_keys_handler', @@ -310,6 +312,9 @@ class HomeServer(object): def build_e2e_room_keys_handler(self): return E2eRoomKeysHandler(self) + def build_acme_handler(self): + return AcmeHandler(self) + def build_application_service_api(self): return ApplicationServiceApi(self) -- cgit 1.5.1 From d528406cb875b78907c7f7dc9872c1d2c22dc46c Mon Sep 17 00:00:00 2001 From: Willem Mulder Date: Wed, 23 Jan 2019 18:41:59 +0100 Subject: Fix error message for optional dependencies Signed-off-by: Willem Mulder --- changelog.d/4450.bugfix | 2 ++ synapse/python_dependencies.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4450.bugfix (limited to 'synapse/python_dependencies.py') diff --git a/changelog.d/4450.bugfix b/changelog.d/4450.bugfix new file mode 100644 index 0000000000..b194e94c15 --- /dev/null +++ b/changelog.d/4450.bugfix @@ -0,0 +1,2 @@ +The dependency checker now correctly reports a version mismatch for optional +dependencies, instead of reporting the dependency missing. diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 756721e304..df3e94dee0 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -143,9 +143,12 @@ def check_requirements(for_feature=None, _get_distribution=get_distribution): for dependency in OPTS: try: _get_distribution(dependency) - except VersionConflict: + except VersionConflict as e: deps_needed.append(dependency) - errors.append("Needed %s but it was not installed" % (dependency,)) + errors.append( + "Needed optional %s, got %s==%s" + % (dependency, e.dist.project_name, e.dist.version) + ) except DistributionNotFound: # If it's not found, we don't care pass -- cgit 1.5.1 From ef2228c890b44963c65f086eb1246c27ef43d256 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:55:58 +0000 Subject: Basic sentry integration --- synapse/app/_base.py | 22 ++++++++++++++++++++++ synapse/config/metrics.py | 8 ++++++++ synapse/python_dependencies.py | 1 + 3 files changed, 31 insertions(+) (limited to 'synapse/python_dependencies.py') diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 5b0ca312e2..d681ff5245 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -25,10 +25,12 @@ from daemonize import Daemonize from twisted.internet import error, reactor from twisted.protocols.tls import TLSMemoryBIOFactory +import synapse from synapse.app import check_bind_error from synapse.crypto import context_factory from synapse.util import PreserveLoggingContext from synapse.util.rlimit import change_resource_limit +from synapse.util.versionstring import get_version_string logger = logging.getLogger(__name__) @@ -266,9 +268,29 @@ def start(hs, listeners=None): # It is now safe to start your Synapse. hs.start_listening(listeners) hs.get_datastore().start_profiling() + + setup_sentry_io(hs) except Exception: traceback.print_exc(file=sys.stderr) reactor = hs.get_reactor() if reactor.running: reactor.stop() sys.exit(1) + + +def setup_sentry_io(hs): + if not hs.config.sentry_enabled: + return + + import sentry_sdk + sentry_sdk.init( + dsn=hs.config.sentry_dsn, + release=get_version_string(synapse), + ) + with sentry_sdk.configure_scope() as scope: + scope.set_tag("matrix_server_name", hs.config.server_name) + + app = hs.config.worker_app if hs.config.worker_app else "synapse.app.homeserver" + name = hs.config.worker_name if hs.config.worker_name else "master" + scope.set_tag("worker_app", app) + scope.set_tag("worker_name", name) diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 718c43ae03..f312010a9b 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -23,12 +23,20 @@ class MetricsConfig(Config): self.metrics_port = config.get("metrics_port") self.metrics_bind_host = config.get("metrics_bind_host", "127.0.0.1") + self.sentry_enabled = "sentry" in config + if self.sentry_enabled: + self.sentry_dsn = config["sentry"]["dsn"] + def default_config(self, report_stats=None, **kwargs): res = """\ ## Metrics ### # Enable collection and rendering of performance metrics enable_metrics: False + + # Enable sentry.io integration + #sentry: + # dsn: "..." """ if report_stats is None: diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 5d087ee26b..444fc2a979 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -86,6 +86,7 @@ CONDITIONAL_REQUIREMENTS = { "saml2": ["pysaml2>=4.5.0"], "url_preview": ["lxml>=3.5.0"], "test": ["mock>=2.0", "parameterized"], + "sentry": ["sentry-sdk>=0.7.2"], } -- cgit 1.5.1