From 78323ccdb359404109bfcdd8b5bf6f641ba3ff9b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 16:17:38 +0100 Subject: Remove syutil dependency in favour of smaller single-purpose libraries --- synapse/http/client.py | 3 ++- synapse/http/matrixfederationclient.py | 4 ++-- synapse/http/server.py | 12 ++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index 49737d55da..4b8fd3d3a3 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -15,9 +15,10 @@ from synapse.api.errors import CodeMessageException from synapse.util.logcontext import preserve_context_over_fn -from syutil.jsonutil import encode_canonical_json import synapse.metrics +from canonicaljson import encode_canonical_json + from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError, diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 854e17a473..1c9e552788 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -25,13 +25,13 @@ from synapse.util.async import sleep from synapse.util.logcontext import preserve_context_over_fn import synapse.metrics -from syutil.jsonutil import encode_canonical_json +from canonicaljson import encode_canonical_json from synapse.api.errors import ( SynapseError, Codes, HttpResponseException, ) -from syutil.crypto.jsonsign import sign_json +from signedjson.sign import sign_json import simplejson as json import logging diff --git a/synapse/http/server.py b/synapse/http/server.py index b60e905a62..50feea6f1c 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -21,8 +21,8 @@ from synapse.util.logcontext import LoggingContext, PreserveLoggingContext import synapse.metrics import synapse.events -from syutil.jsonutil import ( - encode_canonical_json, encode_pretty_printed_json, encode_json +from canonicaljson import ( + encode_canonical_json, encode_pretty_printed_json ) from twisted.internet import defer @@ -33,6 +33,7 @@ from twisted.web.util import redirectTo import collections import logging import urllib +import ujson logger = logging.getLogger(__name__) @@ -270,12 +271,11 @@ def respond_with_json(request, code, json_object, send_cors=False, if pretty_print: json_bytes = encode_pretty_printed_json(json_object) + "\n" else: - if canonical_json: + if canonical_json or synapse.events.USE_FROZEN_DICTS: json_bytes = encode_canonical_json(json_object) else: - json_bytes = encode_json( - json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS - ) + # ujson doesn't like frozen_dicts. + json_bytes = ujson.dumps(json_object, ensure_ascii=False) return respond_with_json_bytes( request, code, json_bytes, -- cgit 1.5.1 From 81a93ddcc8798568276582ed9c7a63bc64dc5bc0 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 12:02:07 +0100 Subject: Allow configuration to ignore invalid SSL certs This will be useful for sytest, and sytest only, hence the aggressive config key name. --- synapse/app/homeserver.py | 8 ++++---- synapse/config/tls.py | 4 ++++ synapse/crypto/keyring.py | 4 ++-- synapse/handlers/auth.py | 3 +-- synapse/http/client.py | 25 +++++++++++++++++++++++-- synapse/http/matrixfederationclient.py | 4 ++-- synapse/server.py | 14 ++++++++++++++ 7 files changed, 50 insertions(+), 12 deletions(-) (limited to 'synapse/http') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ffc6299146..ba76ee362a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -15,6 +15,7 @@ # limitations under the License. import sys + sys.dont_write_bytecode = True from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS @@ -221,7 +222,7 @@ class SynapseHomeServer(HomeServer): listener_config, root_resource, ), - self.tls_context_factory, + self.tls_server_context_factory, interface=bind_address ) else: @@ -365,7 +366,6 @@ def setup(config_options): Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. - should_run (bool): Whether to start the reactor. Returns: HomeServer @@ -388,7 +388,7 @@ def setup(config_options): events.USE_FROZEN_DICTS = config.use_frozen_dicts - tls_context_factory = context_factory.ServerContextFactory(config) + tls_server_context_factory = context_factory.ServerContextFactory(config) database_engine = create_engine(config.database_config["name"]) config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection @@ -396,7 +396,7 @@ def setup(config_options): hs = SynapseHomeServer( config.server_name, db_config=config.database_config, - tls_context_factory=tls_context_factory, + tls_server_context_factory=tls_server_context_factory, config=config, content_addr=config.content_addr, version_string=version_string, diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 4751d39bc9..472cf7ac4a 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -42,6 +42,10 @@ class TlsConfig(Config): config.get("tls_dh_params_path"), "tls_dh_params" ) + self.use_insecure_ssl_client = config.get( + "i_really_want_to_ignore_ssl_certs_when_i_am_an_http_client_even_" + "though_it_is_woefully_insecure_because_i_hate_my_users", False) + def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a692cdbe55..e98a625fea 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -463,7 +463,7 @@ class Keyring(object): continue (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory, + server_name, self.hs.tls_server_context_factory, path=(b"/_matrix/key/v2/server/%s" % ( urllib.quote(requested_key_id), )).encode("ascii"), @@ -597,7 +597,7 @@ class Keyring(object): # Try to fetch the key from the remote server. (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + server_name, self.hs.tls_server_context_factory ) # Check the response. diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 59f687e0f1..793b3fcd8b 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -19,7 +19,6 @@ from ._base import BaseHandler from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes -from synapse.http.client import SimpleHttpClient from synapse.util.async import run_on_reactor from twisted.web.client import PartialDownloadError @@ -187,7 +186,7 @@ class AuthHandler(BaseHandler): # TODO: get this from the homeserver rather than creating a new one for # each request try: - client = SimpleHttpClient(self.hs) + client = self.hs.get_simple_http_client() resp_body = yield client.post_urlencoded_get_json( self.hs.config.recaptcha_siteverify_api, args={ diff --git a/synapse/http/client.py b/synapse/http/client.py index 4b8fd3d3a3..da77c8b0ac 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -12,6 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from OpenSSL import SSL +from OpenSSL.SSL import VERIFY_NONE from synapse.api.errors import CodeMessageException from synapse.util.logcontext import preserve_context_over_fn @@ -19,7 +21,7 @@ import synapse.metrics from canonicaljson import encode_canonical_json -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, ssl from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError, HTTPConnectionPool, @@ -59,7 +61,12 @@ class SimpleHttpClient(object): # 'like a browser' pool = HTTPConnectionPool(reactor) pool.maxPersistentPerHost = 10 - self.agent = Agent(reactor, pool=pool) + self.agent = Agent( + reactor, + pool=pool, + connectTimeout=15, + contextFactory=hs.get_http_client_context_factory() + ) self.version_string = hs.version_string def request(self, method, uri, *args, **kwargs): @@ -252,3 +259,17 @@ def _print_ex(e): _print_ex(ex) else: logger.exception(e) + + +class WoefullyInsecureContextFactory(ssl.ContextFactory): + """ + Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. + + Do not use this unless you really, really hate your users.""" + + def __init__(self): + self._context = SSL.Context(SSL.SSLv23_METHOD) + self._context.set_verify(VERIFY_NONE, lambda *_: None) + + def getContext(self, hostname, port): + return self._context diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1c9e552788..b50a0c445c 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -57,14 +57,14 @@ incoming_responses_counter = metrics.register_counter( class MatrixFederationEndpointFactory(object): def __init__(self, hs): - self.tls_context_factory = hs.tls_context_factory + self.tls_server_context_factory = hs.tls_server_context_factory def endpointForURI(self, uri): destination = uri.netloc return matrix_federation_endpoint( reactor, destination, timeout=10, - ssl_context_factory=self.tls_context_factory + ssl_context_factory=self.tls_server_context_factory ) diff --git a/synapse/server.py b/synapse/server.py index 4d1fb1cbf6..656e534dff 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -19,7 +19,9 @@ # partial one for unit test mocking. # Imports required for the default HomeServer() implementation +from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.federation import initialize_http_replication +from synapse.http.client import SimpleHttpClient, WoefullyInsecureContextFactory from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -87,6 +89,8 @@ class BaseHomeServer(object): 'pusherpool', 'event_builder_factory', 'filtering', + 'http_client_context_factory', + 'simple_http_client', ] def __init__(self, hostname, **kwargs): @@ -174,6 +178,16 @@ class HomeServer(BaseHomeServer): def build_auth(self): return Auth(self) + def build_http_client_context_factory(self): + config = self.get_config() + return ( + WoefullyInsecureContextFactory() if config.use_insecure_ssl_client + else BrowserLikePolicyForHTTPS() + ) + + def build_simple_http_client(self): + return SimpleHttpClient(self) + def build_v1auth(self): orf = Auth(self) # Matrix spec makes no reference to what HTTP status code is returned, -- cgit 1.5.1 From 6485f03d91a5f96da28f9dcc8e9ebc3adb213f6f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:05:00 +0100 Subject: Fix random formatting --- synapse/app/homeserver.py | 1 - synapse/http/client.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse/http') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ba76ee362a..8e60304e29 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -15,7 +15,6 @@ # limitations under the License. import sys - sys.dont_write_bytecode = True from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS diff --git a/synapse/http/client.py b/synapse/http/client.py index da77c8b0ac..815a838729 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -265,7 +265,8 @@ class WoefullyInsecureContextFactory(ssl.ContextFactory): """ Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. - Do not use this unless you really, really hate your users.""" + Do not use this unless you really, really hate your users. + """ def __init__(self): self._context = SSL.Context(SSL.SSLv23_METHOD) -- cgit 1.5.1 From 3bcbabc9fb5446e74a675352e22963d528189957 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 15 Sep 2015 15:46:22 +0100 Subject: Rename context factory Mjark is officially no fun. --- synapse/http/client.py | 6 +++--- synapse/server.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index 815a838729..0933388c04 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -261,11 +261,11 @@ def _print_ex(e): logger.exception(e) -class WoefullyInsecureContextFactory(ssl.ContextFactory): +class InsecureInterceptableContextFactory(ssl.ContextFactory): """ - Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. + Factory for PyOpenSSL SSL contexts which accepts any certificate for any domain. - Do not use this unless you really, really hate your users. + Do not use this since it allows an attacker to intercept your communications. """ def __init__(self): diff --git a/synapse/server.py b/synapse/server.py index 656e534dff..d96c5a573a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -21,7 +21,7 @@ # Imports required for the default HomeServer() implementation from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.federation import initialize_http_replication -from synapse.http.client import SimpleHttpClient, WoefullyInsecureContextFactory +from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -181,7 +181,7 @@ class HomeServer(BaseHomeServer): def build_http_client_context_factory(self): config = self.get_config() return ( - WoefullyInsecureContextFactory() if config.use_insecure_ssl_client + InsecureInterceptableContextFactory() if config.use_insecure_ssl_client else BrowserLikePolicyForHTTPS() ) -- cgit 1.5.1 From 8fc52bc56a0813d40741f45164caa3230d3e00ec Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 2 Oct 2015 17:13:51 -0500 Subject: Allow synapse's useragent to be customized This will allow me to write tests which verify which server made HTTP requests in a federation context. --- synapse/config/server.py | 1 + synapse/http/client.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'synapse/http') diff --git a/synapse/config/server.py b/synapse/config/server.py index 4d12d49857..50c4afdcfe 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -26,6 +26,7 @@ class ServerConfig(Config): self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") self.print_pidfile = config.get("print_pidfile") + self.user_agent_override = config.get("user_agent_override") self.use_frozen_dicts = config.get("use_frozen_dicts", True) self.listeners = config.get("listeners", []) diff --git a/synapse/http/client.py b/synapse/http/client.py index 0933388c04..6adf35c7bf 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -67,7 +67,9 @@ class SimpleHttpClient(object): connectTimeout=15, contextFactory=hs.get_http_client_context_factory() ) - self.version_string = hs.version_string + self.user_agent = hs.config.user_agent_override + if self.user_agent is None: + self.user_agent = hs.version_string def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach @@ -112,7 +114,7 @@ class SimpleHttpClient(object): uri.encode("ascii"), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -131,7 +133,8 @@ class SimpleHttpClient(object): "POST", uri.encode("ascii"), headers=Headers({ - "Content-Type": ["application/json"] + b"Content-Type": [b"application/json"], + b"User-Agent": [self.user_agent], }), bodyProducer=FileBodyProducer(StringIO(json_str)) ) @@ -165,7 +168,7 @@ class SimpleHttpClient(object): "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], }) ) @@ -206,7 +209,7 @@ class SimpleHttpClient(object): "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) @@ -241,7 +244,7 @@ class CaptchaServerHttpClient(SimpleHttpClient): bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], }) ) -- cgit 1.5.1 From b28c7da0a4507825440fd801e1b6dbc5e6a454a7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 5 Oct 2015 20:49:39 -0500 Subject: Preserve version string in user agent --- synapse/config/server.py | 2 +- synapse/http/client.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'synapse/http') diff --git a/synapse/config/server.py b/synapse/config/server.py index 50c4afdcfe..5c2d6bfeab 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -26,7 +26,7 @@ class ServerConfig(Config): self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") self.print_pidfile = config.get("print_pidfile") - self.user_agent_override = config.get("user_agent_override") + self.user_agent_suffix = config.get("user_agent_suffix") self.use_frozen_dicts = config.get("use_frozen_dicts", True) self.listeners = config.get("listeners", []) diff --git a/synapse/http/client.py b/synapse/http/client.py index 6adf35c7bf..5017801773 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -67,9 +67,9 @@ class SimpleHttpClient(object): connectTimeout=15, contextFactory=hs.get_http_client_context_factory() ) - self.user_agent = hs.config.user_agent_override - if self.user_agent is None: - self.user_agent = hs.version_string + self.user_agent = hs.version_string + if hs.config.user_agent_suffix: + self.user_agent += " - " + hs.config.user_agent_suffix def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach -- cgit 1.5.1 From e0b466bcfdabcd70accdacbb72cfaf8616a9d18f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 6 Oct 2015 09:32:26 -0500 Subject: Use space not dash as delimiter --- synapse/http/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index 5017801773..e0dda97ef6 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -69,7 +69,7 @@ class SimpleHttpClient(object): ) self.user_agent = hs.version_string if hs.config.user_agent_suffix: - self.user_agent += " - " + hs.config.user_agent_suffix + self.user_agent += " " + hs.config.user_agent_suffix def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach -- cgit 1.5.1 From 492beb62a823335fe6a6fb4c59a1bc246c3f57e2 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 6 Oct 2015 09:53:33 -0500 Subject: Use space not dash as delimiter --- synapse/http/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index e0dda97ef6..79c529291f 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -69,7 +69,7 @@ class SimpleHttpClient(object): ) self.user_agent = hs.version_string if hs.config.user_agent_suffix: - self.user_agent += " " + hs.config.user_agent_suffix + self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,) def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach -- cgit 1.5.1 From 625e13bfde35a3c6fdd2b3e8263838ec4d4fbcc3 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Fri, 9 Oct 2015 11:02:56 +0100 Subject: Add get_raw method to SimpleHttpClient, use this in CAS auth rather than requests --- synapse/http/client.py | 55 +++++++++++++++++++++++++++-------------- synapse/rest/client/v1/login.py | 13 ++++++---- 2 files changed, 44 insertions(+), 24 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index 79c529291f..ca642a7a06 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -160,16 +160,40 @@ class SimpleHttpClient(object): On a non-2xx HTTP response. The response body will be used as the error message. """ + body = yield self.get_raw(uri, args) + defer.returnValue(json.loads(body)) + + @defer.inlineCallbacks + def put_json(self, uri, json_body, args={}): + """ Puts some json to the given URI. + + Args: + uri (str): The URI to request, not including query parameters + json_body (dict): The JSON to put in the HTTP body, + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + Returns: + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. + Raises: + On a non-2xx HTTP response. + """ if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) + json_str = encode_canonical_json(json_body) + response = yield self.request( - "GET", + "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.user_agent], - }) + b"User-Agent": [self.version_string], + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) ) body = yield preserve_context_over_fn(readBody, response) @@ -183,46 +207,39 @@ class SimpleHttpClient(object): raise CodeMessageException(response.code, body) @defer.inlineCallbacks - def put_json(self, uri, json_body, args={}): - """ Puts some json to the given URI. + def get_raw(self, uri, args={}): + """ Gets raw text from the given URI. Args: uri (str): The URI to request, not including query parameters - json_body (dict): The JSON to put in the HTTP body, args (dict): A dictionary used to create query strings, defaults to None. **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: Deferred: Succeeds when we get *any* 2xx HTTP response, with the - HTTP body as JSON. + HTTP body at text. Raises: - On a non-2xx HTTP response. + On a non-2xx HTTP response. The response body will be used as the + error message. """ if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) - json_str = encode_canonical_json(json_body) - response = yield self.request( - "PUT", + "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.user_agent], - "Content-Type": ["application/json"] - }), - bodyProducer=FileBodyProducer(StringIO(json_str)) + b"User-Agent": [self.version_string], + }) ) body = yield preserve_context_over_fn(readBody, response) if 200 <= response.code < 300: - defer.returnValue(json.loads(body)) + defer.returnValue(body) else: - # NB: This is explicitly not json.loads(body)'d because the contract - # of CodeMessageException is a *string* message. Callers can always - # load it into JSON if they want. raise CodeMessageException(response.code, body) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 4de5f19591..f5cd6a1960 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -16,6 +16,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, LoginError, Codes +from synapse.http.client import SimpleHttpClient from synapse.types import UserID from base import ClientV1RestServlet, client_path_pattern @@ -28,7 +29,6 @@ from saml2 import config from saml2.client import Saml2Client import xml.etree.ElementTree as ET -import requests logger = logging.getLogger(__name__) @@ -79,13 +79,16 @@ class LoginRestServlet(ClientV1RestServlet): defer.returnValue((200, result)) elif self.cas_enabled and (login_submission["type"] == LoginRestServlet.CAS_TYPE): - url = "%s/proxyValidate" % (self.cas_server_url) - parameters = { + # TODO: get this from the homeserver rather than creating a new one for + # each request + http_client = SimpleHttpClient(self.hs) + uri = "%s/proxyValidate" % (self.cas_server_url,) + args = { "ticket": login_submission["ticket"], "service": login_submission["service"] } - response = requests.get(url, verify=False, params=parameters) - result = yield self.do_cas_login(response.text) + body = yield http_client.get_raw(uri, args) + result = yield self.do_cas_login(body) defer.returnValue(result) else: raise SynapseError(400, "Bad login type.") -- cgit 1.5.1 From a80ef851f7e624b9eee91f134b233f3c0742bb3e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 10 Oct 2015 12:35:39 +0100 Subject: Fix previous merge to s/version_string/user_agent/ --- synapse/http/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index ca642a7a06..9a5869abee 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -190,7 +190,7 @@ class SimpleHttpClient(object): "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) @@ -231,7 +231,7 @@ class SimpleHttpClient(object): "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], }) ) -- cgit 1.5.1 From b8dd5b1a2d76f0426c600ae19ea9d9612e5327dc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 16 Oct 2015 14:54:54 +0100 Subject: Verify third party ID server certificates --- synapse/api/auth.py | 11 +++++++++++ synapse/handlers/room.py | 31 +++++++++++++++++++++++++++++-- synapse/http/client.py | 4 ---- synapse/util/third_party_invites.py | 6 +++--- 4 files changed, 43 insertions(+), 9 deletions(-) (limited to 'synapse/http') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e96d747b99..aee9b8a14f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -397,13 +397,24 @@ class Auth(object): (EventTypes.ThirdPartyInvite, token,) ) if not invite_event: + logger.info("Failing 3pid invite because no invite found for token %s", token) return False try: public_key = join_third_party_invite["public_key"] key_validity_url = join_third_party_invite["key_validity_url"] if invite_event.content["public_key"] != public_key: + logger.info( + "Failing 3pid invite because public key invite: %s != join: %s", + invite_event.content["public_key"], + public_key + ) return False if invite_event.content["key_validity_url"] != key_validity_url: + logger.info( + "Failing 3pid invite because key_validity_url invite: %s != join: %s", + invite_event.content["key_validity_url"], + key_validity_url + ) return False verify_key = nacl.signing.VerifyKey(decode_base64(public_key)) encoded_signature = join_third_party_invite["signature"] diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 9ffa521aad..3f0cde56f0 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -22,11 +22,16 @@ from synapse.types import UserID, RoomAlias, RoomID from synapse.api.constants import ( EventTypes, Membership, JoinRules, RoomCreationPreset, ) -from synapse.api.errors import StoreError, SynapseError +from synapse.api.errors import AuthError, StoreError, SynapseError from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor +from signedjson.sign import verify_signed_json +from signedjson.key import decode_verify_key_bytes + from collections import OrderedDict +from unpaddedbase64 import decode_base64 + import logging import string @@ -614,12 +619,34 @@ class RoomMemberHandler(BaseHandler): ) if "mxid" in data: - # TODO: Validate the response signature and such + if "signatures" not in data: + raise AuthError(401, "No signatures on 3pid binding") + self.verify_any_signature(data, id_server) defer.returnValue(data["mxid"]) + except IOError as e: logger.warn("Error from identity server lookup: %s" % (e,)) defer.returnValue(None) + @defer.inlineCallbacks + def verify_any_signature(self, data, server_hostname): + if server_hostname not in data["signatures"]: + raise AuthError(401, "No signature from server %s" % (server_hostname,)) + for key_name, signature in data["signatures"][server_hostname].items(): + key_data = yield self.hs.get_simple_http_client().get_json( + "https://%s/_matrix/identity/api/v1/pubkey/%s" % + (server_hostname, key_name,), + ) + if "public_key" not in key_data: + raise AuthError(401, "No public key named %s from %s" % + (key_name, server_hostname,)) + verify_signed_json( + data, + server_hostname, + decode_verify_key_bytes(key_name, decode_base64(key_data["public_key"])) + ) + return + @defer.inlineCallbacks def _make_and_store_3pid_invite( self, diff --git a/synapse/http/client.py b/synapse/http/client.py index 9a5869abee..27e5190224 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -24,7 +24,6 @@ from canonicaljson import encode_canonical_json from twisted.internet import defer, reactor, ssl from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError, - HTTPConnectionPool, ) from twisted.web.http_headers import Headers @@ -59,11 +58,8 @@ class SimpleHttpClient(object): # The default context factory in Twisted 14.0.0 (which we require) is # BrowserLikePolicyForHTTPS which will do regular cert validation # 'like a browser' - pool = HTTPConnectionPool(reactor) - pool.maxPersistentPerHost = 10 self.agent = Agent( reactor, - pool=pool, connectTimeout=15, contextFactory=hs.get_http_client_context_factory() ) diff --git a/synapse/util/third_party_invites.py b/synapse/util/third_party_invites.py index 41e597d5b9..335a9755b2 100644 --- a/synapse/util/third_party_invites.py +++ b/synapse/util/third_party_invites.py @@ -63,7 +63,7 @@ def check_key_valid(http_client, event): event.content["third_party_invite"]["key_validity_url"], {"public_key": event.content["third_party_invite"]["public_key"]} ) - if not response["valid"]: - raise AuthError(403, "Third party certificate was invalid") - except IOError: + except Exception: raise AuthError(502, "Third party certificate could not be checked") + if "valid" not in response or not response["valid"]: + raise AuthError(403, "Third party certificate was invalid") -- cgit 1.5.1 From eacb068ac2a6df8494d9f7255c80f4429e779209 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Nov 2015 16:49:05 +0000 Subject: Retry dead servers a lot less often --- synapse/http/matrixfederationclient.py | 10 ++++++++-- synapse/util/retryutils.py | 7 +++++-- 2 files changed, 13 insertions(+), 4 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b50a0c445c..6e53538a52 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -35,6 +35,7 @@ from signedjson.sign import sign_json import simplejson as json import logging +import random import sys import urllib import urlparse @@ -55,6 +56,9 @@ incoming_responses_counter = metrics.register_counter( ) +MAX_RETRIES = 4 + + class MatrixFederationEndpointFactory(object): def __init__(self, hs): self.tls_server_context_factory = hs.tls_server_context_factory @@ -119,7 +123,7 @@ class MatrixFederationHttpClient(object): # XXX: Would be much nicer to retry only at the transaction-layer # (once we have reliable transactions in place) - retries_left = 5 + retries_left = MAX_RETRIES http_url_bytes = urlparse.urlunparse( ("", "", path_bytes, param_bytes, query_bytes, "") @@ -180,7 +184,9 @@ class MatrixFederationHttpClient(object): ) if retries_left and not timeout: - yield sleep(2 ** (5 - retries_left)) + delay = 5 ** (MAX_RETRIES + 1 - retries_left) + delay *= random.uniform(0.8, 1.4) + yield sleep(delay) retries_left -= 1 else: raise diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index a42138f556..2fe6814807 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.errors import CodeMessageException import logging +import random logger = logging.getLogger(__name__) @@ -85,8 +86,9 @@ def get_retry_limiter(destination, clock, store, **kwargs): class RetryDestinationLimiter(object): def __init__(self, destination, clock, store, retry_interval, - min_retry_interval=5000, max_retry_interval=60 * 60 * 1000, - multiplier_retry_interval=2,): + min_retry_interval=10 * 60 * 1000, + max_retry_interval=24 * 60 * 60 * 1000, + multiplier_retry_interval=5,): """Marks the destination as "down" if an exception is thrown in the context, except for CodeMessageException with code < 500. @@ -140,6 +142,7 @@ class RetryDestinationLimiter(object): # We couldn't connect. if self.retry_interval: self.retry_interval *= self.multiplier_retry_interval + self.retry_interval *= int(random.uniform(0.8, 1.4)) if self.retry_interval >= self.max_retry_interval: self.retry_interval = self.max_retry_interval -- cgit 1.5.1 From bceec6591311eeefe94efcd4e40a116ee73aad8b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 15:10:05 +0000 Subject: Slightly more aggressive retry timers at HTTP level --- synapse/http/matrixfederationclient.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 6e53538a52..ca9591556d 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -56,7 +56,7 @@ incoming_responses_counter = metrics.register_counter( ) -MAX_RETRIES = 4 +MAX_RETRIES = 10 class MatrixFederationEndpointFactory(object): @@ -184,7 +184,8 @@ class MatrixFederationHttpClient(object): ) if retries_left and not timeout: - delay = 5 ** (MAX_RETRIES + 1 - retries_left) + delay = 4 ** (MAX_RETRIES + 1 - retries_left) + delay = max(delay, 60) delay *= random.uniform(0.8, 1.4) yield sleep(delay) retries_left -= 1 -- cgit 1.5.1