From c8d67beb9cf32f273fd5a78e26636aa4feedde25 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 1 Oct 2014 15:51:49 +0100 Subject: remove "red", "blue" and "green" server_name mappings --- synapse/http/client.py | 7 ------- 1 file changed, 7 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/http/client.py b/synapse/http/client.py index eb11bfd4d5..822afeec1d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -35,13 +35,6 @@ import urllib logger = logging.getLogger(__name__) -# FIXME: SURELY these should be killed?! -_destination_mappings = { - "red": "localhost:8080", - "blue": "localhost:8081", - "green": "localhost:8082", -} - class HttpClient(object): """ Interface for talking json over http -- cgit 1.4.1 From 4f11518934c3e032a763f115a73261414d67f87b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 2 Oct 2014 13:57:48 +0100 Subject: Split PlainHttpClient into separate clients for talking to Identity servers and talking to Capatcha servers --- synapse/app/homeserver.py | 4 +- synapse/handlers/directory.py | 4 +- synapse/handlers/login.py | 6 +- synapse/handlers/register.py | 11 +- synapse/http/client.py | 290 ++++++++++++++++++++------------------- tests/handlers/test_directory.py | 4 +- 6 files changed, 163 insertions(+), 156 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2f1b954902..61d574a00f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -25,7 +25,7 @@ from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect from synapse.http.content_repository import ContentRepoResource -from synapse.http.client import TwistedHttpClient +from synapse.http.client import MatrixHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX ) @@ -47,7 +47,7 @@ logger = logging.getLogger(__name__) class SynapseHomeServer(HomeServer): def build_http_client(self): - return TwistedHttpClient(self) + return MatrixHttpClient(self) def build_resource_for_client(self): return JsonResource() diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 84c3a1d56f..cec7737e09 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -18,7 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError -from synapse.http.client import HttpClient +from synapse.http.client import MatrixHttpClient from synapse.api.events.room import RoomAliasesEvent import logging @@ -98,7 +98,7 @@ class DirectoryHandler(BaseHandler): query_type="directory", args={ "room_alias": room_alias.to_string(), - HttpClient.RETRY_DNS_LOOKUP_FAILURES: False + MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False } ) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 80ffdd2726..3f152e18f0 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import LoginError, Codes -from synapse.http.client import PlainHttpClient +from synapse.http.client import IdentityServerHttpClient from synapse.util.emailutils import EmailException import synapse.util.emailutils as emailutils @@ -97,10 +97,10 @@ class LoginHandler(BaseHandler): @defer.inlineCallbacks def _query_email(self, email): - httpCli = PlainHttpClient(self.hs) + httpCli = IdentityServerHttpClient(self.hs) data = yield httpCli.get_json( 'matrix.org:8090', # TODO FIXME This should be configurable. "/_matrix/identity/api/v1/lookup?medium=email&address=" + "%s" % urllib.quote(email) ) - defer.returnValue(data) \ No newline at end of file + defer.returnValue(data) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index a019d770d4..266495056e 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -22,7 +22,8 @@ from synapse.api.errors import ( ) from ._base import BaseHandler import synapse.util.stringutils as stringutils -from synapse.http.client import PlainHttpClient +from synapse.http.client import IdentityServerHttpClient +from synapse.http.client import CaptchaServerHttpClient import base64 import bcrypt @@ -154,7 +155,9 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _threepid_from_creds(self, creds): - httpCli = PlainHttpClient(self.hs) + # TODO: get this from the homeserver rather than creating a new one for + # each request + httpCli = IdentityServerHttpClient(self.hs) # XXX: make this configurable! trustedIdServers = ['matrix.org:8090'] if not creds['idServer'] in trustedIdServers: @@ -203,7 +206,9 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _submit_captcha(self, ip_addr, private_key, challenge, response): - client = PlainHttpClient(self.hs) + # TODO: get this from the homeserver rather than creating a new one for + # each request + client = CaptchaServerHttpClient(self.hs) data = yield client.post_urlencoded_get_raw( "www.google.com:80", "/recaptcha/api/verify", diff --git a/synapse/http/client.py b/synapse/http/client.py index 822afeec1d..e02cce5642 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -36,49 +36,6 @@ import urllib logger = logging.getLogger(__name__) -class HttpClient(object): - """ Interface for talking json over http - """ - RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" - - def put_json(self, destination, path, data): - """ Sends the specifed json data using PUT - - Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. - data (dict): A dict containing the data that will be used as - the request body. This will be encoded as JSON. - - Returns: - Deferred: Succeeds when we get a 2xx HTTP response. The result - will be the decoded JSON body. On a 4xx or 5xx error response a - CodeMessageException is raised. - """ - pass - - def get_json(self, destination, path, args=None): - """ Get's some json from the given host homeserver and path - - Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. - args (dict): A dictionary used to create query strings, defaults to - None. - **Note**: The value of each key is assumed to be an iterable - and *not* a string. - - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. - """ - pass - - class MatrixHttpAgent(_AgentBase): def __init__(self, reactor, pool=None): @@ -102,23 +59,114 @@ class MatrixHttpAgent(_AgentBase): parsed_URI.originForm) -class TwistedHttpClient(HttpClient): - """ Wrapper around the twisted HTTP client api. +class BaseHttpClient(object): + """Base class for HTTP clients using twisted. + """ + + def __init__(self, hs): + self.agent = MatrixHttpAgent(reactor) + self.hs = hs + + @defer.inlineCallbacks + def _create_request(self, destination, method, path_bytes, param_bytes=b"", + query_bytes=b"", producer=None, headers_dict={}, + retry_on_dns_fail=True, on_send_callback=None): + """ Creates and sends a request to the given url + """ + headers_dict[b"User-Agent"] = [b"Synapse"] + headers_dict[b"Host"] = [destination] + + logger.debug("Sending request to %s: %s %s;%s?%s", + destination, method, path_bytes, param_bytes, query_bytes) + + logger.debug( + "Types: %s", + [ + type(destination), type(method), type(path_bytes), + type(param_bytes), + type(query_bytes) + ] + ) + + retries_left = 5 + + endpoint = self._getEndpoint(reactor, destination); + + while True: + if on_send_callback: + on_send_callback(destination, method, path_bytes, producer) + + try: + response = yield self.agent.request( + destination, + endpoint, + method, + path_bytes, + param_bytes, + query_bytes, + Headers(headers_dict), + producer + ) + + logger.debug("Got response to %s", method) + break + except Exception as e: + if not retry_on_dns_fail and isinstance(e, DNSLookupError): + logger.warn("DNS Lookup failed to %s with %s", destination, + e) + raise SynapseError(400, "Domain specified not found.") + + logger.exception("Got error in _create_request") + _print_ex(e) + + if retries_left: + yield sleep(2 ** (5 - retries_left)) + retries_left -= 1 + else: + raise + + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + pass + else: + # :'( + # Update transactions table? + logger.error( + "Got response %d %s", response.code, response.phrase + ) + raise CodeMessageException( + response.code, response.phrase + ) + + defer.returnValue(response) + + +class MatrixHttpClient(BaseHttpClient): + """ Wrapper around the twisted HTTP client api. Implements Attributes: agent (twisted.web.client.Agent): The twisted Agent used to send the requests. """ - def __init__(self, hs): - self.agent = MatrixHttpAgent(reactor) - self.hs = hs + RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" @defer.inlineCallbacks def put_json(self, destination, path, data, on_send_callback=None): - if destination in _destination_mappings: - destination = _destination_mappings[destination] + """ Sends the specifed json data using PUT + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + data (dict): A dict containing the data that will be used as + the request body. This will be encoded as JSON. + + Returns: + Deferred: Succeeds when we get a 2xx HTTP response. The result + will be the decoded JSON body. On a 4xx or 5xx error response a + CodeMessageException is raised. + """ response = yield self._create_request( destination.encode("ascii"), "PUT", @@ -136,9 +184,23 @@ class TwistedHttpClient(HttpClient): @defer.inlineCallbacks def get_json(self, destination, path, args={}): - if destination in _destination_mappings: - destination = _destination_mappings[destination] + """ Get's some json from the given host homeserver and path + + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + + Returns: + Deferred: Succeeds when we get *any* HTTP response. + The result of the deferred is a tuple of `(code, response)`, + where `response` is a dict representing the decoded JSON body. + """ logger.debug("get_json args: %s", args) retry_on_dns_fail = True @@ -163,6 +225,22 @@ class TwistedHttpClient(HttpClient): defer.returnValue(json.loads(body)) + + def _getEndpoint(self, reactor, destination): + return matrix_endpoint( + reactor, destination, timeout=10, + ssl_context_factory=self.hs.tls_context_factory + ) + + +class IdentityServerHttpClient(BaseHttpClient): + """Separate HTTP client for talking to the Identity servers since they + don't use SRV records and talk x-www-form-urlencoded rather than JSON. + """ + def _getEndpoint(self, reactor, destination): + #TODO: This should be talking TLS + return matrix_endpoint(reactor, destination, timeout=10) + @defer.inlineCallbacks def post_urlencoded_get_json(self, destination, path, args={}): if destination in _destination_mappings: @@ -176,16 +254,25 @@ class TwistedHttpClient(HttpClient): "POST", path.encode("ascii"), producer=FileBodyProducer(StringIO(urllib.urlencode(args))), - headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]} + headers_dict={ + "Content-Type": ["application/x-www-form-urlencoded"] + } ) body = yield readBody(response) defer.returnValue(json.loads(body)) - - # XXX FIXME : I'm so sorry. + + +class CaptchaServerHttpClient(MatrixHttpClient): + """Separate HTTP client for talking to google's captcha servers""" + + def _getEndpoint(self, reactor, destination): + return matrix_endpoint(reactor, destination, timeout=10) + @defer.inlineCallbacks - def post_urlencoded_get_raw(self, destination, path, accept_partial=False, args={}): + def post_urlencoded_get_raw(self, destination, path, accept_partial=False, + args={}): if destination in _destination_mappings: destination = _destination_mappings[destination] @@ -196,7 +283,9 @@ class TwistedHttpClient(HttpClient): "POST", path.encode("ascii"), producer=FileBodyProducer(StringIO(urllib.urlencode(args))), - headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]} + headers_dict={ + "Content-Type": ["application/x-www-form-urlencoded"] + } ) try: @@ -207,93 +296,6 @@ class TwistedHttpClient(HttpClient): defer.returnValue(e.response) else: raise e - - - @defer.inlineCallbacks - def _create_request(self, destination, method, path_bytes, param_bytes=b"", - query_bytes=b"", producer=None, headers_dict={}, - retry_on_dns_fail=True, on_send_callback=None): - """ Creates and sends a request to the given url - """ - headers_dict[b"User-Agent"] = [b"Synapse"] - headers_dict[b"Host"] = [destination] - - logger.debug("Sending request to %s: %s %s;%s?%s", - destination, method, path_bytes, param_bytes, query_bytes) - - logger.debug( - "Types: %s", - [ - type(destination), type(method), type(path_bytes), - type(param_bytes), - type(query_bytes) - ] - ) - - retries_left = 5 - - # TODO: setup and pass in an ssl_context to enable TLS - endpoint = self._getEndpoint(reactor, destination); - - while True: - if on_send_callback: - on_send_callback(destination, method, path_bytes, producer) - - try: - response = yield self.agent.request( - destination, - endpoint, - method, - path_bytes, - param_bytes, - query_bytes, - Headers(headers_dict), - producer - ) - - logger.debug("Got response to %s", method) - break - except Exception as e: - if not retry_on_dns_fail and isinstance(e, DNSLookupError): - logger.warn("DNS Lookup failed to %s with %s", destination, - e) - raise SynapseError(400, "Domain specified not found.") - - logger.exception("Got error in _create_request") - _print_ex(e) - - if retries_left: - yield sleep(2 ** (5 - retries_left)) - retries_left -= 1 - else: - raise - - if 200 <= response.code < 300: - # We need to update the transactions table to say it was sent? - pass - else: - # :'( - # Update transactions table? - logger.error( - "Got response %d %s", response.code, response.phrase - ) - raise CodeMessageException( - response.code, response.phrase - ) - - defer.returnValue(response) - - def _getEndpoint(self, reactor, destination): - return matrix_endpoint( - reactor, destination, timeout=10, - ssl_context_factory=self.hs.tls_context_factory - ) - - -class PlainHttpClient(TwistedHttpClient): - def _getEndpoint(self, reactor, destination): - return matrix_endpoint(reactor, destination, timeout=10) - def _print_ex(e): if hasattr(e, "reasons") and e.reasons: diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index dd5d85dde6..0c31502dd4 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -20,7 +20,7 @@ from twisted.internet import defer from mock import Mock from synapse.server import HomeServer -from synapse.http.client import HttpClient +from synapse.http.client import MatrixHttpClient from synapse.handlers.directory import DirectoryHandler from synapse.storage.directory import RoomAliasMapping @@ -95,7 +95,7 @@ class DirectoryTestCase(unittest.TestCase): query_type="directory", args={ "room_alias": "#another:remote", - HttpClient.RETRY_DNS_LOOKUP_FAILURES: False + MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False } ) -- cgit 1.4.1 From 574377636ee4eafba50580fc4d7a1d0793774332 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 2 Oct 2014 14:09:15 +0100 Subject: Add a keyword argument to get_json to avoid retrying on DNS failures. Rather than passing MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES as a fake query string parameter --- synapse/federation/replication.py | 7 +++++-- synapse/federation/transport.py | 5 +++-- synapse/handlers/directory.py | 5 ++--- synapse/http/client.py | 9 +-------- tests/federation/test_federation.py | 5 +++-- tests/handlers/test_directory.py | 5 ++--- 6 files changed, 16 insertions(+), 20 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 96b82f00cb..5f96f79998 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -159,7 +159,8 @@ class ReplicationLayer(object): return defer.succeed(None) @log_function - def make_query(self, destination, query_type, args): + def make_query(self, destination, query_type, args, + retry_on_dns_fail=True): """Sends a federation Query to a remote homeserver of the given type and arguments. @@ -174,7 +175,9 @@ class ReplicationLayer(object): a Deferred which will eventually yield a JSON object from the response """ - return self.transport_layer.make_query(destination, query_type, args) + return self.transport_layer.make_query( + destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail + ) @defer.inlineCallbacks @log_function diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index afc777ec9e..93296af204 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -193,13 +193,14 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def make_query(self, destination, query_type, args): + def make_query(self, destination, query_type, args, retry_on_dns_fail): path = PREFIX + "/query/%s" % query_type response = yield self.client.get_json( destination=destination, path=path, - args=args + args=args, + retry_on_dns_fail=retry_on_dns_fail, ) defer.returnValue(response) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index cec7737e09..a56830d520 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -18,7 +18,6 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError -from synapse.http.client import MatrixHttpClient from synapse.api.events.room import RoomAliasesEvent import logging @@ -98,8 +97,8 @@ class DirectoryHandler(BaseHandler): query_type="directory", args={ "room_alias": room_alias.to_string(), - MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False - } + }, + retry_on_dns_fail=False, ) if result and "room_id" in result and "servers" in result: diff --git a/synapse/http/client.py b/synapse/http/client.py index e02cce5642..57b49355f2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -183,7 +183,7 @@ class MatrixHttpClient(BaseHttpClient): defer.returnValue((response.code, body)) @defer.inlineCallbacks - def get_json(self, destination, path, args={}): + def get_json(self, destination, path, args={}, retry_on_dns_fail=True): """ Get's some json from the given host homeserver and path Args: @@ -203,13 +203,6 @@ class MatrixHttpClient(BaseHttpClient): """ logger.debug("get_json args: %s", args) - retry_on_dns_fail = True - if HttpClient.RETRY_DNS_LOOKUP_FAILURES in args: - # FIXME: This isn't ideal, but the interface exposed in get_json - # isn't comprehensive enough to give caller's any control over - # their connection mechanics. - retry_on_dns_fail = args.pop(HttpClient.RETRY_DNS_LOOKUP_FAILURES) - query_bytes = urllib.urlencode(args, True) logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index bb17e9aafe..d95b9013a3 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -253,7 +253,7 @@ class FederationTestCase(unittest.TestCase): response = yield self.federation.make_query( destination="remote", query_type="a-question", - args={"one": "1", "two": "2"} + args={"one": "1", "two": "2"}, ) self.assertEquals({"your": "response"}, response) @@ -261,7 +261,8 @@ class FederationTestCase(unittest.TestCase): self.mock_http_client.get_json.assert_called_with( destination="remote", path="/_matrix/federation/v1/query/a-question", - args={"one": "1", "two": "2"} + args={"one": "1", "two": "2"}, + retry_on_dns_fail=True, ) @defer.inlineCallbacks diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 0c31502dd4..e10a49a8ac 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -20,7 +20,6 @@ from twisted.internet import defer from mock import Mock from synapse.server import HomeServer -from synapse.http.client import MatrixHttpClient from synapse.handlers.directory import DirectoryHandler from synapse.storage.directory import RoomAliasMapping @@ -95,8 +94,8 @@ class DirectoryTestCase(unittest.TestCase): query_type="directory", args={ "room_alias": "#another:remote", - MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False - } + }, + retry_on_dns_fail=False, ) @defer.inlineCallbacks -- cgit 1.4.1 From b9cdc443d7acdd1f8095878e3c24b859f82ea31e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 2 Oct 2014 14:37:30 +0100 Subject: Fix pyflakes errors --- synapse/handlers/register.py | 2 +- synapse/http/client.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 266495056e..df562aa762 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -176,7 +176,7 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _bind_threepid(self, creds, mxid): - httpCli = PlainHttpClient(self.hs) + httpCli = IdentityServerHttpClient(self.hs) data = yield httpCli.post_urlencoded_get_json( creds['idServer'], "/_matrix/identity/api/v1/3pid/bind", diff --git a/synapse/http/client.py b/synapse/http/client.py index 57b49355f2..5c2fbd1f87 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -236,9 +236,6 @@ class IdentityServerHttpClient(BaseHttpClient): @defer.inlineCallbacks def post_urlencoded_get_json(self, destination, path, args={}): - if destination in _destination_mappings: - destination = _destination_mappings[destination] - logger.debug("post_urlencoded_get_json args: %s", args) query_bytes = urllib.urlencode(args, True) @@ -246,7 +243,7 @@ class IdentityServerHttpClient(BaseHttpClient): destination.encode("ascii"), "POST", path.encode("ascii"), - producer=FileBodyProducer(StringIO(urllib.urlencode(args))), + producer=FileBodyProducer(StringIO(query_bytes)), headers_dict={ "Content-Type": ["application/x-www-form-urlencoded"] } @@ -266,16 +263,13 @@ class CaptchaServerHttpClient(MatrixHttpClient): @defer.inlineCallbacks def post_urlencoded_get_raw(self, destination, path, accept_partial=False, args={}): - if destination in _destination_mappings: - destination = _destination_mappings[destination] - query_bytes = urllib.urlencode(args, True) response = yield self._create_request( destination.encode("ascii"), "POST", path.encode("ascii"), - producer=FileBodyProducer(StringIO(urllib.urlencode(args))), + producer=FileBodyProducer(StringIO(query_bytes)), headers_dict={ "Content-Type": ["application/x-www-form-urlencoded"] } -- cgit 1.4.1 From 693d0b8f4562d79ca00e05ce0cfd2d8c7b175b59 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 13 Oct 2014 10:49:04 +0100 Subject: Replace on_send_callback with something a bit clearer so that we can sign messages --- synapse/http/client.py | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/http/client.py b/synapse/http/client.py index 5c2fbd1f87..0e8fa2eb25 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -31,6 +31,7 @@ from StringIO import StringIO import json import logging import urllib +import urlparse logger = logging.getLogger(__name__) @@ -68,16 +69,20 @@ class BaseHttpClient(object): self.hs = hs @defer.inlineCallbacks - def _create_request(self, destination, method, path_bytes, param_bytes=b"", - query_bytes=b"", producer=None, headers_dict={}, - retry_on_dns_fail=True, on_send_callback=None): + def _create_request(self, destination, method, path_bytes, + body_callback, headers_dict={}, param_bytes=b"", + query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [b"Synapse"] headers_dict[b"Host"] = [destination] - logger.debug("Sending request to %s: %s %s;%s?%s", - destination, method, path_bytes, param_bytes, query_bytes) + url_bytes = urlparse.urlunparse( + ("", "", path_bytes, param_bytes, query_bytes, "",) + ) + + logger.debug("Sending request to %s: %s %s", + destination, method, url_bytes) logger.debug( "Types: %s", @@ -93,8 +98,8 @@ class BaseHttpClient(object): endpoint = self._getEndpoint(reactor, destination); while True: - if on_send_callback: - on_send_callback(destination, method, path_bytes, producer) + + producer = body_callback(method, url_bytes, headers_dict) try: response = yield self.agent.request( @@ -167,13 +172,22 @@ class MatrixHttpClient(BaseHttpClient): will be the decoded JSON body. On a 4xx or 5xx error response a CodeMessageException is raised. """ + + if not on_send_callback: + def on_send_callback(destination, method, path_bytes, producer): + pass + + def body_callback(method, url_bytes, headers_dict): + producer = _JsonProducer(data) + on_send_callback(destination, method, path, producer) + return producer + response = yield self._create_request( destination.encode("ascii"), "PUT", path.encode("ascii"), - producer=_JsonProducer(data), + body_callback=body_callback, headers_dict={"Content-Type": ["application/json"]}, - on_send_callback=on_send_callback, ) logger.debug("Getting resp body") @@ -206,11 +220,15 @@ class MatrixHttpClient(BaseHttpClient): query_bytes = urllib.urlencode(args, True) logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) + def body_callback(method, url_bytes, headers_dict): + return None + response = yield self._create_request( destination.encode("ascii"), "GET", path.encode("ascii"), query_bytes=query_bytes, + body_callback=body_callback, retry_on_dns_fail=retry_on_dns_fail ) @@ -239,11 +257,14 @@ class IdentityServerHttpClient(BaseHttpClient): logger.debug("post_urlencoded_get_json args: %s", args) query_bytes = urllib.urlencode(args, True) + def body_callback(method, url_bytes, headers_dict): + return FileBodyProducer(StringIO(query_bytes)) + response = yield self._create_request( destination.encode("ascii"), "POST", path.encode("ascii"), - producer=FileBodyProducer(StringIO(query_bytes)), + body_callback=body_callback, headers_dict={ "Content-Type": ["application/x-www-form-urlencoded"] } @@ -265,11 +286,14 @@ class CaptchaServerHttpClient(MatrixHttpClient): args={}): query_bytes = urllib.urlencode(args, True) + def body_callback(method, url_bytes, headers_dict): + return FileBodyProducer(StringIO(query_bytes)) + response = yield self._create_request( destination.encode("ascii"), "POST", path.encode("ascii"), - producer=FileBodyProducer(StringIO(query_bytes)), + body_callback=body_callback, headers_dict={ "Content-Type": ["application/x-www-form-urlencoded"] } -- cgit 1.4.1 From 10ef8e6e4bb9d50fd2c636cfbb66d3dd6d6f94e9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 13 Oct 2014 11:49:40 +0100 Subject: SYN-75 sign at the request level rather than the transaction level --- synapse/federation/replication.py | 13 ---------- synapse/federation/transport.py | 18 +++---------- synapse/federation/units.py | 5 ++++ synapse/http/client.py | 52 ++++++++++++++++++++++++++++++++----- tests/federation/test_federation.py | 4 +-- tests/handlers/test_presence.py | 26 +++++++++---------- tests/handlers/test_typing.py | 4 +-- 7 files changed, 70 insertions(+), 52 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index b4235585a3..2346d55045 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -25,8 +25,6 @@ from .persistence import PduActions, TransactionActions from synapse.util.logutils import log_function -from syutil.crypto.jsonsign import sign_json - import logging @@ -66,8 +64,6 @@ class ReplicationLayer(object): hs, self.transaction_actions, transport_layer ) - self.keyring = hs.get_keyring() - self.handler = None self.edu_handlers = {} self.query_handlers = {} @@ -296,10 +292,6 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function def on_incoming_transaction(self, transaction_data): - yield self.keyring.verify_json_for_server( - transaction_data["origin"], transaction_data - ) - transaction = Transaction(**transaction_data) for p in transaction.pdus: @@ -500,7 +492,6 @@ class _TransactionQueue(object): """ def __init__(self, hs, transaction_actions, transport_layer): - self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.transaction_actions = transaction_actions self.transport_layer = transport_layer @@ -615,9 +606,6 @@ class _TransactionQueue(object): # Actually send the transaction - server_name = self.server_name - signing_key = self.signing_key - # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def json_data_cb(): @@ -627,7 +615,6 @@ class _TransactionQueue(object): for p in data["pdus"]: if "age_ts" in p: p["age"] = now - int(p["age_ts"]) - data = sign_json(data, server_name, signing_key) return data code, response = yield self.transport_layer.send_transaction( diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 1f864f5fa7..48fc9fbf5e 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -163,27 +163,15 @@ class TransportLayer(object): if transaction.destination == self.server_name: raise RuntimeError("Transport layer cannot send to itself!") - if json_data_callback is None: - def json_data_callback(): - return transaction.get_dict() - - # FIXME (erikj): This is a bit of a hack to make the Pdu age - # keys work - def cb(destination, method, path_bytes, producer): - json_data = json_data_callback() - del json_data["destination"] - del json_data["transaction_id"] - producer.reset(json_data) - + # FIXME: This is only used by the tests. The actual json sent is + # generated by the json_data_callback. json_data = transaction.get_dict() - del json_data["destination"] - del json_data["transaction_id"] code, response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, data=json_data, - on_send_callback=cb, + json_data_callback=json_data_callback, ) logger.debug( diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 1ca123d1bf..ecca35ac43 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -190,6 +190,11 @@ class Transaction(JsonEncodedObject): "destination", ] + internal_keys = [ + "transaction_id", + "destination", + ] + required_keys = [ "transaction_id", "origin", diff --git a/synapse/http/client.py b/synapse/http/client.py index 0e8fa2eb25..62fe14fa5e 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -26,6 +26,8 @@ from syutil.jsonutil import encode_canonical_json from synapse.api.errors import CodeMessageException, SynapseError +from syutil.crypto.jsonsign import sign_json + from StringIO import StringIO import json @@ -147,7 +149,7 @@ class BaseHttpClient(object): class MatrixHttpClient(BaseHttpClient): - """ Wrapper around the twisted HTTP client api. Implements + """ Wrapper around the twisted HTTP client api. Implements Attributes: agent (twisted.web.client.Agent): The twisted Agent used to send the @@ -156,8 +158,38 @@ class MatrixHttpClient(BaseHttpClient): RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" + def __init__(self, hs): + self.signing_key = hs.config.signing_key[0] + self.server_name = hs.hostname + BaseHttpClient.__init__(self, hs) + + def sign_request(self, destination, method, url_bytes, headers_dict, + content=None): + request = { + "method": method, + "uri": url_bytes, + "origin": self.server_name, + "destination": destination, + } + + if content is not None: + request["content"] = content + + request = sign_json(request, self.server_name, self.signing_key) + + auth_headers = [] + + for key,sig in request["signatures"][self.server_name].items(): + auth_headers.append( + "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( + self.server_name, key, sig, + ) + ) + + headers_dict["Authorization"] = auth_headers + @defer.inlineCallbacks - def put_json(self, destination, path, data, on_send_callback=None): + def put_json(self, destination, path, data={}, json_data_callback=None): """ Sends the specifed json data using PUT Args: @@ -166,6 +198,8 @@ class MatrixHttpClient(BaseHttpClient): path (str): The HTTP path. data (dict): A dict containing the data that will be used as the request body. This will be encoded as JSON. + json_data_callback (callable): A callable returning the dict to + use as the request body. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result @@ -173,13 +207,16 @@ class MatrixHttpClient(BaseHttpClient): CodeMessageException is raised. """ - if not on_send_callback: - def on_send_callback(destination, method, path_bytes, producer): - pass + if not json_data_callback: + def json_data_callback(): + return data def body_callback(method, url_bytes, headers_dict): - producer = _JsonProducer(data) - on_send_callback(destination, method, path, producer) + json_data = json_data_callback() + self.sign_request( + destination, method, url_bytes, headers_dict, json_data + ) + producer = _JsonProducer(json_data) return producer response = yield self._create_request( @@ -221,6 +258,7 @@ class MatrixHttpClient(BaseHttpClient): logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) def body_callback(method, url_bytes, headers_dict): + self.sign_request(destination, method, url_bytes, headers_dict) return None response = yield self._create_request( diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 01f07ff36d..91edeaa4b9 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -186,7 +186,7 @@ class FederationTestCase(unittest.TestCase): }, ] }, - on_send_callback=ANY, + json_data_callback=ANY, ) @defer.inlineCallbacks @@ -218,7 +218,7 @@ class FederationTestCase(unittest.TestCase): } ], }, - on_send_callback=ANY, + json_data_callback=ANY, ) @defer.inlineCallbacks diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index c9406b70c4..15022b8d05 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -300,7 +300,7 @@ class PresenceInvitesTestCase(unittest.TestCase): "observed_user": "@cabbage:elsewhere", } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -329,7 +329,7 @@ class PresenceInvitesTestCase(unittest.TestCase): "observed_user": "@apple:test", } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -365,7 +365,7 @@ class PresenceInvitesTestCase(unittest.TestCase): "observed_user": "@durian:test", } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -786,7 +786,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -802,7 +802,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -928,7 +928,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -943,7 +943,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -973,7 +973,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1175,7 +1175,7 @@ class PresencePollingTestCase(unittest.TestCase): "poll": [ "@potato:remote" ], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1188,7 +1188,7 @@ class PresencePollingTestCase(unittest.TestCase): "push": [ {"user_id": "@clementine:test" }], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1217,7 +1217,7 @@ class PresencePollingTestCase(unittest.TestCase): "push": [ {"user_id": "@fig:test" }], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1250,7 +1250,7 @@ class PresencePollingTestCase(unittest.TestCase): "unpoll": [ "@potato:remote" ], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1282,7 +1282,7 @@ class PresencePollingTestCase(unittest.TestCase): ], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index ff40343b8c..064b04c217 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -175,7 +175,7 @@ class TypingNotificationsTestCase(unittest.TestCase): "typing": True, } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -226,7 +226,7 @@ class TypingNotificationsTestCase(unittest.TestCase): "typing": False, } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) -- cgit 1.4.1 From 66848557672977e17f0d5ba785b7567b305ccdbe Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 13 Oct 2014 14:37:46 +0100 Subject: Verify signatures for server2server requests --- synapse/federation/__init__.py | 1 + synapse/federation/transport.py | 110 ++++++++++++++++++++++++++++-------- synapse/http/client.py | 10 +++- tests/federation/test_federation.py | 1 + tests/utils.py | 3 + 5 files changed, 100 insertions(+), 25 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/federation/__init__.py b/synapse/federation/__init__.py index 1351b68fd6..0112588656 100644 --- a/synapse/federation/__init__.py +++ b/synapse/federation/__init__.py @@ -22,6 +22,7 @@ from .transport import TransportLayer def initialize_http_replication(homeserver): transport = TransportLayer( + homeserver, homeserver.hostname, server=homeserver.get_resource_for_federation(), client=homeserver.get_http_client() diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 48fc9fbf5e..7b2631fbc8 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -54,7 +54,7 @@ class TransportLayer(object): we receive data. """ - def __init__(self, server_name, server, client): + def __init__(self, homeserver, server_name, server, client): """ Args: server_name (str): Local home server host @@ -63,6 +63,7 @@ class TransportLayer(object): client (synapse.protocol.http.HttpClient): the http client used to send requests """ + self.keyring = homeserver.get_keyring() self.server_name = server_name self.server = server self.client = client @@ -195,6 +196,66 @@ class TransportLayer(object): defer.returnValue(response) + @defer.inlineCallbacks + def _authenticate_request(self, request): + json_request = { + "method": request.method, + "uri": request.uri, + "destination": self.server_name, + "signatures": {}, + } + + content = None + origin = None + + if request.method == "PUT": + #TODO: Handle other method types? other content types? + content_bytes = request.content.read() + content = json.loads(content_bytes) + json_request["content"] = content + + def parse_auth_header(header_str): + params = auth.split(" ")[1].split(",") + param_dict = dict(kv.split("=") for kv in params) + def strip_quotes(value): + if value.startswith("\""): + return value[1:-1] + else: + return value + origin = strip_quotes(param_dict["origin"]) + key = strip_quotes(param_dict["key"]) + sig = strip_quotes(param_dict["sig"]) + return (origin, key, sig) + + auth_headers = request.requestHeaders.getRawHeaders(b"Authorization") + + if not auth_headers: + #TODO(markjh): Send a 401 response? + raise Exception("Missing auth headers") + + for auth in auth_headers: + if auth.startswith("X-Matrix"): + (origin, key, sig) = parse_auth_header(auth) + json_request["origin"] = origin + json_request["signatures"].setdefault(origin,{})[key] = sig + + from syutil.jsonutil import encode_canonical_json + logger.debug("Checking %s %s", + origin, encode_canonical_json(json_request)) + yield self.keyring.verify_json_for_server(origin, json_request) + + defer.returnValue((origin, content)) + + def _with_authentication(self, handler): + @defer.inlineCallbacks + def new_handler(request, *args, **kwargs): + (origin, content) = yield self._authenticate_request(request) + response = yield handler( + origin, content, request.args, *args, **kwargs + ) + defer.returnValue(response) + return new_handler + @log_function def register_received_handler(self, handler): """ Register a handler that will be fired when we receive data. @@ -208,7 +269,7 @@ class TransportLayer(object): self.server.register_path( "PUT", re.compile("^" + PREFIX + "/send/([^/]*)/$"), - self._on_send_request + self._with_authentication(self._on_send_request) ) @log_function @@ -226,9 +287,9 @@ class TransportLayer(object): self.server.register_path( "GET", re.compile("^" + PREFIX + "/pull/$"), - lambda request: handler.on_pull_request( - request.args["origin"][0], - request.args["v"] + self._with_authentication( + lambda origin, content, query: + handler.on_pull_request(query["origin"][0], query["v"]) ) ) @@ -237,8 +298,9 @@ class TransportLayer(object): self.server.register_path( "GET", re.compile("^" + PREFIX + "/pdu/([^/]*)/([^/]*)/$"), - lambda request, pdu_origin, pdu_id: handler.on_pdu_request( - pdu_origin, pdu_id + self._with_authentication( + lambda origin, content, query, pdu_origin, pdu_id: + handler.on_pdu_request(pdu_origin, pdu_id) ) ) @@ -246,38 +308,47 @@ class TransportLayer(object): self.server.register_path( "GET", re.compile("^" + PREFIX + "/state/([^/]*)/$"), - lambda request, context: handler.on_context_state_request( - context + self._with_authentication( + lambda origin, content, query, context: + handler.on_context_state_request(context) ) ) self.server.register_path( "GET", re.compile("^" + PREFIX + "/backfill/([^/]*)/$"), - lambda request, context: self._on_backfill_request( - context, request.args["v"], - request.args["limit"] + self._with_authentication( + lambda origin, content, query, context: + self._on_backfill_request( + context, query["v"], query["limit"] + ) ) ) self.server.register_path( "GET", re.compile("^" + PREFIX + "/context/([^/]*)/$"), - lambda request, context: handler.on_context_pdus_request(context) + self._with_authentication( + lambda origin, content, query, context: + handler.on_context_pdus_request(context) + ) ) # This is when we receive a server-server Query self.server.register_path( "GET", re.compile("^" + PREFIX + "/query/([^/]*)$"), - lambda request, query_type: handler.on_query_request( - query_type, {k: v[0] for k, v in request.args.items()} + self._with_authentication( + lambda origin, content, query, query_type: + handler.on_query_request( + query_type, {k: v[0] for k, v in query.items()} + ) ) ) @defer.inlineCallbacks @log_function - def _on_send_request(self, request, transaction_id): + def _on_send_request(self, origin, content, query, transaction_id): """ Called on PUT /send// Args: @@ -292,12 +363,7 @@ class TransportLayer(object): """ # Parse the request try: - data = request.content.read() - - l = data[:20].encode("string_escape") - logger.debug("Got data: \"%s\"", l) - - transaction_data = json.loads(data) + transaction_data = content logger.debug( "Decoded %s: %s", diff --git a/synapse/http/client.py b/synapse/http/client.py index 62fe14fa5e..9f54b74e3a 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -177,16 +177,20 @@ class MatrixHttpClient(BaseHttpClient): request = sign_json(request, self.server_name, self.signing_key) + from syutil.jsonutil import encode_canonical_json + logger.debug("Signing " + " " * 11 + "%s %s", + self.server_name, encode_canonical_json(request)) + auth_headers = [] for key,sig in request["signatures"][self.server_name].items(): - auth_headers.append( + auth_headers.append(bytes( "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( self.server_name, key, sig, ) - ) + )) - headers_dict["Authorization"] = auth_headers + headers_dict[b"Authorization"] = auth_headers @defer.inlineCallbacks def put_json(self, destination, path, data={}, json_data_callback=None): diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 91edeaa4b9..8d277d6612 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -221,6 +221,7 @@ class FederationTestCase(unittest.TestCase): json_data_callback=ANY, ) + @defer.inlineCallbacks def test_recv_edu(self): recv_observer = Mock() diff --git a/tests/utils.py b/tests/utils.py index 797818be72..83dbd4f4d3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -76,6 +76,9 @@ class MockHttpResource(HttpServer): mock_content.configure_mock(**config) mock_request.content = mock_content + mock_request.method = http_method + mock_request.uri = path + # return the right path if the event requires it mock_request.path = path -- cgit 1.4.1 From f74e850b5cf7947fbbd13d8bfd1daf43d535741f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Oct 2014 11:46:13 +0100 Subject: remove debugging logging for signing requests --- synapse/http/client.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/http/client.py b/synapse/http/client.py index 9f54b74e3a..316ca1ccb9 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -177,10 +177,6 @@ class MatrixHttpClient(BaseHttpClient): request = sign_json(request, self.server_name, self.signing_key) - from syutil.jsonutil import encode_canonical_json - logger.debug("Signing " + " " * 11 + "%s %s", - self.server_name, encode_canonical_json(request)) - auth_headers = [] for key,sig in request["signatures"][self.server_name].items(): -- cgit 1.4.1